YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 数据处理向导配置体系扩展

Browse Source 
新增结构化与非结构化处理选项类型，TaskSetupStep 增加预处理、切分方法、数据集划分等配置 UI，previewModel 实现语义切分与受保护区间算法，CreateView 接入配置状态与草稿持久化并替换为 AppConfirmDialog，回归脚本扩充配置与弹窗断言。
This commit is contained in:
caoxiaozhu
2026-07-11 14:49:10 +08:00
parent 9e16f6358d
commit ddf47eb8fe
5 changed files with 1941 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

281
frontend/src/views/data-process/DataProcessCreateView.vue
View File

@@ -1,17 +1,30 @@
<script setup lang="ts">
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, ElMessageBox, type UploadFile } from 'element-plus'
import { ElMessage, type UploadFile } from 'element-plus'
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
import TaskSetupStep from './create/TaskSetupStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import type { ExternalDataSource, GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
import type {
ExternalDataSource,
GenerationState,
PreviewItem,
ProcessType,
ResultItem,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './create/types'
const router = useRouter()
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
const DRAFT_SCHEMA_VERSION = 2
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与上传源数据' },
@@ -24,6 +37,36 @@ const currentStep = ref(0)
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
const task = reactive({ name: '', description: '' })
const processType = ref<ProcessType>('unstructured')
const structuredOptions = ref<StructuredProcessOptions>({
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
})
const unstructuredOptions = ref<UnstructuredProcessOptions>({
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
chunkMethod: 'semantic',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
customDelimiter: '',
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
semanticEnrichment: false,
qaPairsPerChunk: 1,
contextScope: 'adjacent',
generationTypes: ['factual', 'concept', 'comprehensive'],
skipUnanswerable: true,
datasetSplit: { train: 80, validation: 10, test: 10 },
})
interface UploadedDataFile {
uid: number | string
name: string
@@ -103,9 +146,21 @@ const previousStepLabel = computed(() => currentStep.value > 0
function draftSnapshot() {
return {
schemaVersion: DRAFT_SCHEMA_VERSION,
currentStep: currentStep.value,
task: { ...task },
processType: processType.value,
structuredOptions: {
...structuredOptions.value,
preprocessOptions: [...structuredOptions.value.preprocessOptions],
datasetSplit: { ...structuredOptions.value.datasetSplit },
},
unstructuredOptions: {
...unstructuredOptions.value,
preprocessOptions: [...unstructuredOptions.value.preprocessOptions],
generationTypes: [...unstructuredOptions.value.generationTypes],
datasetSplit: { ...unstructuredOptions.value.datasetSplit },
},
uploadedFiles: uploadedFiles.value,
externalSource: { ...externalSource },
previewSignature: previewSignature.value,
@@ -119,6 +174,74 @@ function draftSnapshot() {
}
}
function previewAffectingOptions() {
if (processType.value === 'structured') {
return {
preprocessOptions: structuredOptions.value.preprocessOptions,
}
}
if (processType.value === 'unstructured') {
const {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
customDelimiter,
preserveTables,
preserveCodeBlocks,
preserveLists,
} = unstructuredOptions.value
return {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
customDelimiter: chunkMethod === 'custom' ? customDelimiter : '',
preserveTables,
preserveCodeBlocks,
preserveLists,
}
}
return null
}
function generationAffectingOptions() {
if (processType.value === 'structured') {
const { semanticEnrichment, qaPairsPerRow, datasetSplit } = structuredOptions.value
return { semanticEnrichment, qaPairsPerRow, datasetSplit }
}
if (processType.value === 'unstructured') {
const {
semanticEnrichment,
qaPairsPerChunk,
contextScope,
generationTypes,
skipUnanswerable,
datasetSplit,
} = unstructuredOptions.value
return {
semanticEnrichment,
qaPairsPerChunk,
contextScope,
generationTypes,
skipUnanswerable,
datasetSplit,
}
}
return null
}
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
function buildPreviewSignature() {
const filesSignature = uploadedFiles.value
.map((file) => `${file.uid}:${file.content}`)
.join('|')
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
}
function persistDraft() {
if (restoringDraft.value) return
try {
@@ -128,7 +251,7 @@ function persistDraft() {
}
}
type DraftSnapshot = ReturnType<typeof draftSnapshot> & {
type DraftSnapshot = Partial<ReturnType<typeof draftSnapshot>> & {
fileName?: string
fileSize?: number
fileCount?: number
@@ -141,14 +264,46 @@ function restoreDraft() {
if (!raw) return
const snapshot = JSON.parse(raw) as DraftSnapshot
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
const requiresPreviewMigration = snapshot.schemaVersion !== DRAFT_SCHEMA_VERSION
restoringDraft.value = true
currentStep.value = Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
currentStep.value = requiresPreviewMigration
? 0
: Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
task.name = snapshot.task?.name || ''
task.description = snapshot.task?.description || ''
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
? snapshot.processType
: 'unstructured'
if (snapshot.structuredOptions) {
structuredOptions.value = {
...structuredOptions.value,
...snapshot.structuredOptions,
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
? snapshot.structuredOptions.preprocessOptions
: structuredOptions.value.preprocessOptions,
datasetSplit: {
...structuredOptions.value.datasetSplit,
...snapshot.structuredOptions.datasetSplit,
},
}
}
if (snapshot.unstructuredOptions) {
unstructuredOptions.value = {
...unstructuredOptions.value,
...snapshot.unstructuredOptions,
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
? snapshot.unstructuredOptions.preprocessOptions
: unstructuredOptions.value.preprocessOptions,
generationTypes: Array.isArray(snapshot.unstructuredOptions.generationTypes)
? snapshot.unstructuredOptions.generationTypes
: unstructuredOptions.value.generationTypes,
datasetSplit: {
...unstructuredOptions.value.datasetSplit,
...snapshot.unstructuredOptions.datasetSplit,
},
}
}
if (snapshot.uploadedFiles) {
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
@@ -163,41 +318,50 @@ function restoreDraft() {
}]
}
previewSignature.value = snapshot.previewSignature || ''
previewSignature.value = requiresPreviewMigration ? '' : snapshot.previewSignature || ''
if (snapshot.externalSource) {
Object.assign(externalSource, snapshot.externalSource)
}
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
previewItems.value = Array.isArray(snapshot.previewItems)
previewItems.value = !requiresPreviewMigration && Array.isArray(snapshot.previewItems)
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
: []
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
selectedPreviewIdsByFile.value = snapshot.selectedPreviewIdsByFile || {}
selectedPreviewId.value = snapshot.selectedPreviewId
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|| activePreviewItems.value[0]?.id
|| null
results.value = Array.isArray(snapshot.results) ? snapshot.results : []
selectedResultId.value = snapshot.selectedResultId || results.value[0]?.id || null
Object.assign(generation, snapshot.generation || {})
selectedPreviewIdsByFile.value = requiresPreviewMigration ? {} : snapshot.selectedPreviewIdsByFile || {}
selectedPreviewId.value = requiresPreviewMigration
? null
: snapshot.selectedPreviewId
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|| activePreviewItems.value[0]?.id
|| null
results.value = !requiresPreviewMigration && Array.isArray(snapshot.results) ? snapshot.results : []
selectedResultId.value = requiresPreviewMigration ? null : snapshot.selectedResultId || results.value[0]?.id || null
if (!requiresPreviewMigration) Object.assign(generation, snapshot.generation || {})
dirty.value = false
nextTick(() => { restoringDraft.value = false })
ElMessage.info('已恢复上次保存的草稿')
ElMessage.info(requiresPreviewMigration
? '已恢复任务信息，切分规则已更新，请重新生成预览'
: '已恢复上次保存的草稿')
} catch {
localStorage.removeItem(DRAFT_STORAGE_KEY)
}
}
watch(
[() => task.name, () => task.description, processType, externalSource],
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
() => {
if (!restoringDraft.value) dirty.value = true
},
{ deep: true },
)
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (restoringDraft.value || currentSignature === previousSignature) return
resetDownstream()
})
watch(
[currentStep, task, processType, uploadedFiles, externalSource, previewSignature,
[currentStep, task, processType, structuredOptions, unstructuredOptions, uploadedFiles, externalSource, previewSignature,
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
results, selectedResultId, generation],
persistDraft,
@@ -330,10 +494,15 @@ async function nextFromCreate() {
return
}
const signature = `${processType.value}:${uploadedFiles.value.map((file) => `${file.uid}:${file.content}`).join('|')}`
const signature = buildPreviewSignature()
if (signature !== previewSignature.value) {
previewItems.value = uploadedFiles.value.flatMap((file) =>
buildPreviewItems(file.content, processType.value, String(file.uid)),
buildPreviewItems(
file.content,
processType.value,
String(file.uid),
processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
),
)
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
@@ -367,6 +536,7 @@ function updatePreviewContent(id: string, value: string) {
item.editedContent = value
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
resetDownstream()
dirty.value = true
}
@@ -376,6 +546,7 @@ function restorePreviewItem(id: string) {
item.editedContent = item.originalContent
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
item.status = 'original'
resetDownstream()
dirty.value = true
}
@@ -395,19 +566,19 @@ function addPreviewItem() {
status: 'manual',
})
selectPreviewItem(id)
resetDownstream()
dirty.value = true
}
async function removePreviewItem(id: string) {
try {
await ElMessageBox.confirm('删除只影响本次处理，不会修改源文件。确认删除吗？', '删除预览内容', {
confirmButtonText: '删除',
cancelButtonText: '取消',
type: 'warning',
})
} catch {
return
}
const confirmed = await confirmDialogRef.value?.open({
title: '删除预览内容？',
message: '删除只影响本次处理，不会修改源文件。删除后可重新从源文件生成预览。',
confirmText: '删除',
cancelText: '取消',
tone: 'danger',
})
if (!confirmed) return
const index = previewItems.value.findIndex((item) => item.id === id)
if (index < 0) return
@@ -416,6 +587,7 @@ async function removePreviewItem(id: string) {
if (selectedPreviewFileId.value && selectedPreviewId.value) {
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
}
resetDownstream()
dirty.value = true
}
@@ -436,8 +608,15 @@ function startGeneration() {
stopGenerationTimer()
generation.status = 'success'
generation.message = `已完成 ${previewItems.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
results.value = createResults(previewItems.value)
results.value = createResults(
previewItems.value,
processType.value === 'structured'
? structuredOptions.value
: processType.value === 'unstructured'
? unstructuredOptions.value
: undefined,
)
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
selectedResultId.value = results.value[0]?.id ?? null
dirty.value = true
ElMessage.success('数据处理完成')
@@ -538,25 +717,29 @@ async function handleCancel() {
router.back()
return
}
try {
await ElMessageBox.confirm('当前存在未保存修改，确定离开吗？', '离开创建任务', {
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
type: 'warning',
})
allowLeave = true
router.back()
} catch {
// 用户继续编辑。
}
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面？',
message: '当前存在未保存修改，离开后这些修改将不会保留。',
confirmText: '放弃修改',
cancelText: '继续编辑',
tone: 'danger',
})
if (!confirmed) return
allowLeave = true
router.back()
}
onBeforeRouteLeave((_to, _from, next) => {
if (allowLeave || !dirty.value) {
next()
return
}
next(window.confirm('当前存在未保存修改，确定离开吗？'))
onBeforeRouteLeave(async () => {
if (allowLeave || !dirty.value) return true
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面？',
message: '当前存在未保存修改，离开后这些修改将不会保留。',
confirmText: '放弃修改',
cancelText: '继续编辑',
tone: 'danger',
})
if (confirmed) allowLeave = true
return Boolean(confirmed)
})
onBeforeUnmount(stopGenerationTimer)
@@ -599,6 +782,8 @@ onMounted(restoreDraft)
v-model:name="task.name"
v-model:description="task.description"
v-model:process-type="processType"
v-model:structured-options="structuredOptions"
v-model:unstructured-options="unstructuredOptions"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
@@ -673,6 +858,8 @@ onMounted(restoreDraft)
</div>
</footer>
</div>
<AppConfirmDialog ref="confirmDialogRef" />
</template>
<style scoped lang="scss">