Files
YG_FT/frontend/src/views/data-process/DataProcessCreateView.vue
caoxiaozhu 762d48d090 feat: 数据处理向导新增模型配置步骤
StepId 新增 model 步骤,DataProcessCreateView 增加模型选择与生成参数编排,各选项面板与样式适配新步骤流程,回归脚本补充断言。
2026-07-13 17:12:49 +08:00

754 lines
24 KiB
Vue
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<script setup lang="ts">
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, type UploadFile } from 'element-plus'
import { storeToRefs } from 'pinia'
import AppConfirmDialog from '@/components/AppConfirmDialog.vue'
import TaskSetupStep from './create/TaskSetupStep.vue'
import ModelSelectionStep from './create/ModelSelectionStep.vue'
import SourceUploadStep from './create/SourceUploadStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import {
createDefaultStructuredOptions,
createDefaultUnstructuredOptions,
} from './create/dataProcessCreateState'
import {
DATA_PROCESS_DRAFT_STORAGE_KEY,
useDataProcessDraft,
} from './create/useDataProcessDraft'
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useModelsStore } from '@/stores/models'
import type {
ExternalDataSource,
GenerationControlOptions,
PreviewItem,
ProcessType,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
UploadedDataFile,
} from './create/types'
const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList } = storeToRefs(modelsStore)
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const modelSelectionRef = ref<InstanceType<typeof ModelSelectionStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
{ id: 'upload', title: '上传文件', desc: '上传或接入待处理的源数据' },
{ id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
{ id: 'results', title: '编辑与保存', desc: '检查、修改并保存结果' },
] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }>
const currentStep = ref(0)
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
const task = reactive({ name: '', description: '' })
const processType = ref<ProcessType>('structured')
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
const modelSelectionOptions = computed<GenerationControlOptions>(() => (
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
))
const uploadedFiles = ref<UploadedDataFile[]>([])
const externalSource = reactive<ExternalDataSource>({
type: 'mysql',
url: '',
authMode: 'none',
username: '',
password: '',
token: '',
limit: 1000,
})
const externalPulling = ref(false)
const externalConnected = ref(false)
let connectionTimer: ReturnType<typeof setTimeout> | null = null
let pullTimer: ReturnType<typeof setTimeout> | null = null
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
const previewSignature = ref('')
const previewItems = ref<PreviewItem[]>([])
const selectedPreviewFileId = ref<string | null>(null)
const selectedPreviewId = ref<string | null>(null)
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
const dirty = ref(false)
const restoringDraft = ref(false)
let allowLeave = false
const {
generation,
results,
selectedResultId,
resetDownstream,
restoreResult,
startGeneration,
stopGeneration,
stopGenerationTimer,
updateResultField,
validateResults,
} = useDataProcessGeneration({
previewItems,
processType,
structuredOptions,
unstructuredOptions,
dirty,
})
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
const previewItemsByFile = computed(() => {
const grouped = new Map<string, PreviewItem[]>()
for (const item of previewItems.value) {
const items = grouped.get(item.sourceFileId) || []
items.push(item)
grouped.set(item.sourceFileId, items)
}
return grouped
})
const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value))
const activePreviewItems = computed(() => previewItemsByFile.value.get(selectedPreviewFileId.value || '') || [])
const activeSourceText = computed(() => activePreviewFile.value?.content ?? '')
const previewFiles = computed(() => uploadedFiles.value.map((file) => {
const items = previewItemsByFile.value.get(String(file.uid)) || []
return {
id: String(file.uid),
name: file.name,
count: items.length,
modifiedCount: items.filter((item) => item.status !== 'original').length,
}
}))
const primaryActionLabel = computed(() => {
if (currentStepId.value === 'create') return '继续:选择大模型'
if (currentStepId.value === 'model') return '继续:上传文件'
if (currentStepId.value === 'upload') return '继续:数据预览'
if (currentStepId.value === 'preview') return '确认预览并继续'
if (currentStepId.value === 'results') return '保存任务'
if (generation.status === 'running') return '正在生成'
if (generation.status === 'success') return '查看生成结果'
if (generation.status === 'failed') return '重新生成'
return '开始生成'
})
const primaryActionIcon = computed(() => {
if (currentStepId.value === 'results') return 'fa-check'
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
return 'fa-arrow-right'
})
const previousStepLabel = computed(() => currentStep.value > 0
? WIZARD_STEPS[currentStep.value - 1].title
: '')
function goToStep(stepId: StepId) {
const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
if (nextStepIndex >= 0) currentStep.value = nextStepIndex
}
function updateModelSelectionOptions(value: GenerationControlOptions) {
if (processType.value === 'unstructured') {
unstructuredOptions.value = { ...unstructuredOptions.value, ...value }
return
}
structuredOptions.value = { ...structuredOptions.value, ...value }
}
const { persistDraft, restoreDraft } = useDataProcessDraft({
currentStepId,
task,
processType,
structuredOptions,
unstructuredOptions,
externalSource,
restoringDraft,
dirty,
goToStep,
})
function previewAffectingOptions() {
if (processType.value === 'structured') {
return {
preprocessOptions: structuredOptions.value.preprocessOptions,
}
}
if (processType.value === 'unstructured') {
const {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
customDelimiter,
preserveTables,
preserveCodeBlocks,
preserveLists,
} = unstructuredOptions.value
return {
preprocessOptions,
chunkMethod,
chunkSize,
chunkOverlap,
minChunkSize,
customDelimiter: chunkMethod === 'custom' ? customDelimiter : '',
preserveTables,
preserveCodeBlocks,
preserveLists,
}
}
return null
}
function generationAffectingOptions() {
if (processType.value === 'structured') {
const {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
} = structuredOptions.value
return {
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
temperature, maxTokens, jsonMode,
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
}
}
if (processType.value === 'unstructured') {
const {
semanticEnrichment,
qaPairsPerChunk,
datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
} = unstructuredOptions.value
return {
semanticEnrichment,
qaPairsPerChunk,
datasetSplit,
generationModelId,
generationPrompt,
temperature,
maxTokens,
jsonMode,
qualityFilterEnabled,
filterLowQuality,
filterShortContent,
minOutputLength,
}
}
return null
}
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
function buildPreviewSignature() {
const filesSignature = uploadedFiles.value
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`)
.join('|')
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
}
watch(
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
() => {
if (!restoringDraft.value) dirty.value = true
},
{ deep: true },
)
watch(processType, (nextType, previousType) => {
if (restoringDraft.value || nextType === previousType || uploadedFiles.value.length === 0) return
resetSourceDataForProcessTypeChange()
ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据')
})
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (restoringDraft.value || currentSignature === previousSignature) return
resetDownstream()
})
watch(
[task, processType, structuredOptions, unstructuredOptions, externalSource],
persistDraft,
{ deep: true },
)
function scrollToStepTop() {
document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
}
watch(currentStep, () => nextTick(scrollToStepTop))
async function handleFileChange(uploadFile: UploadFile) {
const raw = uploadFile.raw
if (!raw) return
if (raw.size > 200 * 1024 * 1024) {
ElMessage.warning('单文件不能超过 200MB')
return
}
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
const textExtensions = ['txt', 'md', 'json', 'jsonl', 'csv']
let content = ''
if (textExtensions.includes(extension)) {
try {
content = await raw.text()
} catch {
content = ''
}
}
const fileContent = content.trim() ? content : DEFAULT_SOURCE_TEXT
const linesCount = fileContent.split('\n').filter((line) => line.trim()).length
// Prevent duplicate upload of the same file
if (!uploadedFiles.value.some(f => f.name === raw.name && f.size === raw.size)) {
uploadedFiles.value.push({
uid: uploadFile.uid || Date.now() + Math.random(),
name: raw.name,
size: raw.size,
count: linesCount,
content: fileContent
})
}
dirty.value = true
}
function useSampleFile() {
uploadedFiles.value = [{
uid: 'sample-1',
name: 'finance_qa.jsonl',
size: 128 * 1024 * 1024,
count: DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length,
content: DEFAULT_SOURCE_TEXT
}]
dirty.value = true
}
function updateExternalSource(value: ExternalDataSource) {
Object.assign(externalSource, value)
externalConnected.value = false
}
function handleTestConnection() {
if (!externalSource.url.trim()) {
ElMessage.warning('请先填写数据源地址')
return
}
if (connectionTimer) clearTimeout(connectionTimer)
externalPulling.value = true
connectionTimer = setTimeout(() => {
connectionTimer = null
externalPulling.value = false
externalConnected.value = true
ElMessage.success('数据源连接测试成功')
}, 1500)
}
function handlePullData() {
if (!externalSource.url.trim()) {
ElMessage.warning('请先填写数据源地址')
return
}
if (pullTimer) clearTimeout(pullTimer)
externalPulling.value = true
pullTimer = setTimeout(() => {
pullTimer = null
externalPulling.value = false
externalConnected.value = true
const typeName = externalSource.type.toUpperCase()
const id = `external-${Date.now()}`
uploadedFiles.value.push({
uid: id,
name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
size: Math.min(externalSource.limit, 5000) * 64,
count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
content: DEFAULT_SOURCE_TEXT,
})
dirty.value = true
ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
}, 2000)
}
function handleRemoveFile(uid: string | number) {
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
if (index > -1) {
uploadedFiles.value.splice(index, 1)
previewSignature.value = ''
previewItems.value = []
selectedPreviewId.value = null
resetDownstream()
dirty.value = true
}
}
function resetSourceDataForProcessTypeChange() {
uploadedFiles.value = []
previewSignature.value = ''
previewItems.value = []
selectedPreviewFileId.value = null
selectedPreviewId.value = null
selectedPreviewIdsByFile.value = {}
externalConnected.value = false
resetDownstream()
}
async function nextFromCreate() {
const valid = await taskSetupRef.value?.validate()
if (!valid) return
goToStep('model')
}
async function nextFromModel() {
const valid = await modelSelectionRef.value?.validate()
if (!valid) return
goToStep('upload')
}
function nextFromUpload() {
if (uploadedFiles.value.length === 0) {
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
return
}
const signature = buildPreviewSignature()
if (signature !== previewSignature.value) {
previewItems.value = uploadedFiles.value.flatMap((file) =>
buildPreviewItems(
file.content,
processType.value,
String(file.uid),
processType.value === 'unstructured' ? unstructuredOptions.value : undefined,
),
)
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
selectedPreviewIdsByFile.value = selectedPreviewId.value && selectedPreviewFileId.value
? { [selectedPreviewFileId.value]: selectedPreviewId.value }
: {}
previewSignature.value = signature
resetDownstream()
}
goToStep('preview')
}
function selectPreviewFile(fileId: string) {
if (selectedPreviewFileId.value && selectedPreviewId.value) {
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
}
selectedPreviewFileId.value = fileId
selectedPreviewId.value = selectedPreviewIdsByFile.value[fileId]
|| activePreviewItems.value[0]?.id
|| null
}
function selectPreviewItem(id: string) {
selectedPreviewId.value = id
if (selectedPreviewFileId.value) selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = id
}
function updatePreviewContent(id: string, value: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item) return
item.editedContent = value
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
resetDownstream()
dirty.value = true
}
function restorePreviewItem(id: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item || item.sourceStart == null) return
item.editedContent = item.originalContent
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
item.status = 'original'
resetDownstream()
dirty.value = true
}
function addPreviewItem() {
if (!selectedPreviewFileId.value) return
const id = `manual-${Date.now()}`
previewItems.value.push({
id,
sourceFileId: selectedPreviewFileId.value,
originalContent: '',
editedContent: '',
sourceStart: null,
sourceEnd: null,
sourceStartLine: null,
sourceEndLine: null,
tokenCount: 1,
status: 'manual',
})
selectPreviewItem(id)
resetDownstream()
dirty.value = true
}
async function removePreviewItem(id: string) {
const confirmed = await confirmDialogRef.value?.open({
title: '删除预览内容?',
message: '删除只影响本次处理,不会修改源文件。删除后可重新从源文件生成预览。',
confirmText: '删除',
cancelText: '取消',
tone: 'danger',
})
if (!confirmed) return
const index = previewItems.value.findIndex((item) => item.id === id)
if (index < 0) return
previewItems.value.splice(index, 1)
selectedPreviewId.value = activePreviewItems.value[Math.min(index, activePreviewItems.value.length - 1)]?.id ?? null
if (selectedPreviewFileId.value && selectedPreviewId.value) {
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
}
resetDownstream()
dirty.value = true
}
async function handlePrimaryAction() {
if (currentStepId.value === 'create') {
await nextFromCreate()
return
}
if (currentStepId.value === 'model') {
await nextFromModel()
return
}
if (currentStepId.value === 'upload') {
nextFromUpload()
return
}
if (currentStepId.value === 'preview') {
if (!previewItems.value.length) {
ElMessage.warning('当前没有可生成的预览内容')
return
}
goToStep('generate')
return
}
if (currentStepId.value === 'generate') {
if (generation.status === 'success') {
goToStep('results')
} else if (generation.status !== 'running') {
startGeneration()
}
return
}
await saveTask()
}
function handleBack() {
if (generation.status === 'running') {
ElMessage.warning('请先停止当前生成任务')
return
}
if (currentStep.value > 0) currentStep.value -= 1
}
async function saveTask() {
if (!validateResults()) {
ElMessage.warning('请先修正校验失败的结果')
return
}
dirty.value = false
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
allowLeave = true
ElMessage.success('数据处理任务已保存')
await router.push('/data-process')
}
async function handleCancel() {
if (!dirty.value) {
allowLeave = true
router.back()
return
}
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?',
message: '当前存在未保存修改,离开后这些修改将不会保留。',
confirmText: '放弃修改',
cancelText: '继续编辑',
tone: 'danger',
})
if (!confirmed) return
allowLeave = true
router.back()
}
onBeforeRouteLeave(async () => {
if (allowLeave || !dirty.value) return true
const confirmed = await confirmDialogRef.value?.open({
title: '确认离开当前页面?',
message: '当前存在未保存修改,离开后这些修改将不会保留。',
confirmText: '放弃修改',
cancelText: '继续编辑',
tone: 'danger',
})
if (confirmed) allowLeave = true
return Boolean(confirmed)
})
onBeforeUnmount(() => {
stopGenerationTimer()
if (connectionTimer) clearTimeout(connectionTimer)
if (pullTimer) clearTimeout(pullTimer)
})
onMounted(() => {
restoreDraft()
modelsStore.load()
})
</script>
<template>
<div class="create-wizard-layout">
<main class="wizard-main">
<div class="wizard-main-inner">
<div class="wizard-steps-container">
<div class="custom-wizard-steps">
<template v-for="(step, index) in WIZARD_STEPS" :key="step.id">
<div
v-if="index !== 0"
class="step-connector"
:class="{ 'is-active': currentStep >= index }"
></div>
<div
class="step-item"
:class="{
'is-active': currentStep === index,
'is-completed': currentStep > index
}"
:aria-current="currentStep === index ? 'step' : undefined"
>
<div class="step-node">
<div class="step-icon">
<i v-if="currentStep > index" class="fa fa-check" />
<span v-else>{{ index + 1 }}</span>
</div>
<div class="step-text">
<div class="step-title">{{ step.title }}</div>
</div>
</div>
</div>
</template>
</div>
</div>
<div class="wizard-content">
<TaskSetupStep
v-if="currentStepId === 'create'"
ref="taskSetupRef"
v-model:name="task.name"
v-model:description="task.description"
v-model:process-type="processType"
v-model:structured-options="structuredOptions"
v-model:unstructured-options="unstructuredOptions"
/>
<ModelSelectionStep
v-else-if="currentStepId === 'model'"
ref="modelSelectionRef"
:options="modelSelectionOptions"
:models="generationModels"
@update:options="updateModelSelectionOptions"
/>
<SourceUploadStep
v-else-if="currentStepId === 'upload'"
:process-type="processType"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
:external-connected="externalConnected"
@update:external-source="updateExternalSource"
@file-change="handleFileChange"
@remove-file="handleRemoveFile"
@use-sample="useSampleFile"
@test-connection="handleTestConnection"
@pull-data="handlePullData"
/>
<PreviewCompareStep
v-else-if="currentStepId === 'preview'"
:selected-id="selectedPreviewId"
:selected-file-id="selectedPreviewFileId"
:source-text="activeSourceText"
:items="activePreviewItems"
:process-type="processType"
:file-name="activePreviewFile?.name ?? ''"
:files="previewFiles"
@update:selected-id="selectPreviewItem"
@update:selected-file-id="selectPreviewFile"
@update:item-content="updatePreviewContent"
@restore:item="restorePreviewItem"
@add:item="addPreviewItem"
@remove:item="removePreviewItem"
/>
<GenerationStep
v-else-if="currentStepId === 'generate'"
:task-name="task.name"
:process-type="processType"
:file-name="fileName"
:preview-count="previewItems.length"
:modified-count="modifiedPreviewCount"
:generation="generation"
@stop="stopGeneration"
@retry="startGeneration"
/>
<ResultEditorStep
v-else-if="currentStepId === 'results'"
v-model:selected-id="selectedResultId"
:items="results"
@update:field="updateResultField"
@restore:item="restoreResult"
/>
</div>
</div>
</main>
<footer class="wizard-footer">
<div class="footer-left">
<el-button v-if="currentStep > 0" @click="handleBack">
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回{{ previousStepLabel }}
</el-button>
<el-button v-else @click="handleCancel">取消</el-button>
</div>
<div class="footer-center">
</div>
<div class="footer-right">
<el-button
class="wizard-primary-action"
type="primary"
:loading="generation.status === 'running'"
:disabled="currentStepId === 'generate' && generation.status === 'running'"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
</el-button>
</div>
</footer>
</div>
<AppConfirmDialog ref="confirmDialogRef" />
</template>
<style scoped lang="scss" src="./create/data-process-create.scss"></style>