Files
YG_FT/frontend/src/views/data-process/DataProcessCreateView.vue

830 lines
25 KiB
Vue
Raw Normal View History

<script setup lang="ts">
import { computed, nextTick, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
import { onBeforeRouteLeave, useRouter } from 'vue-router'
import { ElMessage, ElMessageBox, type UploadFile } from 'element-plus'
import TaskSetupStep from './create/TaskSetupStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import type { ExternalDataSource, GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
const router = useRouter()
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与上传源数据' },
{ id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
{ id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }>
const currentStep = ref(0)
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
const task = reactive({ name: '', description: '' })
const processType = ref<ProcessType>('unstructured')
interface UploadedDataFile {
uid: number | string
name: string
size: number
count: number
content: string
}
const uploadedFiles = ref<UploadedDataFile[]>([])
const externalSource = reactive<ExternalDataSource>({
type: 'mysql',
url: '',
authMode: 'none',
username: '',
password: '',
token: '',
limit: 1000,
})
const externalPulling = ref(false)
const externalConnected = ref(false)
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
const fileSize = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.size, 0))
const fileCount = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.count, 0))
const previewSignature = ref('')
const previewItems = ref<PreviewItem[]>([])
const selectedPreviewFileId = ref<string | null>(null)
const selectedPreviewId = ref<string | null>(null)
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
const results = ref<ResultItem[]>([])
const selectedResultId = ref<string | null>(null)
const dirty = ref(false)
const restoringDraft = ref(false)
let allowLeave = false
const generation = reactive<GenerationState>({
status: 'idle',
progress: 0,
message: '确认摘要后即可开始生成,过程中可查看实时进度。',
})
let generationTimer: ReturnType<typeof setInterval> | null = null
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value))
const activePreviewItems = computed(() => previewItems.value.filter((item) => item.sourceFileId === selectedPreviewFileId.value))
const activeSourceText = computed(() => activePreviewFile.value?.content ?? '')
const previewFiles = computed(() => uploadedFiles.value.map((file) => {
const items = previewItems.value.filter((item) => item.sourceFileId === String(file.uid))
return {
id: String(file.uid),
name: file.name,
count: items.length,
modifiedCount: items.filter((item) => item.status !== 'original').length,
}
}))
const primaryActionLabel = computed(() => {
if (currentStepId.value === 'create') return '继续:数据预览'
if (currentStepId.value === 'preview') return '确认预览并继续'
if (currentStepId.value === 'results') return '保存任务'
if (generation.status === 'running') return '正在生成'
if (generation.status === 'success') return '查看生成结果'
if (generation.status === 'failed') return '重新生成'
return '开始生成'
})
const primaryActionIcon = computed(() => {
if (currentStepId.value === 'results') return 'fa-check'
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
return 'fa-arrow-right'
})
const previousStepLabel = computed(() => currentStep.value > 0
? WIZARD_STEPS[currentStep.value - 1].title
: '')
function draftSnapshot() {
return {
currentStep: currentStep.value,
task: { ...task },
processType: processType.value,
uploadedFiles: uploadedFiles.value,
externalSource: { ...externalSource },
previewSignature: previewSignature.value,
previewItems: previewItems.value,
selectedPreviewFileId: selectedPreviewFileId.value,
selectedPreviewId: selectedPreviewId.value,
selectedPreviewIdsByFile: selectedPreviewIdsByFile.value,
results: results.value,
selectedResultId: selectedResultId.value,
generation: { ...generation },
}
}
function persistDraft() {
if (restoringDraft.value) return
try {
localStorage.setItem(DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
} catch {
ElMessage.warning('草稿保存失败,请检查浏览器存储空间')
}
}
type DraftSnapshot = ReturnType<typeof draftSnapshot> & {
fileName?: string
fileSize?: number
fileCount?: number
sourceText?: string
}
function restoreDraft() {
try {
const raw = localStorage.getItem(DRAFT_STORAGE_KEY)
if (!raw) return
const snapshot = JSON.parse(raw) as DraftSnapshot
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
restoringDraft.value = true
currentStep.value = Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
task.name = snapshot.task?.name || ''
task.description = snapshot.task?.description || ''
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
? snapshot.processType
: 'unstructured'
if (snapshot.uploadedFiles) {
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
} else if (snapshot.fileName && snapshot.sourceText) {
// Migrate old draft
uploadedFiles.value = [{
uid: 'migrated-draft',
name: snapshot.fileName,
size: Number(snapshot.fileSize) || 0,
count: Number(snapshot.fileCount) || 0,
content: snapshot.sourceText
}]
}
previewSignature.value = snapshot.previewSignature || ''
if (snapshot.externalSource) {
Object.assign(externalSource, snapshot.externalSource)
}
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
previewItems.value = Array.isArray(snapshot.previewItems)
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
: []
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
selectedPreviewIdsByFile.value = snapshot.selectedPreviewIdsByFile || {}
selectedPreviewId.value = snapshot.selectedPreviewId
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|| activePreviewItems.value[0]?.id
|| null
results.value = Array.isArray(snapshot.results) ? snapshot.results : []
selectedResultId.value = snapshot.selectedResultId || results.value[0]?.id || null
Object.assign(generation, snapshot.generation || {})
dirty.value = false
nextTick(() => { restoringDraft.value = false })
ElMessage.info('已恢复上次保存的草稿')
} catch {
localStorage.removeItem(DRAFT_STORAGE_KEY)
}
}
watch(
[() => task.name, () => task.description, processType, externalSource],
() => {
if (!restoringDraft.value) dirty.value = true
},
{ deep: true },
)
watch(
[currentStep, task, processType, uploadedFiles, externalSource, previewSignature,
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
results, selectedResultId, generation],
persistDraft,
{ deep: true },
)
function scrollToStepTop() {
document.querySelector<HTMLElement>('.layout-content')?.scrollTo({ top: 0, behavior: 'smooth' })
}
watch(currentStep, () => nextTick(scrollToStepTop))
async function handleFileChange(uploadFile: UploadFile) {
const raw = uploadFile.raw
if (!raw) return
if (raw.size > 200 * 1024 * 1024) {
ElMessage.warning('单文件不能超过 200MB')
return
}
const extension = raw.name.split('.').pop()?.toLowerCase() ?? ''
const textExtensions = ['txt', 'md', 'json', 'jsonl', 'csv']
let content = ''
if (textExtensions.includes(extension)) {
try {
content = await raw.text()
} catch {
content = ''
}
}
const fileContent = content.trim() ? content : DEFAULT_SOURCE_TEXT
const linesCount = fileContent.split('\n').filter((line) => line.trim()).length
// Prevent duplicate upload of the same file
if (!uploadedFiles.value.some(f => f.name === raw.name && f.size === raw.size)) {
uploadedFiles.value.push({
uid: uploadFile.uid || Date.now() + Math.random(),
name: raw.name,
size: raw.size,
count: linesCount,
content: fileContent
})
}
dirty.value = true
}
function useSampleFile() {
uploadedFiles.value = [{
uid: 'sample-1',
name: 'finance_qa.jsonl',
size: 128 * 1024 * 1024,
count: DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length,
content: DEFAULT_SOURCE_TEXT
}]
if (!task.name) task.name = '金融问答清洗任务'
if (!task.description) task.description = '清洗金融领域问答数据,统一格式并生成高质量训练数据。'
dirty.value = true
}
function updateExternalSource(value: ExternalDataSource) {
Object.assign(externalSource, value)
externalConnected.value = false
}
function handleTestConnection() {
if (!externalSource.url.trim()) {
ElMessage.warning('请先填写数据源地址')
return
}
externalPulling.value = true
setTimeout(() => {
externalPulling.value = false
externalConnected.value = true
ElMessage.success('数据源连接测试成功')
}, 1500)
}
function handlePullData() {
if (!externalSource.url.trim()) {
ElMessage.warning('请先填写数据源地址')
return
}
externalPulling.value = true
setTimeout(() => {
externalPulling.value = false
externalConnected.value = true
const typeName = externalSource.type.toUpperCase()
const id = `external-${Date.now()}`
uploadedFiles.value.push({
uid: id,
name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
size: Math.min(externalSource.limit, 5000) * 64,
count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
content: DEFAULT_SOURCE_TEXT,
})
if (!task.name) task.name = `${typeName} 数据拉取任务`
dirty.value = true
ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
}, 2000)
}
function handleRemoveFile(uid: string | number) {
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
if (index > -1) {
uploadedFiles.value.splice(index, 1)
previewSignature.value = ''
previewItems.value = []
selectedPreviewId.value = null
resetDownstream()
dirty.value = true
}
}
function resetDownstream() {
stopGenerationTimer()
generation.status = 'idle'
generation.progress = 0
generation.message = '确认摘要后即可开始生成,过程中可查看实时进度。'
results.value = []
selectedResultId.value = null
}
async function nextFromCreate() {
const valid = await taskSetupRef.value?.validate()
if (!valid) return
if (uploadedFiles.value.length === 0) {
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
return
}
const signature = `${processType.value}:${uploadedFiles.value.map((file) => `${file.uid}:${file.content}`).join('|')}`
if (signature !== previewSignature.value) {
previewItems.value = uploadedFiles.value.flatMap((file) =>
buildPreviewItems(file.content, processType.value, String(file.uid)),
)
selectedPreviewFileId.value = String(uploadedFiles.value[0]?.uid ?? '') || null
selectedPreviewId.value = activePreviewItems.value[0]?.id ?? null
selectedPreviewIdsByFile.value = selectedPreviewId.value && selectedPreviewFileId.value
? { [selectedPreviewFileId.value]: selectedPreviewId.value }
: {}
previewSignature.value = signature
resetDownstream()
}
currentStep.value = 1
}
function selectPreviewFile(fileId: string) {
if (selectedPreviewFileId.value && selectedPreviewId.value) {
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
}
selectedPreviewFileId.value = fileId
selectedPreviewId.value = selectedPreviewIdsByFile.value[fileId]
|| activePreviewItems.value[0]?.id
|| null
}
function selectPreviewItem(id: string) {
selectedPreviewId.value = id
if (selectedPreviewFileId.value) selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = id
}
function updatePreviewContent(id: string, value: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item) return
item.editedContent = value
item.tokenCount = Math.max(1, Math.ceil(value.length / 2))
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
dirty.value = true
}
function restorePreviewItem(id: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item || item.sourceStart == null) return
item.editedContent = item.originalContent
item.tokenCount = Math.max(1, Math.ceil(item.originalContent.length / 2))
item.status = 'original'
dirty.value = true
}
function addPreviewItem() {
if (!selectedPreviewFileId.value) return
const id = `manual-${Date.now()}`
previewItems.value.push({
id,
sourceFileId: selectedPreviewFileId.value,
originalContent: '',
editedContent: '',
sourceStart: null,
sourceEnd: null,
sourceStartLine: null,
sourceEndLine: null,
tokenCount: 1,
status: 'manual',
})
selectPreviewItem(id)
dirty.value = true
}
async function removePreviewItem(id: string) {
try {
await ElMessageBox.confirm('删除只影响本次处理,不会修改源文件。确认删除吗?', '删除预览内容', {
confirmButtonText: '删除',
cancelButtonText: '取消',
type: 'warning',
})
} catch {
return
}
const index = previewItems.value.findIndex((item) => item.id === id)
if (index < 0) return
previewItems.value.splice(index, 1)
selectedPreviewId.value = activePreviewItems.value[Math.min(index, activePreviewItems.value.length - 1)]?.id ?? null
if (selectedPreviewFileId.value && selectedPreviewId.value) {
selectedPreviewIdsByFile.value[selectedPreviewFileId.value] = selectedPreviewId.value
}
dirty.value = true
}
function stopGenerationTimer() {
if (generationTimer) clearInterval(generationTimer)
generationTimer = null
}
function startGeneration() {
stopGenerationTimer()
generation.status = 'running'
generation.progress = 0
generation.message = '正在应用预览修改并生成标准化结果,请稍候。'
generationTimer = setInterval(() => {
generation.progress = Math.min(100, generation.progress + 8)
if (generation.progress < 100) return
stopGenerationTimer()
generation.status = 'success'
generation.message = `已完成 ${previewItems.value.length.toLocaleString()} 条数据处理,可进入结果页检查。`
results.value = createResults(previewItems.value)
selectedResultId.value = results.value[0]?.id ?? null
dirty.value = true
ElMessage.success('数据处理完成')
}, 180)
}
function stopGeneration() {
stopGenerationTimer()
generation.status = 'failed'
generation.message = '任务已停止,预览修改仍然保留,可以重新生成。'
}
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item[field] = value
const valid = item.instruction.trim() && item.output.trim()
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
const changed = item.instruction !== item.originalInstruction
|| item.input !== item.originalInput
|| item.output !== item.originalOutput
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
dirty.value = true
}
function restoreResult(id: string) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item.instruction = item.originalInstruction
item.input = item.originalInput
item.output = item.originalOutput
item.error = undefined
item.status = 'valid'
dirty.value = true
}
function validateResults() {
let firstInvalidId: string | null = null
for (const item of results.value) {
if (!item.instruction.trim() || !item.output.trim()) {
item.error = 'Instruction 和 Output 不能为空'
item.status = 'invalid'
firstInvalidId ??= item.id
}
}
if (firstInvalidId) selectedResultId.value = firstInvalidId
return firstInvalidId == null
}
async function handlePrimaryAction() {
if (currentStepId.value === 'create') {
await nextFromCreate()
return
}
if (currentStepId.value === 'preview') {
if (!previewItems.value.length) {
ElMessage.warning('当前没有可生成的预览内容')
return
}
currentStep.value = 2
return
}
if (currentStepId.value === 'generate') {
if (generation.status === 'success') {
currentStep.value = 3
} else if (generation.status !== 'running') {
startGeneration()
}
return
}
await saveTask()
}
function handleBack() {
if (generation.status === 'running') {
ElMessage.warning('请先停止当前生成任务')
return
}
if (currentStep.value > 0) currentStep.value -= 1
}
async function saveTask() {
if (!validateResults()) {
ElMessage.warning('请先修正校验失败的结果')
return
}
dirty.value = false
localStorage.removeItem(DRAFT_STORAGE_KEY)
allowLeave = true
ElMessage.success('数据处理任务已保存')
await router.push('/data-process')
}
async function handleCancel() {
if (!dirty.value) {
allowLeave = true
router.back()
return
}
try {
await ElMessageBox.confirm('当前存在未保存修改,确定离开吗?', '离开创建任务', {
confirmButtonText: '放弃修改',
cancelButtonText: '继续编辑',
type: 'warning',
})
allowLeave = true
router.back()
} catch {
// 用户继续编辑。
}
}
onBeforeRouteLeave((_to, _from, next) => {
if (allowLeave || !dirty.value) {
next()
return
}
next(window.confirm('当前存在未保存修改,确定离开吗?'))
})
onBeforeUnmount(stopGenerationTimer)
onMounted(restoreDraft)
</script>
<template>
<div class="create-wizard-layout">
<main class="wizard-main">
<div class="wizard-main-inner">
<div class="wizard-steps-container">
<div class="custom-wizard-steps">
<div
v-for="(step, index) in WIZARD_STEPS"
:key="step.id"
class="step-item"
:class="{
'is-active': currentStep === index,
'is-completed': currentStep > index
}"
>
<div v-if="index !== 0" class="step-connector"></div>
<div class="step-node">
<div class="step-icon">
<i v-if="currentStep > index" class="fa fa-check" />
<span v-else>{{ index + 1 }}</span>
</div>
<div class="step-text">
<div class="step-title">{{ step.title }}</div>
</div>
</div>
</div>
</div>
</div>
<div class="wizard-content">
<TaskSetupStep
v-if="currentStepId === 'create'"
ref="taskSetupRef"
v-model:name="task.name"
v-model:description="task.description"
v-model:process-type="processType"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
:external-connected="externalConnected"
@update:external-source="updateExternalSource"
@file-change="handleFileChange"
@remove-file="handleRemoveFile"
@use-sample="useSampleFile"
@test-connection="handleTestConnection"
@pull-data="handlePullData"
/>
<PreviewCompareStep
v-else-if="currentStepId === 'preview'"
:selected-id="selectedPreviewId"
:selected-file-id="selectedPreviewFileId"
:source-text="activeSourceText"
:items="activePreviewItems"
:process-type="processType"
:file-name="activePreviewFile?.name ?? ''"
:files="previewFiles"
@update:selected-id="selectPreviewItem"
@update:selected-file-id="selectPreviewFile"
@update:item-content="updatePreviewContent"
@restore:item="restorePreviewItem"
@add:item="addPreviewItem"
@remove:item="removePreviewItem"
/>
<GenerationStep
v-else-if="currentStepId === 'generate'"
:task-name="task.name"
:process-type="processType"
:file-name="fileName"
:preview-count="previewItems.length"
:modified-count="modifiedPreviewCount"
:generation="generation"
@stop="stopGeneration"
@retry="startGeneration"
/>
<ResultEditorStep
v-else
v-model:selected-id="selectedResultId"
:items="results"
@update:field="updateResultField"
@restore:item="restoreResult"
/>
</div>
</div>
</main>
<footer class="wizard-footer">
<div class="footer-left">
<el-button v-if="currentStep > 0" @click="handleBack">
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回{{ previousStepLabel }}
</el-button>
<el-button v-else @click="handleCancel">取消</el-button>
</div>
<div class="footer-center">
</div>
<div class="footer-right">
<el-button
class="wizard-primary-action"
type="primary"
:loading="generation.status === 'running'"
:disabled="currentStepId === 'generate' && generation.status === 'running'"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
</el-button>
</div>
</footer>
</div>
</template>
<style scoped lang="scss">
.create-wizard-layout {
display: flex;
flex-direction: column;
height: 100%;
border: 1px solid #eef0f5;
border-radius: 8px;
background: #fff;
}
.wizard-main {
flex: 1;
overflow-y: auto;
padding: 32px;
/* 自定义滚动条 */
&::-webkit-scrollbar {
width: 6px;
}
&::-webkit-scrollbar-thumb {
background: #cbd5e1;
border-radius: 3px;
}
}
.wizard-main-inner {
min-height: 100%;
box-sizing: border-box;
}
.wizard-steps-container {
margin-bottom: 32px;
padding-bottom: 24px;
border-bottom: 1px dashed #e2e8f0;
}
.custom-wizard-steps {
display: flex;
align-items: center;
justify-content: space-between;
max-width: 860px;
margin: 0 auto;
padding: 0 20px;
}
.step-item {
display: flex;
align-items: center;
flex: 1;
&:first-child {
flex: 0;
}
}
.step-connector {
flex: 1;
height: 2px;
background-color: #e2e8f0;
margin: 0 16px;
transition: background-color 0.3s;
}
.step-item.is-completed .step-connector,
.step-item.is-active .step-connector {
background-color: #5146e5;
}
.step-node {
display: flex;
align-items: center;
gap: 10px;
}
.step-icon {
display: flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
border-radius: 50%;
border: 2px solid #cbd5e1;
background-color: #fff;
color: #64748b;
font-size: 13px;
font-weight: 650;
transition: all 0.3s;
}
.step-title {
font-size: 14px;
font-weight: 600;
color: #64748b;
white-space: nowrap;
}
/* State: Active */
.step-item.is-active {
.step-icon {
border-color: #5146e5;
background-color: #eef2ff;
color: #5146e5;
}
.step-title {
color: #1e293b;
}
}
/* State: Completed */
.step-item.is-completed {
.step-icon {
background-color: #5146e5;
border-color: #5146e5;
color: #fff;
}
.step-title {
color: #1e293b;
}
}
.wizard-content {
min-height: 400px;
}
.wizard-footer {
flex-shrink: 0;
height: 64px;
background: #ffffff;
border-top: 1px solid #e2e8f0;
display: grid;
grid-template-columns: 1fr auto 1fr;
align-items: center;
padding: 0 32px;
box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.02);
.footer-left {
justify-self: start;
}
.footer-center {
justify-self: center;
}
.footer-right {
justify-self: end;
}
}
.wizard-primary-action {
min-width: 160px;
}
</style>