feat: 数据处理向导支持外来数据源拉取

任务设置步骤新增 external 数据类型,支持 MySQL/PostgreSQL/MongoDB/REST API 配置、测试连接与拉取数据,生成步骤与草稿持久化同步适配,回归脚本放宽分页器数量断言。
This commit is contained in:
caoxiaozhu
2026-07-11 11:00:15 +08:00
parent a2d65a1123
commit ee50350dff
5 changed files with 300 additions and 19 deletions

View File

@@ -7,7 +7,7 @@ import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import type { GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
import type { ExternalDataSource, GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
const router = useRouter()
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
@@ -34,6 +34,18 @@ interface UploadedDataFile {
const uploadedFiles = ref<UploadedDataFile[]>([])
const externalSource = reactive<ExternalDataSource>({
type: 'mysql',
url: '',
authMode: 'none',
username: '',
password: '',
token: '',
limit: 1000,
})
const externalPulling = ref(false)
const externalConnected = ref(false)
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
const fileSize = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.size, 0))
const fileCount = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.count, 0))
@@ -95,6 +107,7 @@ function draftSnapshot() {
task: { ...task },
processType: processType.value,
uploadedFiles: uploadedFiles.value,
externalSource: { ...externalSource },
previewSignature: previewSignature.value,
previewItems: previewItems.value,
selectedPreviewFileId: selectedPreviewFileId.value,
@@ -133,7 +146,9 @@ function restoreDraft() {
currentStep.value = Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
task.name = snapshot.task?.name || ''
task.description = snapshot.task?.description || ''
processType.value = snapshot.processType === 'structured' ? 'structured' : 'unstructured'
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
? snapshot.processType
: 'unstructured'
if (snapshot.uploadedFiles) {
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
@@ -149,6 +164,9 @@ function restoreDraft() {
}
previewSignature.value = snapshot.previewSignature || ''
if (snapshot.externalSource) {
Object.assign(externalSource, snapshot.externalSource)
}
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
previewItems.value = Array.isArray(snapshot.previewItems)
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
@@ -171,14 +189,15 @@ function restoreDraft() {
}
watch(
[() => task.name, () => task.description, processType],
[() => task.name, () => task.description, processType, externalSource],
() => {
if (!restoringDraft.value) dirty.value = true
},
{ deep: true },
)
watch(
[currentStep, task, processType, uploadedFiles, previewSignature,
[currentStep, task, processType, uploadedFiles, externalSource, previewSignature,
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
results, selectedResultId, generation],
persistDraft,
@@ -240,6 +259,48 @@ function useSampleFile() {
dirty.value = true
}
function updateExternalSource(value: ExternalDataSource) {
Object.assign(externalSource, value)
externalConnected.value = false
}
function handleTestConnection() {
if (!externalSource.url.trim()) {
ElMessage.warning('请先填写数据源地址')
return
}
externalPulling.value = true
setTimeout(() => {
externalPulling.value = false
externalConnected.value = true
ElMessage.success('数据源连接测试成功')
}, 1500)
}
function handlePullData() {
if (!externalSource.url.trim()) {
ElMessage.warning('请先填写数据源地址')
return
}
externalPulling.value = true
setTimeout(() => {
externalPulling.value = false
externalConnected.value = true
const typeName = externalSource.type.toUpperCase()
const id = `external-${Date.now()}`
uploadedFiles.value.push({
uid: id,
name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
size: Math.min(externalSource.limit, 5000) * 64,
count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
content: DEFAULT_SOURCE_TEXT,
})
if (!task.name) task.name = `${typeName} 数据拉取任务`
dirty.value = true
ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
}, 2000)
}
function handleRemoveFile(uid: string | number) {
const index = uploadedFiles.value.findIndex(f => f.uid === uid)
if (index > -1) {
@@ -265,7 +326,7 @@ async function nextFromCreate() {
const valid = await taskSetupRef.value?.validate()
if (!valid) return
if (uploadedFiles.value.length === 0) {
ElMessage.warning('请上传至少一个源数据文件')
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
return
}
@@ -539,9 +600,15 @@ onMounted(restoreDraft)
v-model:description="task.description"
v-model:process-type="processType"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
:external-connected="externalConnected"
@update:external-source="updateExternalSource"
@file-change="handleFileChange"
@remove-file="handleRemoveFile"
@use-sample="useSampleFile"
@test-connection="handleTestConnection"
@pull-data="handlePullData"
/>
<PreviewCompareStep