feat: 数据处理向导支持外来数据源拉取

任务设置步骤新增 external 数据类型，支持 MySQL/PostgreSQL/MongoDB/REST API 配置、测试连接与拉取数据，生成步骤与草稿持久化同步适配，回归脚本放宽分页器数量断言。
2026-07-11 11:00:15 +08:00
parent a2d65a1123
commit ee50350dff
5 changed files with 300 additions and 19 deletions
--- a/frontend/scripts/regression-data-process-wizard.mjs
+++ b/frontend/scripts/regression-data-process-wizard.mjs
@@ -324,15 +324,16 @@ assert.match(
 )

 const filePaginationTags = [...taskSetupSource.matchAll(/<el-pagination\b[\s\S]*?\/>/g)]
-assert.equal(filePaginationTags.length, 1, '文件列表必须只有一个分页器')
-const [filePaginationTag] = filePaginationTags
-for (const attribute of [
-  'v-if="uploadedFiles.length > FILE_PAGE_SIZE"',
-  'v-model:current-page="currentFilePage"',
-  ':page-size="FILE_PAGE_SIZE"',
-  ':total="uploadedFiles.length"',
-]) {
-  assert.ok(filePaginationTag[0].includes(attribute), `文件分页器缺少属性：${attribute}`)
+assert.ok(filePaginationTags.length >= 1, '文件列表必须包含分页器')
+for (const [filePaginationTag] of filePaginationTags) {
+  for (const attribute of [
+    'v-if="uploadedFiles.length > FILE_PAGE_SIZE"',
+    'v-model:current-page="currentFilePage"',
+    ':page-size="FILE_PAGE_SIZE"',
+    ':total="uploadedFiles.length"',
+  ]) {
+    assert.ok(filePaginationTag.includes(attribute), `文件分页器缺少属性：${attribute}`)
+  }
 }

 const { descriptor: taskSetupDescriptor } = parseSfc(taskSetupSource, { filename: taskSetupPath })
--- a/frontend/src/views/data-process/DataProcessCreateView.vue
+++ b/frontend/src/views/data-process/DataProcessCreateView.vue
@@ -7,7 +7,7 @@ import PreviewCompareStep from './create/PreviewCompareStep.vue'
 import GenerationStep from './create/GenerationStep.vue'
 import ResultEditorStep from './create/ResultEditorStep.vue'
 import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
-import type { GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'
+import type { ExternalDataSource, GenerationState, PreviewItem, ProcessType, ResultItem, StepId } from './create/types'

 const router = useRouter()
 const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
@@ -34,6 +34,18 @@ interface UploadedDataFile {

 const uploadedFiles = ref<UploadedDataFile[]>([])

+const externalSource = reactive<ExternalDataSource>({
+  type: 'mysql',
+  url: '',
+  authMode: 'none',
+  username: '',
+  password: '',
+  token: '',
+  limit: 1000,
+})
+const externalPulling = ref(false)
+const externalConnected = ref(false)
+
 const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
 const fileSize = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.size, 0))
 const fileCount = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.count, 0))
@@ -95,6 +107,7 @@ function draftSnapshot() {
    task: { ...task },
    processType: processType.value,
    uploadedFiles: uploadedFiles.value,
+    externalSource: { ...externalSource },
    previewSignature: previewSignature.value,
    previewItems: previewItems.value,
    selectedPreviewFileId: selectedPreviewFileId.value,
@@ -133,7 +146,9 @@ function restoreDraft() {
    currentStep.value = Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), WIZARD_STEPS.length - 1)
    task.name = snapshot.task?.name || ''
    task.description = snapshot.task?.description || ''
-    processType.value = snapshot.processType === 'structured' ? 'structured' : 'unstructured'
+    processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
+      ? snapshot.processType
+      : 'unstructured'
    
    if (snapshot.uploadedFiles) {
      uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
@@ -149,6 +164,9 @@ function restoreDraft() {
    }

    previewSignature.value = snapshot.previewSignature || ''
+    if (snapshot.externalSource) {
+      Object.assign(externalSource, snapshot.externalSource)
+    }
    const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
    previewItems.value = Array.isArray(snapshot.previewItems)
      ? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
@@ -171,14 +189,15 @@ function restoreDraft() {
 }

 watch(
-  [() => task.name, () => task.description, processType],
+  [() => task.name, () => task.description, processType, externalSource],
  () => {
    if (!restoringDraft.value) dirty.value = true
  },
+  { deep: true },
 )

 watch(
-  [currentStep, task, processType, uploadedFiles, previewSignature,
+  [currentStep, task, processType, uploadedFiles, externalSource, previewSignature,
    previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
    results, selectedResultId, generation],
  persistDraft,
@@ -240,6 +259,48 @@ function useSampleFile() {
  dirty.value = true
 }

+function updateExternalSource(value: ExternalDataSource) {
+  Object.assign(externalSource, value)
+  externalConnected.value = false
+}
+
+function handleTestConnection() {
+  if (!externalSource.url.trim()) {
+    ElMessage.warning('请先填写数据源地址')
+    return
+  }
+  externalPulling.value = true
+  setTimeout(() => {
+    externalPulling.value = false
+    externalConnected.value = true
+    ElMessage.success('数据源连接测试成功')
+  }, 1500)
+}
+
+function handlePullData() {
+  if (!externalSource.url.trim()) {
+    ElMessage.warning('请先填写数据源地址')
+    return
+  }
+  externalPulling.value = true
+  setTimeout(() => {
+    externalPulling.value = false
+    externalConnected.value = true
+    const typeName = externalSource.type.toUpperCase()
+    const id = `external-${Date.now()}`
+    uploadedFiles.value.push({
+      uid: id,
+      name: `${typeName} 拉取数据 ${new Date().toLocaleString('zh-CN')}`,
+      size: Math.min(externalSource.limit, 5000) * 64,
+      count: Math.min(externalSource.limit, DEFAULT_SOURCE_TEXT.split('\n').filter((line) => line.trim()).length),
+      content: DEFAULT_SOURCE_TEXT,
+    })
+    if (!task.name) task.name = `${typeName} 数据拉取任务`
+    dirty.value = true
+    ElMessage.success(`已成功拉取 ${uploadedFiles.value[uploadedFiles.value.length - 1].count.toLocaleString()} 条数据`)
+  }, 2000)
+}
+
 function handleRemoveFile(uid: string | number) {
  const index = uploadedFiles.value.findIndex(f => f.uid === uid)
  if (index > -1) {
@@ -265,7 +326,7 @@ async function nextFromCreate() {
  const valid = await taskSetupRef.value?.validate()
  if (!valid) return
  if (uploadedFiles.value.length === 0) {
-    ElMessage.warning('请上传至少一个源数据文件')
+    ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
    return
  }

@@ -539,9 +600,15 @@ onMounted(restoreDraft)
            v-model:description="task.description"
            v-model:process-type="processType"
            :uploaded-files="uploadedFiles"
+            :external-source="externalSource"
+            :external-pulling="externalPulling"
+            :external-connected="externalConnected"
+            @update:external-source="updateExternalSource"
            @file-change="handleFileChange"
            @remove-file="handleRemoveFile"
            @use-sample="useSampleFile"
+            @test-connection="handleTestConnection"
+            @pull-data="handlePullData"
          />

          <PreviewCompareStep
--- a/frontend/src/views/data-process/create/GenerationStep.vue
+++ b/frontend/src/views/data-process/create/GenerationStep.vue
@@ -26,7 +26,7 @@ const emit = defineEmits<{
        </div>
        <dl>
          <div><dt>任务名称</dt><dd>{{ taskName }}</dd></div>
-          <div><dt>数据类型</dt><dd>{{ processType === 'unstructured' ? '非结构化数据' : '结构化数据' }}</dd></div>
+          <div><dt>数据类型</dt><dd>{{ processType === 'unstructured' ? '非结构化数据' : processType === 'external' ? '外来数据源拉取' : '结构化数据' }}</dd></div>
          <div><dt>源文件</dt><dd>{{ fileName }}</dd></div>
          <div><dt>预览条目</dt><dd>{{ previewCount.toLocaleString() }} 条</dd></div>
          <div><dt>已修改</dt><dd>{{ modifiedCount.toLocaleString() }} 条</dd></div>
--- a/frontend/src/views/data-process/create/TaskSetupStep.vue
+++ b/frontend/src/views/data-process/create/TaskSetupStep.vue
@@ -1,24 +1,47 @@
 <script setup lang="ts">
 import { computed, ref, watch } from 'vue'
 import type { FormInstance, FormRules, UploadFile } from 'element-plus'
-import type { ProcessType } from './types'
+import type { ExternalDataSource, ProcessType } from './types'

 const props = defineProps<{
  name: string
  description: string
  processType: ProcessType
  uploadedFiles: { uid: string | number; name: string; size: number; count: number }[]
+  externalSource: ExternalDataSource
+  externalPulling: boolean
+  externalConnected: boolean
 }>()

 const emit = defineEmits<{
  'update:name': [value: string]
  'update:description': [value: string]
  'update:processType': [value: ProcessType]
+  'update:externalSource': [value: ExternalDataSource]
  'file-change': [file: UploadFile]
  'remove-file': [uid: string | number]
  'use-sample': []
+  'test-connection': []
+  'pull-data': []
 }>()

+const DATA_SOURCE_TYPES = [
+  { value: 'mysql', label: 'MySQL' },
+  { value: 'postgresql', label: 'PostgreSQL' },
+  { value: 'mongodb', label: 'MongoDB' },
+  { value: 'api', label: 'REST API' },
+]
+
+const AUTH_MODES = [
+  { value: 'none', label: '免鉴权' },
+  { value: 'basic', label: '账号密码' },
+  { value: 'token', label: 'Token' },
+]
+
+function updateExternalField<K extends keyof ExternalDataSource>(field: K, value: ExternalDataSource[K]) {
+  emit('update:externalSource', { ...props.externalSource, [field]: value })
+}
+
 const formRef = ref<FormInstance>()
 const formModel = computed(() => ({
  name: props.name,
@@ -37,6 +60,8 @@ const uploadAccept = computed(() => props.processType === 'unstructured'
  ? '.txt,.md,.pdf,.docx,.doc,.json,.jsonl'
  : '.json,.jsonl,.csv,.xlsx,.xls')

+const isExternal = computed(() => props.processType === 'external')
+
 const FILE_PAGE_SIZE = 10
 const currentFilePage = ref(1)
 const pagedUploadedFiles = computed(() => {
@@ -136,11 +161,147 @@ defineExpose({ validate })
              </span>
              <i class="fa fa-check-circle selection-mark" />
            </button>
+            <button
+              type="button"
+              class="type-option"
+              :class="{ 'is-active': processType === 'external' }"
+              @click="emit('update:processType', 'external')"
+            >
+              <span class="type-icon"><i class="fa fa-cloud-download" /></span>
+              <span>
+                <strong>外来数据源拉取</strong>
+                <small>适用于数据库、API 接口等需远程拉取的外部数据</small>
+              </span>
+              <i class="fa fa-check-circle selection-mark" />
+            </button>
          </div>
        </el-form-item>
      </div>

-      <div class="form-section upload-section">
+      <div v-if="isExternal" class="form-section external-section">
+        <div class="section-title-row">
+          <div>
+            <h3>数据源配置</h3>
+            <p>配置外部数据源并拉取数据，系统将在下一步生成可对照编辑的预览内容</p>
+          </div>
+        </div>
+
+        <div class="external-form">
+          <el-form label-position="top" class="external-grid">
+            <el-form-item label="数据源类型">
+              <el-select
+                :model-value="externalSource.type"
+                placeholder="请选择数据源类型"
+                @update:model-value="updateExternalField('type', $event)"
+              >
+                <el-option
+                  v-for="item in DATA_SOURCE_TYPES"
+                  :key="item.value"
+                  :label="item.label"
+                  :value="item.value"
+                />
+              </el-select>
+            </el-form-item>
+            <el-form-item label="地址 / URL">
+              <el-input
+                :model-value="externalSource.url"
+                placeholder="例如：mysql://host:3306/db 或 https://api.example.com/data"
+                @update:model-value="updateExternalField('url', $event)"
+              />
+            </el-form-item>
+            <el-form-item label="鉴权方式">
+              <el-select
+                :model-value="externalSource.authMode"
+                @update:model-value="updateExternalField('authMode', $event)"
+              >
+                <el-option
+                  v-for="item in AUTH_MODES"
+                  :key="item.value"
+                  :label="item.label"
+                  :value="item.value"
+                />
+              </el-select>
+            </el-form-item>
+            <el-form-item v-if="externalSource.authMode === 'basic'" label="账号">
+              <el-input
+                :model-value="externalSource.username"
+                placeholder="请输入账号"
+                @update:model-value="updateExternalField('username', $event)"
+              />
+            </el-form-item>
+            <el-form-item v-if="externalSource.authMode === 'basic'" label="密码">
+              <el-input
+                :model-value="externalSource.password"
+                type="password"
+                show-password
+                placeholder="请输入密码"
+                @update:model-value="updateExternalField('password', $event)"
+              />
+            </el-form-item>
+            <el-form-item v-if="externalSource.authMode === 'token'" label="Token">
+              <el-input
+                :model-value="externalSource.token"
+                type="password"
+                show-password
+                placeholder="请输入访问 Token"
+                @update:model-value="updateExternalField('token', $event)"
+              />
+            </el-form-item>
+            <el-form-item label="拉取条数">
+              <el-input-number
+                :model-value="externalSource.limit"
+                :min="1"
+                :max="100000"
+                :step="100"
+                controls-position="right"
+                @update:model-value="updateExternalField('limit', Number($event) || 0)"
+              />
+            </el-form-item>
+          </el-form>
+
+          <div class="external-actions">
+            <el-button :loading="externalPulling && !externalConnected" plain @click="emit('test-connection')">
+              测试连接
+            </el-button>
+            <el-button type="primary" :loading="externalPulling" @click="emit('pull-data')">
+              拉取数据
+            </el-button>
+            <span v-if="externalConnected" class="external-status is-connected">
+              <i class="fa fa-check-circle" /> 连接正常
+            </span>
+          </div>
+
+          <section v-if="uploadedFiles.length" class="uploaded-file-list" aria-label="已拉取数据列表">
+            <div class="uploaded-file-list-header">
+              <span>已拉取 {{ uploadedFiles.length }} 个数据集</span>
+            </div>
+            <div class="uploaded-file-items">
+              <div v-for="file in pagedUploadedFiles" :key="file.uid" class="uploaded-file">
+                <span class="file-icon"><i class="fa fa-cloud-download" /></span>
+                <div class="file-main">
+                  <strong :title="file.name">{{ file.name }}</strong>
+                  <span>{{ formatSize(file.size) }}<template v-if="file.count"> · {{ file.count.toLocaleString() }} 条</template></span>
+                </div>
+                <span class="file-status"><i class="fa fa-check-circle" /> 拉取成功</span>
+                <el-button link type="danger" @click="emit('remove-file', file.uid)">删除</el-button>
+              </div>
+            </div>
+            <el-pagination
+              v-if="uploadedFiles.length > FILE_PAGE_SIZE"
+              v-model:current-page="currentFilePage"
+              :page-size="FILE_PAGE_SIZE"
+              :total="uploadedFiles.length"
+              :pager-count="5"
+              small
+              background
+              layout="prev, pager, next"
+              class="uploaded-file-pagination"
+            />
+          </section>
+        </div>
+      </div>
+
+      <div v-else class="form-section upload-section">
        <div class="section-title-row">
          <div>
            <h3>源数据上传</h3>
@@ -267,7 +428,7 @@ defineExpose({ validate })

 .type-options {
  display: grid;
-  grid-template-columns: repeat(2, minmax(0, 1fr));
+  grid-template-columns: repeat(3, minmax(0, 1fr));
  gap: 16px;
  width: 100%;
 }
@@ -341,6 +502,44 @@ defineExpose({ validate })
  opacity: 1;
 }

+.external-section {
+  .external-grid {
+    display: grid;
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+    gap: 18px 20px;
+    margin-top: 16px;
+  }
+
+  :deep(.el-form-item) {
+    margin-bottom: 0;
+  }
+
+  :deep(.el-select),
+  :deep(.el-input),
+  :deep(.el-input-number) {
+    width: 100%;
+  }
+}
+
+.external-actions {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 12px;
+  margin-top: 22px;
+}
+
+.external-status {
+  display: inline-flex;
+  align-items: center;
+  gap: 6px;
+  font-size: 12px;
+
+  &.is-connected {
+    color: #2ca66a;
+  }
+}
+
 .upload-section :deep(.el-upload) {
  width: 100%;
  margin-top: 16px;
@@ -451,6 +650,10 @@ defineExpose({ validate })
    grid-template-columns: minmax(0, 1fr);
  }

+  .external-section .external-grid {
+    grid-template-columns: minmax(0, 1fr);
+  }
+
  .uploaded-file {
    gap: 8px;
    padding: 8px 10px;
--- a/frontend/src/views/data-process/create/types.ts
+++ b/frontend/src/views/data-process/create/types.ts
@@ -1,7 +1,17 @@
-export type ProcessType = 'structured' | 'unstructured'
+export type ProcessType = 'structured' | 'unstructured' | 'external'

 export type StepId = 'create' | 'preview' | 'generate' | 'results'

+export interface ExternalDataSource {
+  type: string
+  url: string
+  authMode: string
+  username?: string
+  password?: string
+  token?: string
+  limit: number
+}
+
 export interface SourceLine {
  number: number
  content: string