feat: 数据处理向导支持外来数据源拉取

任务设置步骤新增 external 数据类型,支持 MySQL/PostgreSQL/MongoDB/REST API 配置、测试连接与拉取数据,生成步骤与草稿持久化同步适配,回归脚本放宽分页器数量断言。
This commit is contained in:
caoxiaozhu
2026-07-11 11:00:15 +08:00
parent a2d65a1123
commit ee50350dff
5 changed files with 300 additions and 19 deletions

View File

@@ -26,7 +26,7 @@ const emit = defineEmits<{
</div>
<dl>
<div><dt>任务名称</dt><dd>{{ taskName }}</dd></div>
<div><dt>数据类型</dt><dd>{{ processType === 'unstructured' ? '非结构化数据' : '结构化数据' }}</dd></div>
<div><dt>数据类型</dt><dd>{{ processType === 'unstructured' ? '非结构化数据' : processType === 'external' ? '外来数据源拉取' : '结构化数据' }}</dd></div>
<div><dt>源文件</dt><dd>{{ fileName }}</dd></div>
<div><dt>预览条目</dt><dd>{{ previewCount.toLocaleString() }} </dd></div>
<div><dt>已修改</dt><dd>{{ modifiedCount.toLocaleString() }} </dd></div>

View File

@@ -1,24 +1,47 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import type { FormInstance, FormRules, UploadFile } from 'element-plus'
import type { ProcessType } from './types'
import type { ExternalDataSource, ProcessType } from './types'
const props = defineProps<{
name: string
description: string
processType: ProcessType
uploadedFiles: { uid: string | number; name: string; size: number; count: number }[]
externalSource: ExternalDataSource
externalPulling: boolean
externalConnected: boolean
}>()
const emit = defineEmits<{
'update:name': [value: string]
'update:description': [value: string]
'update:processType': [value: ProcessType]
'update:externalSource': [value: ExternalDataSource]
'file-change': [file: UploadFile]
'remove-file': [uid: string | number]
'use-sample': []
'test-connection': []
'pull-data': []
}>()
const DATA_SOURCE_TYPES = [
{ value: 'mysql', label: 'MySQL' },
{ value: 'postgresql', label: 'PostgreSQL' },
{ value: 'mongodb', label: 'MongoDB' },
{ value: 'api', label: 'REST API' },
]
const AUTH_MODES = [
{ value: 'none', label: '免鉴权' },
{ value: 'basic', label: '账号密码' },
{ value: 'token', label: 'Token' },
]
function updateExternalField<K extends keyof ExternalDataSource>(field: K, value: ExternalDataSource[K]) {
emit('update:externalSource', { ...props.externalSource, [field]: value })
}
const formRef = ref<FormInstance>()
const formModel = computed(() => ({
name: props.name,
@@ -37,6 +60,8 @@ const uploadAccept = computed(() => props.processType === 'unstructured'
? '.txt,.md,.pdf,.docx,.doc,.json,.jsonl'
: '.json,.jsonl,.csv,.xlsx,.xls')
const isExternal = computed(() => props.processType === 'external')
const FILE_PAGE_SIZE = 10
const currentFilePage = ref(1)
const pagedUploadedFiles = computed(() => {
@@ -136,11 +161,147 @@ defineExpose({ validate })
</span>
<i class="fa fa-check-circle selection-mark" />
</button>
<button
type="button"
class="type-option"
:class="{ 'is-active': processType === 'external' }"
@click="emit('update:processType', 'external')"
>
<span class="type-icon"><i class="fa fa-cloud-download" /></span>
<span>
<strong>外来数据源拉取</strong>
<small>适用于数据库API 接口等需远程拉取的外部数据</small>
</span>
<i class="fa fa-check-circle selection-mark" />
</button>
</div>
</el-form-item>
</div>
<div class="form-section upload-section">
<div v-if="isExternal" class="form-section external-section">
<div class="section-title-row">
<div>
<h3>数据源配置</h3>
<p>配置外部数据源并拉取数据系统将在下一步生成可对照编辑的预览内容</p>
</div>
</div>
<div class="external-form">
<el-form label-position="top" class="external-grid">
<el-form-item label="数据源类型">
<el-select
:model-value="externalSource.type"
placeholder="请选择数据源类型"
@update:model-value="updateExternalField('type', $event)"
>
<el-option
v-for="item in DATA_SOURCE_TYPES"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
</el-form-item>
<el-form-item label="地址 / URL">
<el-input
:model-value="externalSource.url"
placeholder="例如mysql://host:3306/db 或 https://api.example.com/data"
@update:model-value="updateExternalField('url', $event)"
/>
</el-form-item>
<el-form-item label="鉴权方式">
<el-select
:model-value="externalSource.authMode"
@update:model-value="updateExternalField('authMode', $event)"
>
<el-option
v-for="item in AUTH_MODES"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
</el-form-item>
<el-form-item v-if="externalSource.authMode === 'basic'" label="账号">
<el-input
:model-value="externalSource.username"
placeholder="请输入账号"
@update:model-value="updateExternalField('username', $event)"
/>
</el-form-item>
<el-form-item v-if="externalSource.authMode === 'basic'" label="密码">
<el-input
:model-value="externalSource.password"
type="password"
show-password
placeholder="请输入密码"
@update:model-value="updateExternalField('password', $event)"
/>
</el-form-item>
<el-form-item v-if="externalSource.authMode === 'token'" label="Token">
<el-input
:model-value="externalSource.token"
type="password"
show-password
placeholder="请输入访问 Token"
@update:model-value="updateExternalField('token', $event)"
/>
</el-form-item>
<el-form-item label="拉取条数">
<el-input-number
:model-value="externalSource.limit"
:min="1"
:max="100000"
:step="100"
controls-position="right"
@update:model-value="updateExternalField('limit', Number($event) || 0)"
/>
</el-form-item>
</el-form>
<div class="external-actions">
<el-button :loading="externalPulling && !externalConnected" plain @click="emit('test-connection')">
测试连接
</el-button>
<el-button type="primary" :loading="externalPulling" @click="emit('pull-data')">
拉取数据
</el-button>
<span v-if="externalConnected" class="external-status is-connected">
<i class="fa fa-check-circle" /> 连接正常
</span>
</div>
<section v-if="uploadedFiles.length" class="uploaded-file-list" aria-label="已拉取数据列表">
<div class="uploaded-file-list-header">
<span>已拉取 {{ uploadedFiles.length }} 个数据集</span>
</div>
<div class="uploaded-file-items">
<div v-for="file in pagedUploadedFiles" :key="file.uid" class="uploaded-file">
<span class="file-icon"><i class="fa fa-cloud-download" /></span>
<div class="file-main">
<strong :title="file.name">{{ file.name }}</strong>
<span>{{ formatSize(file.size) }}<template v-if="file.count"> · {{ file.count.toLocaleString() }} </template></span>
</div>
<span class="file-status"><i class="fa fa-check-circle" /> 拉取成功</span>
<el-button link type="danger" @click="emit('remove-file', file.uid)">删除</el-button>
</div>
</div>
<el-pagination
v-if="uploadedFiles.length > FILE_PAGE_SIZE"
v-model:current-page="currentFilePage"
:page-size="FILE_PAGE_SIZE"
:total="uploadedFiles.length"
:pager-count="5"
small
background
layout="prev, pager, next"
class="uploaded-file-pagination"
/>
</section>
</div>
</div>
<div v-else class="form-section upload-section">
<div class="section-title-row">
<div>
<h3>源数据上传</h3>
@@ -267,7 +428,7 @@ defineExpose({ validate })
.type-options {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 16px;
width: 100%;
}
@@ -341,6 +502,44 @@ defineExpose({ validate })
opacity: 1;
}
.external-section {
.external-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 18px 20px;
margin-top: 16px;
}
:deep(.el-form-item) {
margin-bottom: 0;
}
:deep(.el-select),
:deep(.el-input),
:deep(.el-input-number) {
width: 100%;
}
}
.external-actions {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 12px;
margin-top: 22px;
}
.external-status {
display: inline-flex;
align-items: center;
gap: 6px;
font-size: 12px;
&.is-connected {
color: #2ca66a;
}
}
.upload-section :deep(.el-upload) {
width: 100%;
margin-top: 16px;
@@ -451,6 +650,10 @@ defineExpose({ validate })
grid-template-columns: minmax(0, 1fr);
}
.external-section .external-grid {
grid-template-columns: minmax(0, 1fr);
}
.uploaded-file {
gap: 8px;
padding: 8px 10px;

View File

@@ -1,7 +1,17 @@
export type ProcessType = 'structured' | 'unstructured'
export type ProcessType = 'structured' | 'unstructured' | 'external'
export type StepId = 'create' | 'preview' | 'generate' | 'results'
export interface ExternalDataSource {
type: string
url: string
authMode: string
username?: string
password?: string
token?: string
limit: number
}
export interface SourceLine {
number: number
content: string