feat: 新增外部数据源拉取与 DPO 输出格式支持

- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源
- 新增 DPO (Direct Preference Optimization) 输出类型
- 支持 chosen/rejected 字段的编辑、校验和发布
- 完善数据预处理切分逻辑和元数据管理
- 移除 OCR 扫描 PDF 功能，保持基础文本解析能力

Co-Authored-By: Claude <noreply@anthropic.com>

frontend/src/views/data-process/DataProcessCreateView.vue

@@ -20,6 +20,7 @@ import {
 import { useDataProcessGeneration } from './create/useDataProcessGeneration'
 import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
 import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
+import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
 import {
   loadCanonicalSourceContent,
   mapDataProcessSourceFile,
@@ -39,26 +40,20 @@ import {
   updateDataProcessPreview,
   updateDataProcessTask,
   updateDataProcessWorkflowStep,
-  type DataProcessExternalSourcePayload,
   type DataProcessPreviewItem,
 } from '@/api/modules/dataProcess'
-import type {
-  DataProcessConfig,
-  DataProcessPreviewProgress,
-  DataProcessTask,
-  DataProcessWorkflowStep,
-} from '@/types/dataProcess'
+import type { DataProcessConfig, DataProcessPreviewProgress, DataProcessTask, DataProcessWorkflowStep } from '@/types/dataProcess'
 import type {
   ExternalDataSource,
   GenerationControlOptions,
   PreviewItem,
   ProcessType,
+  SourceMode,
   StepId,
   StructuredProcessOptions,
   UnstructuredProcessOptions,
   UploadedDataFile,
 } from './create/types'
-
 const router = useRouter()
 const modelsStore = useModelsStore()
 const { list: modelList, loaded: modelsLoaded } = storeToRefs(modelsStore)
@@ -71,7 +66,7 @@ const PREVIEW_MODEL_VERSION = 'backend-pipeline-v4'
 const WIZARD_STEPS = [
   { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
   { id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
-  { id: 'upload', title: '上传文件', desc: '上传或接入待处理的源数据' },
+  { id: 'upload', title: '数据来源', desc: '选择本地上传或外部数据源拉取' },
   { id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
   { id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
   { id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
@@ -81,11 +76,13 @@ const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id
 const task = reactive({ name: '', description: '' })
 const taskId = ref<string | null>(null)
 const processType = ref<ProcessType>('structured')
+const sourceMode = ref<SourceMode>('local')
 const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
 const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
 const modelSelectionOptions = computed<GenerationControlOptions>(() => (
   processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
 ))
+const activeOutputType = computed(() => modelSelectionOptions.value.outputType)
 const uploadedFiles = ref<UploadedDataFile[]>([])
 const previewBuilding = ref(false)
 const {
@@ -93,16 +90,7 @@ const {
   startPreviewBuild,
   stopPreviewPolling,
 } = useDataProcessPreviewBuild()
-const externalSource = reactive<ExternalDataSource>({
-  type: 'postgresql',
-  url: '',
-  authMode: 'none',
-  username: '',
-  password: '',
-  limit: 1000,
-  query: '',
-  fileName: 'external-data.jsonl',
-})
+const externalSource = reactive<ExternalDataSource>(createDefaultExternalSource())
 const externalPulling = ref(false)
 const externalConnected = ref(false)
 const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
@@ -135,6 +123,7 @@ const {
 } = useDataProcessGeneration({
   taskId,
   dirty,
+  outputType: activeOutputType,
   beforeGenerate: beforeStartGeneration,
 })
 const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
@@ -174,7 +163,7 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
 }))
 const primaryActionLabel = computed(() => {
   if (currentStepId.value === 'create') return '继续：选择大模型'
-  if (currentStepId.value === 'model') return '继续：上传文件'
+  if (currentStepId.value === 'model') return '继续：选择数据来源'
   if (currentStepId.value === 'upload') {
     if (sourceUploading.value) return '正在上传'
     return previewBuilding.value ? '正在切分' : '继续：数据预览'
@@ -198,7 +187,6 @@ function goToStep(stepId: StepId) {
   const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
   if (nextStepIndex >= 0) currentStep.value = nextStepIndex
 }
-
 function updateModelSelectionOptions(value: GenerationControlOptions) {
   if (processType.value === 'unstructured') {
     unstructuredOptions.value = { ...unstructuredOptions.value, ...value }
@@ -206,12 +194,12 @@ function updateModelSelectionOptions(value: GenerationControlOptions) {
   }
   structuredOptions.value = { ...structuredOptions.value, ...value }
 }
-
 function toBackendConfig(): DataProcessConfig {
   const options = processType.value === 'unstructured'
     ? unstructuredOptions.value
     : structuredOptions.value
   const common = {
+    ...sourceConfigForBackend(sourceMode.value, externalSource),
     preprocess_options: [...options.preprocessOptions],
     semantic_enrichment: options.semanticEnrichment,
     dataset_split: { ...options.datasetSplit },
@@ -247,7 +235,6 @@ function toBackendConfig(): DataProcessConfig {
     qa_pairs_per_row: structuredOptions.value.qaPairsPerRow,
   }
 }
-
 function taskPayload() {
   return {
     name: task.name.trim(),
@@ -256,12 +243,10 @@ function taskPayload() {
     config: toBackendConfig(),
   }
 }
-
 async function persistWorkflowStep(step: StepId) {
   if (!taskId.value) return
   await updateDataProcessWorkflowStep(taskId.value, step as DataProcessWorkflowStep)
 }
-
 async function saveTaskConfiguration() {
   if (isRegeneration.value) {
     const regenerated = await prepareRegeneration(taskPayload())
@@ -277,19 +262,6 @@ async function saveTaskConfiguration() {
   return saved
 }
 
-function externalPayload(): DataProcessExternalSourcePayload {
-  return {
-    type: 'postgresql',
-    url: externalSource.url.trim(),
-    auth_mode: externalSource.authMode,
-    username: externalSource.username || undefined,
-    password: externalSource.password || undefined,
-    limit: externalSource.limit,
-    query: externalSource.query?.trim() || undefined,
-    file_name: externalSource.fileName || 'external-data.jsonl',
-  }
-}
-
 function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
   const sourceLocator = item.quality_score?.source_locator
   return {
@@ -378,7 +350,7 @@ async function beforeStartGeneration() {
   )
 }
 watch(
-  [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
+  [() => task.name, () => task.description, processType, sourceMode, structuredOptions, unstructuredOptions, externalSource],
   () => {
     if (!hydrating.value) dirty.value = true
   },
@@ -396,6 +368,13 @@ watch(processType, (nextType, previousType) => {
   ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
 })
 
+watch(processType, (nextType) => {
+  if (nextType === 'unstructured' && sourceMode.value === 'external') {
+    sourceMode.value = 'local'
+    externalConnected.value = false
+  }
+})
+
 watch(generationOptionsSignature, (currentSignature, previousSignature) => {
   if (hydrating.value) return
   if (currentSignature === previousSignature) return
@@ -453,6 +432,16 @@ function updateExternalSource(value: ExternalDataSource) {
   externalConnected.value = false
 }
 
+async function updateSourceMode(value: SourceMode) {
+  const previous = sourceMode.value
+  sourceMode.value = value
+  externalConnected.value = false
+  dirty.value = true
+  externalPulling.value = true
+  try { await saveTaskConfiguration() } catch { sourceMode.value = previous }
+  finally { externalPulling.value = false }
+}
+
 async function handleTestConnection() {
   if (!taskId.value) {
     ElMessage.error('任务尚未创建，请返回模型选择步骤后重试')
@@ -464,7 +453,8 @@ async function handleTestConnection() {
   }
   externalPulling.value = true
   try {
-    const result = await testDataProcessExternalSource(taskId.value, externalPayload())
+    await saveTaskConfiguration()
+    const result = await testDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
     externalConnected.value = result.connected
     if (result.connected) ElMessage.success(result.message || '数据源连接测试成功')
     else ElMessage.warning(result.message || '数据源连接失败')
@@ -490,7 +480,8 @@ async function handlePullData() {
   }
   externalPulling.value = true
   try {
-    const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
+    await saveTaskConfiguration()
+    const response = await pullDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
     const newFiles: UploadedDataFile[] = []
     for (const file of response.files) {
       const content = await loadCanonicalSourceContent(taskId.value, file.id)
@@ -620,6 +611,12 @@ function applyPreviewProgress(progress: DataProcessPreviewProgress) {
 
 async function completePreviewWorkspace() {
   previewItems.value = await loadAllPreviewItems()
+  for (const file of uploadedFiles.value) {
+    if (!file.content && file.sourceFileId && file.fileFormat?.replace('.', '') === 'pdf') {
+      file.content = await loadCanonicalSourceContent(taskId.value as string, file.sourceFileId)
+        .catch(() => file.content)
+    }
+  }
   const configSignature = buildPreviewConfigSignature()
   const previewCounts = new Map<string, number>()
   for (const item of previewItems.value) {
@@ -678,7 +675,7 @@ async function nextFromUpload() {
     return
   }
   if (uploadedFiles.value.length === 0) {
-    ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
+    ElMessage.warning(sourceMode.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
     return
   }
   const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
@@ -992,6 +989,9 @@ async function initializeExistingWorkflow() {
   const sourceTask = await loadRegenerationSource()
   if (!sourceTask) return
   taskId.value = String(sourceTask.id)
+  const restoredSource = restoreExternalSourceConfig(sourceTask.config || {})
+  sourceMode.value = restoredSource.mode
+  Object.assign(externalSource, restoredSource.source)
   if (!isWorkflowResume.value) return
   if (sourceTask.status === 'completed' && sourceTask.results_confirmed !== false) {
     allowLeave = true
@@ -1102,6 +1102,7 @@ onMounted(() => {
           <SourceUploadStep
             v-else-if="currentStepId === 'upload'"
             :process-type="processType"
+            :source-mode="sourceMode"
             :uploaded-files="uploadedFiles"
             :external-source="externalSource"
             :external-pulling="externalPulling"
@@ -1109,6 +1110,7 @@ onMounted(() => {
             :preview-building="previewBuilding"
             :source-uploading="sourceUploading"
             @update:external-source="updateExternalSource"
+            @update:source-mode="updateSourceMode"
             @file-change="handleFileChange"
             @remove-file="handleRemoveFile"
             @use-sample="useSampleFile"
@@ -1154,6 +1156,7 @@ onMounted(() => {
             :preview-items="previewItems"
             :regenerating-result-id="regeneratingResultId"
             :bulk-regeneration="bulkRegeneration"
+            :output-type="activeOutputType"
             @update:field="updateResultField"
             @regenerate:all="regenerateAllResults"
             @regenerate:item="regenerateResult"
@@ -1179,8 +1182,8 @@ onMounted(() => {
         <el-button
           class="wizard-primary-action"
           type="primary"
-          :loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
-          :disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
+          :loading="externalPulling || modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
+          :disabled="externalPulling || hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
           @click="handlePrimaryAction"
         >
           {{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />