feat: 新增外部数据源拉取与 DPO 输出格式支持

- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源
- 新增 DPO (Direct Preference Optimization) 输出类型
- 支持 chosen/rejected 字段的编辑、校验和发布
- 完善数据预处理切分逻辑和元数据管理
- 移除 OCR 扫描 PDF 功能,保持基础文本解析能力

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
caoxiaozhu
2026-08-11 14:17:45 +08:00
parent f809825a7d
commit 5f6e7523cf
26 changed files with 927 additions and 144 deletions

View File

@@ -1,6 +1,7 @@
<script setup lang="ts">
import { computed, ref } from 'vue'
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
import type { DataProcessOutputType } from '@/types/dataProcess'
const props = defineProps<{
items: ResultItem[]
@@ -8,11 +9,12 @@ const props = defineProps<{
selectedId: string | null
regeneratingResultId: string | null
bulkRegeneration: BulkResultRegenerationState
outputType: DataProcessOutputType
}>()
const emit = defineEmits<{
'update:selectedId': [value: string]
'update:field': [id: string, field: 'instruction' | 'input' | 'output', value: string]
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
'regenerate:item': [id: string]
'regenerate:all': []
}>()
@@ -59,6 +61,11 @@ const selectedSourceMeta = computed(() => {
const source = selectedSource.value
if (!source) return ''
const parts: string[] = []
if (source.sourcePages?.length) {
const first = source.sourcePages[0]
const last = source.sourcePages[source.sourcePages.length - 1]
parts.push(first === last ? `${first}` : `${first}${last}`)
}
if (source.sourceStartLine != null) {
parts.push(
source.sourceEndLine != null && source.sourceEndLine !== source.sourceStartLine
@@ -75,6 +82,8 @@ const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !keyword
|| item.instruction.toLowerCase().includes(keyword)
|| item.output.toLowerCase().includes(keyword)
|| item.chosen.toLowerCase().includes(keyword)
|| item.rejected.toLowerCase().includes(keyword)
|| String(index + 1).includes(keyword)
return matchesSearch && (!invalidOnly.value || item.status === 'invalid')
}))
@@ -135,7 +144,7 @@ function selectRelative(offset: number) {
<span class="result-index">#{{ String(items.findIndex((entry) => entry.id === item.id) + 1).padStart(3, '0') }}</span>
<span class="result-copy">
<strong>{{ item.instruction || '未填写指令' }}</strong>
<small>{{ item.output || '未填写输出' }}</small>
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
</span>
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
<i
@@ -207,7 +216,7 @@ function selectRelative(offset: number) {
@update:model-value="emit('update:field', selectedItem.id, 'input', $event)"
/>
</div>
<div class="field-editor">
<div v-if="outputType !== 'dpo'" class="field-editor">
<label>Output <em>必填</em></label>
<el-input
:model-value="selectedItem.output"
@@ -217,6 +226,28 @@ function selectRelative(offset: number) {
@update:model-value="emit('update:field', selectedItem.id, 'output', $event)"
/>
</div>
<template v-else>
<div class="field-editor dpo-field is-chosen">
<label>Chosen <em>优选回答必填</em></label>
<el-input
:model-value="selectedItem.chosen"
:disabled="selectedItemRegenerating"
type="textarea"
:rows="6"
@update:model-value="emit('update:field', selectedItem.id, 'chosen', $event)"
/>
</div>
<div class="field-editor dpo-field is-rejected">
<label>Rejected <em>拒选回答必填</em></label>
<el-input
:model-value="selectedItem.rejected"
:disabled="selectedItemRegenerating"
type="textarea"
:rows="6"
@update:model-value="emit('update:field', selectedItem.id, 'rejected', $event)"
/>
</div>
</template>
<div v-if="selectedItem.error" class="validation-error">
<i class="fa fa-exclamation-circle" /> {{ selectedItem.error }}
</div>