This commit is contained in:
wangjiming
2026-08-11 15:45:23 +08:00
26 changed files with 927 additions and 144 deletions

View File

@@ -101,7 +101,9 @@ assert.match(detailSource, /!\/\(\?:password\|secret\|token\|api_key\)\/i\.test\
assert.match(detailSource, /key !== 'generation_model_snapshot'/, '处理配置仍直接展示内部模型快照')
assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文')
assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称')
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举')
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : value === 'dpo' \? 'DPO 偏好对' : '标准回答'/, '处理配置没有转换输出类型枚举')
assert.match(detailSource, /prop="chosen"[\s\S]*?prop="rejected"/, 'DPO 结果没有展示 Chosen 与 Rejected')
assert.match(detailSource, /publishForm\.format = isDpoOutput\.value \? 'dpo'/, 'DPO 发布没有锁定原生格式')
assert.match(detailSource, /reasoning_detail:\s*'推理详细程度'/, '处理配置没有显示推理详细程度名称')
assert.match(detailSource, /value === 'detailed' \? '详细推理' : '普通推理'/, '处理配置没有转换推理详细程度枚举')
assert.match(detailSource, /key !== 'reasoning_detail' \|\| config\.output_type === 'reasoning'/, '标准回答任务不应展示无关的推理详细程度')

View File

@@ -122,13 +122,13 @@ assert.match(regenerationSource, /route\.name === 'data-process-regenerate'[\s\S
assert.match(viewSource, /const currentStep = ref\(0\)/, '重新生成必须从向导第一步开始')
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
for (const title of ['创建任务', '大模型选择', '上传文件', '数据预览', '开始生成', '结果编辑与保存']) {
for (const title of ['创建任务', '大模型选择', '数据来源', '数据预览', '开始生成', '结果编辑与保存']) {
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
}
assert.match(
viewSource,
/\{ id: 'create',[\s\S]*?\{ id: 'model',[\s\S]*?\{ id: 'upload',[\s\S]*?\{ id: 'preview',[\s\S]*?\{ id: 'generate',[\s\S]*?\{ id: 'results'/,
'六步向导顺序必须为创建任务、大模型选择、上传文件、数据预览、开始生成、结果编辑与保存',
'六步向导顺序必须为创建任务、大模型选择、数据来源、数据预览、开始生成、结果编辑与保存',
)
assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减')
assert.match(
@@ -380,7 +380,7 @@ for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploa
assert.match(viewSource, /<ModelSelectionStep\s+[\s\S]*?v-else-if="currentStepId === 'model'"/, '第二步没有挂载独立大模型选择组件')
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第三步没有挂载独立上传组件')
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续:选择大模型'/, '第一步主按钮没有指向大模型选择')
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续:上传文件'/, '第二步主按钮没有指向上传文件')
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续:选择数据来源'/, '第二步主按钮没有指向数据来源选择')
assert.match(
viewSource,
/if \(currentStepId\.value === 'upload'\) \{[\s\S]*?sourceUploading\.value[\s\S]*?'正在上传'[\s\S]*?previewBuilding\.value \? '正在切分' : '继续:数据预览'/,
@@ -1032,7 +1032,7 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
assert.match(stateSource, /const outputType = configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
assert.match(stateSource, /configuredOutputType === 'reasoning' \|\| configuredOutputType === 'dpo'[\s\S]*?\? configuredOutputType[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
assert.match(stateSource, /reasoningDetail:\s*configValue\(config, 'reasoning_detail', defaults\.reasoningDetail\) === 'detailed'[\s\S]*?\? 'detailed'[\s\S]*?: 'normal'/, '推理详细程度没有从任务配置安全回填')
assert.match(stateSource, /isBuiltInGenerationPrompt\(configuredPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)/, '旧版内置提示语没有按输出类型迁移')
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
@@ -1412,7 +1412,7 @@ assert.match(
)
assert.match(
sourceUploadSource,
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
'无文件时未保留原有大拖拽上传区或上传配置',
)
assert.match(
@@ -1432,7 +1432,7 @@ assert.ok(sourceUploadSource.includes('旧版 DOC/PPT 请先转换'), '非结构
assert.ok(sourceUploadSource.includes('旧版 XLS 请先转换'), '结构化格式提示没有说明旧版 XLS 需转换')
assert.match(
sourceUploadSource,
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding \|\| externalPulling">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
'有文件时缺少标题右侧的继续上传触发器或上传配置',
)
assert.match(
@@ -1476,4 +1476,13 @@ assert.match(
)
assert.match(previewSource, /height:\s*clamp\(560px,\s*calc\(100vh - 370px\),\s*720px\)/, '对照预览高度不足以展示切片正文')
assert.doesNotMatch(taskSetupSource, /外来数据源拉取/, '外部数据源仍错误地放在第一步处理类型中')
assert.match(sourceUploadSource, /<h3>数据来源<\/h3>[\s\S]*?本地上传[\s\S]*?外部数据源/, '第三步缺少本地与外部数据来源选择')
for (const field of ['地址 / URL', '鉴权方式', 'SSL 模式', '连接超时', '查询超时', '只读查询语句', '拉取条数', '落地文件名']) {
assert.ok(sourceUploadSource.includes(field), `外部数据源标准配置缺少:${field}`)
}
assert.match(generationControlSource, /label="DPO 偏好对" value="dpo"/, '输出类型缺少 DPO 偏好对')
assert.match(resultEditorSource, /Chosen[\s\S]*?Rejected/, '结果编辑器缺少 DPO 成对字段')
assert.match(viewSource, /sourceConfigForBackend\(sourceMode\.value, externalSource\)/, '任务配置没有保存第三步数据来源模式')
console.log('数据处理六步向导回归检查通过')

View File

@@ -4,7 +4,7 @@ export type DataProcessStatus = 'pending' | 'running' | 'completed' | 'failed' |
export type DataProcessType = 'structured' | 'unstructured' | 'external'
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
export type DataProcessSplit = 'train' | 'validation' | 'test'
export type DataProcessOutputType = 'standard' | 'reasoning'
export type DataProcessOutputType = 'standard' | 'reasoning' | 'dpo'
export type DataProcessReasoningDetail = 'normal' | 'detailed'
export type DataProcessWorkflowStep = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
export type DataProcessPreviewStatus = 'idle' | 'queued' | 'running' | 'completed' | 'failed' | 'cancelled'
@@ -221,6 +221,9 @@ export interface DataProcessExternalSourcePayload {
username?: string
password?: string
limit: number
connect_timeout_seconds: number
statement_timeout_seconds: number
ssl_mode: 'disable' | 'prefer' | 'require' | 'verify-ca' | 'verify-full'
query?: string
file_name?: string
}
@@ -342,9 +345,13 @@ export interface DataProcessResult {
instruction: string
input: string
output: string
chosen?: string
rejected?: string
original_instruction?: string | null
original_input?: string | null
original_output?: string | null
original_chosen?: string | null
original_rejected?: string | null
status: DataProcessResultStatus
error?: string | null
split?: DataProcessSplit | null
@@ -356,6 +363,8 @@ export interface DataProcessResultUpdatePayload {
instruction: string
input: string
output: string
chosen?: string
rejected?: string
expected_updated_at?: string
}
@@ -410,7 +419,7 @@ export interface DataProcessPublishPayload {
dataset_type: 'train' | 'test' | 'eval' | 'val' | 'other'
storage_type: 'local'
split: DataProcessDatasetSplit
format: 'alpaca_jsonl' | 'jsonl'
format: 'alpaca_jsonl' | 'jsonl' | 'dpo'
}
export interface DataProcessPublishResult {

View File

@@ -20,6 +20,7 @@ import {
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
@@ -39,26 +40,20 @@ import {
updateDataProcessPreview,
updateDataProcessTask,
updateDataProcessWorkflowStep,
type DataProcessExternalSourcePayload,
type DataProcessPreviewItem,
} from '@/api/modules/dataProcess'
import type {
DataProcessConfig,
DataProcessPreviewProgress,
DataProcessTask,
DataProcessWorkflowStep,
} from '@/types/dataProcess'
import type { DataProcessConfig, DataProcessPreviewProgress, DataProcessTask, DataProcessWorkflowStep } from '@/types/dataProcess'
import type {
ExternalDataSource,
GenerationControlOptions,
PreviewItem,
ProcessType,
SourceMode,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
UploadedDataFile,
} from './create/types'
const router = useRouter()
const modelsStore = useModelsStore()
const { list: modelList, loaded: modelsLoaded } = storeToRefs(modelsStore)
@@ -71,7 +66,7 @@ const PREVIEW_MODEL_VERSION = 'backend-pipeline-v4'
const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
{ id: 'upload', title: '上传文件', desc: '上传或接入待处理的源数据' },
{ id: 'upload', title: '数据来源', desc: '选择本地上传或外部数据源拉取' },
{ id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
{ id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
@@ -81,11 +76,13 @@ const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id
const task = reactive({ name: '', description: '' })
const taskId = ref<string | null>(null)
const processType = ref<ProcessType>('structured')
const sourceMode = ref<SourceMode>('local')
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
const modelSelectionOptions = computed<GenerationControlOptions>(() => (
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
))
const activeOutputType = computed(() => modelSelectionOptions.value.outputType)
const uploadedFiles = ref<UploadedDataFile[]>([])
const previewBuilding = ref(false)
const {
@@ -93,16 +90,7 @@ const {
startPreviewBuild,
stopPreviewPolling,
} = useDataProcessPreviewBuild()
const externalSource = reactive<ExternalDataSource>({
type: 'postgresql',
url: '',
authMode: 'none',
username: '',
password: '',
limit: 1000,
query: '',
fileName: 'external-data.jsonl',
})
const externalSource = reactive<ExternalDataSource>(createDefaultExternalSource())
const externalPulling = ref(false)
const externalConnected = ref(false)
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
@@ -135,6 +123,7 @@ const {
} = useDataProcessGeneration({
taskId,
dirty,
outputType: activeOutputType,
beforeGenerate: beforeStartGeneration,
})
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
@@ -174,7 +163,7 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
}))
const primaryActionLabel = computed(() => {
if (currentStepId.value === 'create') return '继续:选择大模型'
if (currentStepId.value === 'model') return '继续:上传文件'
if (currentStepId.value === 'model') return '继续:选择数据来源'
if (currentStepId.value === 'upload') {
if (sourceUploading.value) return '正在上传'
return previewBuilding.value ? '正在切分' : '继续:数据预览'
@@ -198,7 +187,6 @@ function goToStep(stepId: StepId) {
const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
if (nextStepIndex >= 0) currentStep.value = nextStepIndex
}
function updateModelSelectionOptions(value: GenerationControlOptions) {
if (processType.value === 'unstructured') {
unstructuredOptions.value = { ...unstructuredOptions.value, ...value }
@@ -206,12 +194,12 @@ function updateModelSelectionOptions(value: GenerationControlOptions) {
}
structuredOptions.value = { ...structuredOptions.value, ...value }
}
function toBackendConfig(): DataProcessConfig {
const options = processType.value === 'unstructured'
? unstructuredOptions.value
: structuredOptions.value
const common = {
...sourceConfigForBackend(sourceMode.value, externalSource),
preprocess_options: [...options.preprocessOptions],
semantic_enrichment: options.semanticEnrichment,
dataset_split: { ...options.datasetSplit },
@@ -247,7 +235,6 @@ function toBackendConfig(): DataProcessConfig {
qa_pairs_per_row: structuredOptions.value.qaPairsPerRow,
}
}
function taskPayload() {
return {
name: task.name.trim(),
@@ -256,12 +243,10 @@ function taskPayload() {
config: toBackendConfig(),
}
}
async function persistWorkflowStep(step: StepId) {
if (!taskId.value) return
await updateDataProcessWorkflowStep(taskId.value, step as DataProcessWorkflowStep)
}
async function saveTaskConfiguration() {
if (isRegeneration.value) {
const regenerated = await prepareRegeneration(taskPayload())
@@ -277,19 +262,6 @@ async function saveTaskConfiguration() {
return saved
}
function externalPayload(): DataProcessExternalSourcePayload {
return {
type: 'postgresql',
url: externalSource.url.trim(),
auth_mode: externalSource.authMode,
username: externalSource.username || undefined,
password: externalSource.password || undefined,
limit: externalSource.limit,
query: externalSource.query?.trim() || undefined,
file_name: externalSource.fileName || 'external-data.jsonl',
}
}
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
const sourceLocator = item.quality_score?.source_locator
return {
@@ -378,7 +350,7 @@ async function beforeStartGeneration() {
)
}
watch(
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
[() => task.name, () => task.description, processType, sourceMode, structuredOptions, unstructuredOptions, externalSource],
() => {
if (!hydrating.value) dirty.value = true
},
@@ -396,6 +368,13 @@ watch(processType, (nextType, previousType) => {
ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据')
})
watch(processType, (nextType) => {
if (nextType === 'unstructured' && sourceMode.value === 'external') {
sourceMode.value = 'local'
externalConnected.value = false
}
})
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
if (hydrating.value) return
if (currentSignature === previousSignature) return
@@ -453,6 +432,16 @@ function updateExternalSource(value: ExternalDataSource) {
externalConnected.value = false
}
async function updateSourceMode(value: SourceMode) {
const previous = sourceMode.value
sourceMode.value = value
externalConnected.value = false
dirty.value = true
externalPulling.value = true
try { await saveTaskConfiguration() } catch { sourceMode.value = previous }
finally { externalPulling.value = false }
}
async function handleTestConnection() {
if (!taskId.value) {
ElMessage.error('任务尚未创建,请返回模型选择步骤后重试')
@@ -464,7 +453,8 @@ async function handleTestConnection() {
}
externalPulling.value = true
try {
const result = await testDataProcessExternalSource(taskId.value, externalPayload())
await saveTaskConfiguration()
const result = await testDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
externalConnected.value = result.connected
if (result.connected) ElMessage.success(result.message || '数据源连接测试成功')
else ElMessage.warning(result.message || '数据源连接失败')
@@ -490,7 +480,8 @@ async function handlePullData() {
}
externalPulling.value = true
try {
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
await saveTaskConfiguration()
const response = await pullDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
const newFiles: UploadedDataFile[] = []
for (const file of response.files) {
const content = await loadCanonicalSourceContent(taskId.value, file.id)
@@ -620,6 +611,12 @@ function applyPreviewProgress(progress: DataProcessPreviewProgress) {
async function completePreviewWorkspace() {
previewItems.value = await loadAllPreviewItems()
for (const file of uploadedFiles.value) {
if (!file.content && file.sourceFileId && file.fileFormat?.replace('.', '') === 'pdf') {
file.content = await loadCanonicalSourceContent(taskId.value as string, file.sourceFileId)
.catch(() => file.content)
}
}
const configSignature = buildPreviewConfigSignature()
const previewCounts = new Map<string, number>()
for (const item of previewItems.value) {
@@ -678,7 +675,7 @@ async function nextFromUpload() {
return
}
if (uploadedFiles.value.length === 0) {
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
ElMessage.warning(sourceMode.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
return
}
const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
@@ -992,6 +989,9 @@ async function initializeExistingWorkflow() {
const sourceTask = await loadRegenerationSource()
if (!sourceTask) return
taskId.value = String(sourceTask.id)
const restoredSource = restoreExternalSourceConfig(sourceTask.config || {})
sourceMode.value = restoredSource.mode
Object.assign(externalSource, restoredSource.source)
if (!isWorkflowResume.value) return
if (sourceTask.status === 'completed' && sourceTask.results_confirmed !== false) {
allowLeave = true
@@ -1102,6 +1102,7 @@ onMounted(() => {
<SourceUploadStep
v-else-if="currentStepId === 'upload'"
:process-type="processType"
:source-mode="sourceMode"
:uploaded-files="uploadedFiles"
:external-source="externalSource"
:external-pulling="externalPulling"
@@ -1109,6 +1110,7 @@ onMounted(() => {
:preview-building="previewBuilding"
:source-uploading="sourceUploading"
@update:external-source="updateExternalSource"
@update:source-mode="updateSourceMode"
@file-change="handleFileChange"
@remove-file="handleRemoveFile"
@use-sample="useSampleFile"
@@ -1154,6 +1156,7 @@ onMounted(() => {
:preview-items="previewItems"
:regenerating-result-id="regeneratingResultId"
:bulk-regeneration="bulkRegeneration"
:output-type="activeOutputType"
@update:field="updateResultField"
@regenerate:all="regenerateAllResults"
@regenerate:item="regenerateResult"
@@ -1179,8 +1182,8 @@ onMounted(() => {
<el-button
class="wizard-primary-action"
type="primary"
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
:loading="externalPulling || modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="externalPulling || hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

View File

@@ -53,7 +53,7 @@ const resultCellTooltipOptions = {
} as const
let resultFilterTimer: ReturnType<typeof setTimeout> | null = null
const editForm = reactive({ instruction: '', input: '', output: '' })
const editForm = reactive({ instruction: '', input: '', output: '', chosen: '', rejected: '' })
const publishForm = reactive<DataProcessPublishPayload>({
dataset_name: '',
dataset_type: 'train',
@@ -69,6 +69,8 @@ const processTypeMap: Record<DataProcessType, string> = {
}
const configLabelMap: Record<string, string> = {
source_mode: '数据来源',
external_source: '外部数据源配置',
preprocess_options: '预处理规则',
dataset_split: '数据集划分',
generation_model_id: '数据生成模型',
@@ -212,6 +214,7 @@ const outputDatasetName = computed(() => (
))
const outputDatasetId = computed(() => detail.value?.output_dataset_id || null)
const hasPublishedOutputs = computed(() => outputDatasets.value.length > 0)
const isDpoOutput = computed(() => detail.value?.config?.output_type === 'dpo')
const hasCurrentPublishedDataset = computed(() => (
Boolean(outputDatasetId.value)
&& outputDatasets.value.some((dataset) => String(dataset.id) === String(outputDatasetId.value))
@@ -294,6 +297,18 @@ const configRows = computed(() => {
})
function formatConfigValue(key: string, value: unknown) {
if (key === 'source_mode') return value === 'external' ? '外部数据源' : '本地上传'
if (key === 'external_source' && value && typeof value === 'object') {
const source = value as Record<string, unknown>
return [
source.type,
source.url,
source.ssl_mode ? `SSL ${source.ssl_mode}` : '',
source.limit ? `上限 ${source.limit}` : '',
source.connect_timeout_seconds ? `连接 ${source.connect_timeout_seconds}s` : '',
source.statement_timeout_seconds ? `查询 ${source.statement_timeout_seconds}s` : '',
].filter(Boolean).join(' · ')
}
if (key === 'generation_model_id') {
const snapshot = detail.value?.config?.generation_model_snapshot
if (snapshot && typeof snapshot === 'object' && !Array.isArray(snapshot)) {
@@ -305,7 +320,7 @@ function formatConfigValue(key: string, value: unknown) {
return chunkMethodLabelMap[value] || value
}
if (key === 'output_type') {
return value === 'reasoning' ? '思维链回答' : '标准回答'
return value === 'reasoning' ? '思维链回答' : value === 'dpo' ? 'DPO 偏好对' : '标准回答'
}
if (key === 'reasoning_detail') {
return value === 'detailed' ? '详细推理' : '普通推理'
@@ -461,14 +476,26 @@ function openResultEditor(result: DataProcessResult) {
instruction: result.instruction,
input: result.input,
output: result.output,
chosen: result.chosen || result.output || '',
rejected: result.rejected || '',
})
editDialogVisible.value = true
}
async function saveResult() {
if (!editingResult.value) return
if (!editForm.instruction.trim() || !editForm.output.trim()) {
ElMessage.warning('Instruction 和 Output 不能为空')
const invalid = isDpoOutput.value
? !editForm.instruction.trim()
|| !editForm.chosen.trim()
|| !editForm.rejected.trim()
|| editForm.chosen.trim() === editForm.rejected.trim()
: !editForm.instruction.trim() || !editForm.output.trim()
if (invalid) {
ElMessage.warning(
isDpoOutput.value
? 'Instruction、Chosen、Rejected 均不能为空,且两个回答不能相同'
: 'Instruction 和 Output 不能为空',
)
return
}
@@ -478,6 +505,8 @@ async function saveResult() {
instruction: editForm.instruction,
input: editForm.input,
output: editForm.output,
chosen: editForm.chosen,
rejected: editForm.rejected,
expected_updated_at: editingResult.value.updated_at,
})
replaceResult(updated)
@@ -530,6 +559,7 @@ function openPublishDialog() {
if (!detail.value) return
publishForm.dataset_name = `${detail.value.name}-数据集`
publishForm.split = configuredSplit()
publishForm.format = isDpoOutput.value ? 'dpo' : 'alpaca_jsonl'
publishDialogVisible.value = true
}
@@ -808,7 +838,11 @@ onBeforeUnmount(() => {
<el-table-column type="index" label="#" width="56" align="center" />
<el-table-column label="指令" min-width="190" show-overflow-tooltip prop="instruction" />
<el-table-column label="输入" min-width="160" show-overflow-tooltip prop="input" />
<el-table-column label="输出" min-width="230" show-overflow-tooltip prop="output" />
<el-table-column v-if="!isDpoOutput" label="输出" min-width="230" show-overflow-tooltip prop="output" />
<template v-else>
<el-table-column label="Chosen" min-width="210" show-overflow-tooltip prop="chosen" />
<el-table-column label="Rejected" min-width="210" show-overflow-tooltip prop="rejected" />
</template>
<el-table-column label="质量分" width="88" align="center">
<template #default="{ row }">
<el-tooltip :content="qualityFlagsLabel((row as DataProcessResult).quality_score)">
@@ -862,9 +896,17 @@ onBeforeUnmount(() => {
<el-form-item label="Input">
<el-input v-model="editForm.input" type="textarea" :rows="3" maxlength="10000" show-word-limit />
</el-form-item>
<el-form-item label="Output" required>
<el-form-item v-if="!isDpoOutput" label="Output" required>
<el-input v-model="editForm.output" type="textarea" :rows="6" maxlength="20000" show-word-limit />
</el-form-item>
<template v-else>
<el-form-item label="Chosen优选回答" required>
<el-input v-model="editForm.chosen" type="textarea" :rows="6" maxlength="20000" show-word-limit />
</el-form-item>
<el-form-item label="Rejected拒选回答" required>
<el-input v-model="editForm.rejected" type="textarea" :rows="6" maxlength="20000" show-word-limit />
</el-form-item>
</template>
</el-form>
<template #footer>
<el-button @click="editDialogVisible = false">取消</el-button>
@@ -884,7 +926,8 @@ onBeforeUnmount(() => {
</el-select>
</el-form-item>
<el-form-item label="输出格式">
<el-select v-model="publishForm.format">
<el-select v-model="publishForm.format" :disabled="isDpoOutput">
<el-option v-if="isDpoOutput" label="DPO JSONL" value="dpo" />
<el-option label="Alpaca JSONL" value="alpaca_jsonl" />
<el-option label="JSONL" value="jsonl" />
</el-select>

View File

@@ -76,7 +76,7 @@ function updateQualityRules(value: Array<string | number>) {
}
function updateOutputType(value: string | number | boolean | undefined) {
const outputType = value === 'reasoning' ? 'reasoning' : 'standard'
const outputType = value === 'reasoning' || value === 'dpo' ? value : 'standard'
emit('update:options', {
...props.options,
outputType,
@@ -153,6 +153,9 @@ function modelMeta(model: ModelItem) {
<small v-if="options.outputType === 'reasoning'">
当前使用思维链专用提示语系统还会按所选详细程度约束推理结构
</small>
<small v-else-if="options.outputType === 'dpo'">
当前使用 DPO 专用提示语模型会同时生成 Chosen Rejected 偏好回答
</small>
<small v-else>当前使用标准回答提示语只要求问题和最终答案</small>
</div>
<div class="prompt-input-wrapper">
@@ -237,7 +240,7 @@ function modelMeta(model: ModelItem) {
<div class="output-type-row">
<div class="field-copy">
<strong>输出类型</strong>
<small>控制答案是否包含可用于推理模型训练的思维链内容</small>
<small>选择标准监督思维链或 DPO 成对偏好数据</small>
</div>
<el-select
class="output-type-select"
@@ -247,6 +250,7 @@ function modelMeta(model: ModelItem) {
>
<el-option label="标准回答" value="standard" />
<el-option label="思维链回答" value="reasoning" />
<el-option label="DPO 偏好对" value="dpo" />
</el-select>
</div>
<div v-if="options.outputType === 'reasoning'" class="output-type-row">
@@ -272,6 +276,9 @@ function modelMeta(model: ModelItem) {
<template v-else>保留关键依据与必要步骤</template>
最终按 <code>&lt;think&gt;推理过程&lt;/think&gt;</code>
</template>
<template v-else-if="options.outputType === 'dpo'">
保存同一问题的 <code>chosen</code> 优选回答和 <code>rejected</code> 拒选回答用于直接偏好优化训练
</template>
<template v-else>仅保存最终答案不包含推理过程</template>
</p>
</div>

View File

@@ -1,6 +1,7 @@
<script setup lang="ts">
import { computed, ref } from 'vue'
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
import type { DataProcessOutputType } from '@/types/dataProcess'
const props = defineProps<{
items: ResultItem[]
@@ -8,11 +9,12 @@ const props = defineProps<{
selectedId: string | null
regeneratingResultId: string | null
bulkRegeneration: BulkResultRegenerationState
outputType: DataProcessOutputType
}>()
const emit = defineEmits<{
'update:selectedId': [value: string]
'update:field': [id: string, field: 'instruction' | 'input' | 'output', value: string]
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
'regenerate:item': [id: string]
'regenerate:all': []
}>()
@@ -59,6 +61,11 @@ const selectedSourceMeta = computed(() => {
const source = selectedSource.value
if (!source) return ''
const parts: string[] = []
if (source.sourcePages?.length) {
const first = source.sourcePages[0]
const last = source.sourcePages[source.sourcePages.length - 1]
parts.push(first === last ? `${first}` : `${first}${last}`)
}
if (source.sourceStartLine != null) {
parts.push(
source.sourceEndLine != null && source.sourceEndLine !== source.sourceStartLine
@@ -75,6 +82,8 @@ const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !keyword
|| item.instruction.toLowerCase().includes(keyword)
|| item.output.toLowerCase().includes(keyword)
|| item.chosen.toLowerCase().includes(keyword)
|| item.rejected.toLowerCase().includes(keyword)
|| String(index + 1).includes(keyword)
return matchesSearch && (!invalidOnly.value || item.status === 'invalid')
}))
@@ -135,7 +144,7 @@ function selectRelative(offset: number) {
<span class="result-index">#{{ String(items.findIndex((entry) => entry.id === item.id) + 1).padStart(3, '0') }}</span>
<span class="result-copy">
<strong>{{ item.instruction || '未填写指令' }}</strong>
<small>{{ item.output || '未填写输出' }}</small>
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
</span>
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
<i
@@ -207,7 +216,7 @@ function selectRelative(offset: number) {
@update:model-value="emit('update:field', selectedItem.id, 'input', $event)"
/>
</div>
<div class="field-editor">
<div v-if="outputType !== 'dpo'" class="field-editor">
<label>Output <em>必填</em></label>
<el-input
:model-value="selectedItem.output"
@@ -217,6 +226,28 @@ function selectRelative(offset: number) {
@update:model-value="emit('update:field', selectedItem.id, 'output', $event)"
/>
</div>
<template v-else>
<div class="field-editor dpo-field is-chosen">
<label>Chosen <em>优选回答必填</em></label>
<el-input
:model-value="selectedItem.chosen"
:disabled="selectedItemRegenerating"
type="textarea"
:rows="6"
@update:model-value="emit('update:field', selectedItem.id, 'chosen', $event)"
/>
</div>
<div class="field-editor dpo-field is-rejected">
<label>Rejected <em>拒选回答必填</em></label>
<el-input
:model-value="selectedItem.rejected"
:disabled="selectedItemRegenerating"
type="textarea"
:rows="6"
@update:model-value="emit('update:field', selectedItem.id, 'rejected', $event)"
/>
</div>
</template>
<div v-if="selectedItem.error" class="validation-error">
<i class="fa fa-exclamation-circle" /> {{ selectedItem.error }}
</div>

View File

@@ -1,10 +1,11 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import type { UploadFile } from 'element-plus'
import type { ExternalDataSource, ProcessType, UploadedDataFile } from './types'
import type { ExternalDataSource, ProcessType, SourceMode, UploadedDataFile } from './types'
const props = defineProps<{
processType: ProcessType
sourceMode: SourceMode
uploadedFiles: UploadedDataFile[]
externalSource: ExternalDataSource
externalPulling: boolean
@@ -14,6 +15,7 @@ const props = defineProps<{
}>()
const emit = defineEmits<{
'update:sourceMode': [value: SourceMode]
'update:externalSource': [value: ExternalDataSource]
'file-change': [file: UploadFile]
'remove-file': [uid: string | number]
@@ -31,6 +33,14 @@ const AUTH_MODES = [
{ value: 'basic', label: '账号密码' },
]
const SSL_MODES = [
{ value: 'disable', label: '关闭' },
{ value: 'prefer', label: '优先使用(默认)' },
{ value: 'require', label: '必须加密' },
{ value: 'verify-ca', label: '验证 CA' },
{ value: 'verify-full', label: '完整验证' },
]
const FILE_PAGE_SIZE = 10
const currentFilePage = ref(1)
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
@@ -45,7 +55,10 @@ const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
}
const isExternal = computed(() => props.processType === 'external')
const isExternal = computed(() => props.sourceMode === 'external')
const sourceModeLocked = computed(() => (
props.uploadedFiles.length > 0 || props.sourceUploading || props.previewBuilding || props.externalPulling
))
const uploadAccept = computed(() => props.processType === 'unstructured'
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
@@ -124,6 +137,44 @@ function getFileError(file: UploadedDataFile) {
<template>
<section class="source-upload-step" aria-labelledby="source-upload-title">
<div class="source-mode-section">
<div class="section-title-row">
<div>
<h3>数据来源</h3>
<p>在当前步骤选择本地文件上传或从外部系统拉取处理类型保持不变</p>
</div>
</div>
<div class="source-mode-options">
<button
type="button"
class="source-mode-option"
:class="{ 'is-active': sourceMode === 'local' }"
:disabled="sourceModeLocked"
@click="emit('update:sourceMode', 'local')"
>
<i class="fa fa-upload" aria-hidden="true" />
<span><strong>本地上传</strong><small>从当前设备选择文件</small></span>
<i class="fa fa-check-circle selection-mark" aria-hidden="true" />
</button>
<el-tooltip
:content="processType === 'structured' ? '从标准化外部数据源配置拉取' : '当前仅支持把外部数据库拉取为结构化数据'"
>
<button
type="button"
class="source-mode-option"
:class="{ 'is-active': sourceMode === 'external' }"
:disabled="sourceModeLocked || processType !== 'structured'"
@click="emit('update:sourceMode', 'external')"
>
<i class="fa fa-cloud-download" aria-hidden="true" />
<span><strong>外部数据源</strong><small>连接数据库并执行只读拉取</small></span>
<i class="fa fa-check-circle selection-mark" aria-hidden="true" />
</button>
</el-tooltip>
</div>
<small v-if="sourceModeLocked" class="source-mode-lock-hint">已有来源文件时不可切换删除现有文件后可重新选择</small>
</div>
<div v-if="isExternal" class="form-section external-section">
<div class="section-title-row">
<div>
@@ -202,6 +253,45 @@ function getFileError(file: UploadedDataFile) {
@update:model-value="updateExternalField('limit', Number($event) || 0)"
/>
</el-form-item>
<el-form-item label="SSL 模式">
<el-select
:model-value="externalSource.sslMode"
aria-label="SSL 模式"
@update:model-value="updateExternalField('sslMode', $event)"
>
<el-option v-for="item in SSL_MODES" :key="item.value" :label="item.label" :value="item.value" />
</el-select>
</el-form-item>
<el-form-item label="连接超时">
<el-input-number
:model-value="externalSource.connectTimeoutSeconds"
:min="1"
:max="30"
controls-position="right"
aria-label="连接超时秒数"
@update:model-value="updateExternalField('connectTimeoutSeconds', Number($event) || 5)"
/>
<small>单位</small>
</el-form-item>
<el-form-item label="查询超时">
<el-input-number
:model-value="externalSource.statementTimeoutSeconds"
:min="1"
:max="300"
controls-position="right"
aria-label="查询超时秒数"
@update:model-value="updateExternalField('statementTimeoutSeconds', Number($event) || 30)"
/>
<small>单位</small>
</el-form-item>
<el-form-item label="落地文件名">
<el-input
:model-value="externalSource.fileName"
placeholder="external-data.jsonl"
aria-label="外部数据落地文件名"
@update:model-value="updateExternalField('fileName', $event)"
/>
</el-form-item>
<el-form-item label="只读查询语句" class="external-query-field">
<el-input
:model-value="externalSource.query"
@@ -314,7 +404,7 @@ function getFileError(file: UploadedDataFile) {
v-if="uploadedFiles.length === 0"
link
type="primary"
:disabled="previewBuilding"
:disabled="previewBuilding || externalPulling"
@click="emit('use-sample')"
>
使用示例数据
@@ -326,7 +416,7 @@ function getFileError(file: UploadedDataFile) {
drag
multiple
:accept="uploadAccept"
:disabled="previewBuilding"
:disabled="previewBuilding || externalPulling"
:auto-upload="false"
:show-file-list="false"
:on-change="(file: UploadFile) => emit('file-change', file)"
@@ -353,13 +443,13 @@ function getFileError(file: UploadedDataFile) {
<el-upload
multiple
:accept="uploadAccept"
:disabled="previewBuilding"
:disabled="previewBuilding || externalPulling"
:auto-upload="false"
:show-file-list="false"
:on-change="(file: UploadFile) => emit('file-change', file)"
aria-label="继续添加源数据文件"
>
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
<el-button size="small" type="primary" :disabled="previewBuilding || externalPulling">继续上传</el-button>
</el-upload>
</div>
</div>
@@ -430,6 +520,68 @@ function getFileError(file: UploadedDataFile) {
width: 100%;
}
.source-mode-section {
padding-bottom: 22px;
margin-bottom: 22px;
border-bottom: 1px solid #edf0f5;
}
.source-mode-options {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 14px;
margin-top: 14px;
}
.source-mode-option {
position: relative;
display: flex;
align-items: center;
gap: 12px;
min-height: 76px;
padding: 14px 16px;
color: #667085;
text-align: left;
background: #fff;
border: 1px solid #dfe3ea;
border-radius: 8px;
cursor: pointer;
> .fa:first-child {
width: 28px;
color: #5b50f2;
font-size: 20px;
text-align: center;
}
span {
display: flex;
flex: 1;
flex-direction: column;
gap: 5px;
}
strong { color: #344054; font-size: 14px; }
small { color: #8a93a3; font-size: 12px; }
.selection-mark { color: transparent; }
&.is-active {
background: #fafaff;
border-color: #5b50f2;
box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08);
}
&.is-active .selection-mark { color: #5b50f2; }
&:disabled { cursor: not-allowed; opacity: 0.55; }
}
.source-mode-lock-hint {
display: block;
margin-top: 9px;
color: #8a93a3;
font-size: 11px;
}
.form-section {
padding: 0;
@@ -672,6 +824,7 @@ function getFileError(file: UploadedDataFile) {
}
@media (max-width: 900px) {
.source-mode-options,
.external-section .external-grid {
grid-template-columns: minmax(0, 1fr);
}

View File

@@ -188,20 +188,6 @@ defineExpose({ validate })
</span>
<i class="fa fa-check-circle selection-mark" />
</button>
<button
type="button"
class="type-option"
:class="{ 'is-active': processType === 'external' }"
:disabled="processTypeLocked"
@click="emit('update:processType', 'external')"
>
<span class="type-icon"><i class="fa fa-cloud-download" /></span>
<span>
<strong>外来数据源拉取</strong>
<small>适用于数据库API 接口等需远程拉取的外部数据</small>
</span>
<i class="fa fa-check-circle selection-mark" />
</button>
</div>
</el-form-item>
</div>
@@ -274,7 +260,7 @@ defineExpose({ validate })
.type-options {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 16px;
width: 100%;
}

View File

@@ -73,10 +73,21 @@ export const DEFAULT_REASONING_GENERATION_PROMPT = `你是一名具有深厚专
3. 信息不足时如实处理:遇到来源信息缺失或相互矛盾时,需在推理中明确指出,并基于来源做出合理假设,但不得编造与来源无关的内容。
4. 最终答案须能从前面的推理过程中自然得出,语言简洁、结构清晰。`
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
return outputType === 'reasoning'
? DEFAULT_REASONING_GENERATION_PROMPT
: DEFAULT_STANDARD_GENERATION_PROMPT
export const DEFAULT_DPO_GENERATION_PROMPT = `你是一名偏好数据构造专家。请基于下方来源内容生成可用于直接偏好优化DPO的成对问答数据。
来源内容:
{{ content }}
任务要求:
1. 问题应聚焦来源内容的核心信息,并具有实际训练价值。
2. Chosen 必须准确、完整、清晰且严格忠于来源内容。
3. Rejected 应当表面合理但包含可辨认的缺陷,例如遗漏关键条件、事实偏差、逻辑不完整或表达含混;不得用乱码、空话或危险内容凑数。
4. Chosen 与 Rejected 必须明显不同,且都直接回答同一个问题。`
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning' | 'dpo') {
if (outputType === 'reasoning') return DEFAULT_REASONING_GENERATION_PROMPT
if (outputType === 'dpo') return DEFAULT_DPO_GENERATION_PROMPT
return DEFAULT_STANDARD_GENERATION_PROMPT
}
export function isBuiltInGenerationPrompt(value: string) {
@@ -92,6 +103,7 @@ export function isBuiltInGenerationPrompt(value: string) {
PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_3,
DEFAULT_STANDARD_GENERATION_PROMPT,
DEFAULT_REASONING_GENERATION_PROMPT,
DEFAULT_DPO_GENERATION_PROMPT,
].map(normalize).includes(normalizedValue)
}
@@ -171,8 +183,9 @@ function generationOptionsFromConfig(
config: DataProcessConfig,
defaults: GenerationControlOptions,
): GenerationControlOptions {
const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
? 'reasoning'
const configuredOutputType = configValue(config, 'output_type', defaults.outputType)
const outputType = configuredOutputType === 'reasoning' || configuredOutputType === 'dpo'
? configuredOutputType
: 'standard'
const configuredPrompt = String(
configValue(config, 'generation_prompt', defaults.generationPrompt),

View File

@@ -0,0 +1,77 @@
import type { DataProcessExternalSourcePayload } from '@/api/modules/dataProcess'
import type { DataProcessConfig } from '@/types/dataProcess'
import type { ExternalDataSource, SourceMode } from './types'
export function createDefaultExternalSource(): ExternalDataSource {
return {
type: 'postgresql',
url: '',
authMode: 'none',
username: '',
password: '',
limit: 1000,
connectTimeoutSeconds: 5,
statementTimeoutSeconds: 30,
sslMode: 'prefer',
query: '',
fileName: 'external-data.jsonl',
}
}
export function externalSourcePayload(source: ExternalDataSource): DataProcessExternalSourcePayload {
return {
type: source.type,
url: source.url.trim(),
auth_mode: source.authMode,
username: source.username?.trim() || undefined,
password: source.password || undefined,
limit: source.limit,
connect_timeout_seconds: source.connectTimeoutSeconds,
statement_timeout_seconds: source.statementTimeoutSeconds,
ssl_mode: source.sslMode,
query: source.query?.trim() || undefined,
file_name: source.fileName || 'external-data.jsonl',
}
}
export function sourceConfigForBackend(mode: SourceMode, source: ExternalDataSource) {
const payload = externalSourcePayload(source)
const { password: _password, ...safeSource } = payload
return {
source_mode: mode,
external_source: mode === 'external' ? safeSource : undefined,
}
}
export function restoreExternalSourceConfig(config: DataProcessConfig): {
mode: SourceMode
source: ExternalDataSource
} {
const defaults = createDefaultExternalSource()
const value = config.external_source
if (!value || typeof value !== 'object') {
return {
mode: config.source_mode === 'external' ? 'external' : 'local',
source: defaults,
}
}
const source = value as Record<string, unknown>
const sslMode = String(source.ssl_mode || '')
return {
mode: config.source_mode === 'external' ? 'external' : 'local',
source: {
...defaults,
url: String(source.url || ''),
authMode: source.auth_mode === 'basic' ? 'basic' : 'none',
username: String(source.username || ''),
limit: Number(source.limit) || defaults.limit,
connectTimeoutSeconds: Number(source.connect_timeout_seconds) || defaults.connectTimeoutSeconds,
statementTimeoutSeconds: Number(source.statement_timeout_seconds) || defaults.statementTimeoutSeconds,
sslMode: ['disable', 'prefer', 'require', 'verify-ca', 'verify-full'].includes(sslMode)
? sslMode as ExternalDataSource['sslMode']
: defaults.sslMode,
query: String(source.query || ''),
fileName: String(source.file_name || defaults.fileName),
},
}
}

View File

@@ -5,6 +5,7 @@ import type {
} from '@/types/dataProcess'
export type ProcessType = 'structured' | 'unstructured' | 'external'
export type SourceMode = 'local' | 'external'
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
@@ -88,6 +89,9 @@ export interface ExternalDataSource {
username?: string
password?: string
limit: number
connectTimeoutSeconds: number
statementTimeoutSeconds: number
sslMode: 'disable' | 'prefer' | 'require' | 'verify-ca' | 'verify-full'
query?: string
fileName?: string
}
@@ -167,12 +171,18 @@ export interface ResultItem {
instruction: string
input: string
output: string
chosen: string
rejected: string
originalInstruction: string
originalInput: string
originalOutput: string
originalChosen: string
originalRejected: string
savedInstruction: string
savedInput: string
savedOutput: string
savedChosen: string
savedRejected: string
savedStatus: 'valid' | 'modified' | 'invalid'
status: 'valid' | 'modified' | 'invalid'
error?: string

View File

@@ -12,10 +12,12 @@ import {
type DataProcessResult,
} from '@/api/modules/dataProcess'
import type { BulkResultRegenerationState, GenerationState, ResultItem } from './types'
import type { DataProcessOutputType } from '@/types/dataProcess'
interface GenerationBindings {
taskId: Ref<string | null>
dirty: Ref<boolean>
outputType: Ref<DataProcessOutputType>
beforeGenerate?: () => Promise<boolean | void>
}
@@ -32,12 +34,18 @@ function mapResult(item: DataProcessResult): ResultItem {
instruction: item.instruction,
input: item.input || '',
output: item.output,
chosen: item.chosen || item.output || '',
rejected: item.rejected || '',
originalInstruction: item.original_instruction ?? item.instruction,
originalInput: item.original_input ?? item.input ?? '',
originalOutput: item.original_output ?? item.output,
originalChosen: item.original_chosen ?? item.chosen ?? item.output ?? '',
originalRejected: item.original_rejected ?? item.rejected ?? '',
savedInstruction: item.instruction,
savedInput: item.input || '',
savedOutput: item.output,
savedChosen: item.chosen || item.output || '',
savedRejected: item.rejected || '',
savedStatus: item.status,
status: item.status,
error: item.error || undefined,
@@ -246,15 +254,34 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
}
}
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
function updateResultField(
id: string,
field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected',
value: string,
) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item[field] = value
const valid = item.instruction.trim() && item.output.trim()
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
if (field === 'chosen') item.output = value
const isDpo = bindings.outputType.value === 'dpo'
const valid = isDpo
? Boolean(
item.instruction.trim()
&& item.chosen.trim()
&& item.rejected.trim()
&& item.chosen.trim() !== item.rejected.trim()
)
: Boolean(item.instruction.trim() && item.output.trim())
item.error = valid
? undefined
: isDpo
? 'Instruction、Chosen、Rejected 均不能为空,且两个回答不能相同'
: 'Instruction 和 Output 不能为空'
const changed = item.instruction !== item.originalInstruction
|| item.input !== item.originalInput
|| item.output !== item.originalOutput
|| item.chosen !== item.originalChosen
|| item.rejected !== item.originalRejected
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
bindings.dirty.value = true
}
@@ -311,6 +338,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
item.instruction !== item.savedInstruction
|| item.input !== item.savedInput
|| item.output !== item.savedOutput
|| item.chosen !== item.savedChosen
|| item.rejected !== item.savedRejected
))
if (unsaved) {
selectedResultId.value = unsaved.id
@@ -409,12 +438,16 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
item.instruction !== item.savedInstruction
|| item.input !== item.savedInput
|| item.output !== item.savedOutput
|| item.chosen !== item.savedChosen
|| item.rejected !== item.savedRejected
))
for (const item of changed) {
const saved = await updateDataProcessResult(taskId, item.id, {
instruction: item.instruction,
input: item.input,
output: item.output,
chosen: item.chosen,
rejected: item.rejected,
expected_updated_at: item.updatedAt,
})
const index = results.value.findIndex((entry) => entry.id === item.id)
@@ -425,7 +458,13 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
function validateResults() {
let firstInvalidId: string | null = null
for (const item of results.value) {
if (!item.instruction.trim() || !item.output.trim() || item.status === 'invalid') {
const requiredFieldsInvalid = bindings.outputType.value === 'dpo'
? !item.instruction.trim()
|| !item.chosen.trim()
|| !item.rejected.trim()
|| item.chosen.trim() === item.rejected.trim()
: !item.instruction.trim() || !item.output.trim()
if (requiredFieldsInvalid || item.status === 'invalid') {
item.error ||= '结果未通过后端质量校验,请修改后重新保存'
item.status = 'invalid'
firstInvalidId ??= item.id