Merge branch 'ft_wyt' of http://www.caoxiaozhu.com:13001/YG-Soft/YG_FT into ft_wyt
This commit is contained in:
@@ -101,7 +101,9 @@ assert.match(detailSource, /!\/\(\?:password\|secret\|token\|api_key\)\/i\.test\
|
||||
assert.match(detailSource, /key !== 'generation_model_snapshot'/, '处理配置仍直接展示内部模型快照')
|
||||
assert.match(detailSource, /preprocessOptionLabelMap/, '处理配置没有把预处理内部枚举转换为中文')
|
||||
assert.match(detailSource, /output_type:\s*'输出类型'/, '处理配置没有显示输出类型名称')
|
||||
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : '标准回答'/, '处理配置没有转换输出类型枚举')
|
||||
assert.match(detailSource, /value === 'reasoning' \? '思维链回答' : value === 'dpo' \? 'DPO 偏好对' : '标准回答'/, '处理配置没有转换输出类型枚举')
|
||||
assert.match(detailSource, /prop="chosen"[\s\S]*?prop="rejected"/, 'DPO 结果没有展示 Chosen 与 Rejected')
|
||||
assert.match(detailSource, /publishForm\.format = isDpoOutput\.value \? 'dpo'/, 'DPO 发布没有锁定原生格式')
|
||||
assert.match(detailSource, /reasoning_detail:\s*'推理详细程度'/, '处理配置没有显示推理详细程度名称')
|
||||
assert.match(detailSource, /value === 'detailed' \? '详细推理' : '普通推理'/, '处理配置没有转换推理详细程度枚举')
|
||||
assert.match(detailSource, /key !== 'reasoning_detail' \|\| config\.output_type === 'reasoning'/, '标准回答任务不应展示无关的推理详细程度')
|
||||
|
||||
@@ -122,13 +122,13 @@ assert.match(regenerationSource, /route\.name === 'data-process-regenerate'[\s\S
|
||||
assert.match(viewSource, /const currentStep = ref\(0\)/, '重新生成必须从向导第一步开始')
|
||||
|
||||
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
|
||||
for (const title of ['创建任务', '大模型选择', '上传文件', '数据预览', '开始生成', '结果编辑与保存']) {
|
||||
for (const title of ['创建任务', '大模型选择', '数据来源', '数据预览', '开始生成', '结果编辑与保存']) {
|
||||
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
|
||||
}
|
||||
assert.match(
|
||||
viewSource,
|
||||
/\{ id: 'create',[\s\S]*?\{ id: 'model',[\s\S]*?\{ id: 'upload',[\s\S]*?\{ id: 'preview',[\s\S]*?\{ id: 'generate',[\s\S]*?\{ id: 'results'/,
|
||||
'六步向导顺序必须为创建任务、大模型选择、上传文件、数据预览、开始生成、结果编辑与保存',
|
||||
'六步向导顺序必须为创建任务、大模型选择、数据来源、数据预览、开始生成、结果编辑与保存',
|
||||
)
|
||||
assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减')
|
||||
assert.match(
|
||||
@@ -380,7 +380,7 @@ for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploa
|
||||
assert.match(viewSource, /<ModelSelectionStep\s+[\s\S]*?v-else-if="currentStepId === 'model'"/, '第二步没有挂载独立大模型选择组件')
|
||||
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第三步没有挂载独立上传组件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续:选择大模型'/, '第一步主按钮没有指向大模型选择')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续:上传文件'/, '第二步主按钮没有指向上传文件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续:选择数据来源'/, '第二步主按钮没有指向数据来源选择')
|
||||
assert.match(
|
||||
viewSource,
|
||||
/if \(currentStepId\.value === 'upload'\) \{[\s\S]*?sourceUploading\.value[\s\S]*?'正在上传'[\s\S]*?previewBuilding\.value \? '正在切分' : '继续:数据预览'/,
|
||||
@@ -1032,7 +1032,7 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\
|
||||
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
|
||||
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
|
||||
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
|
||||
assert.match(stateSource, /const outputType = configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
|
||||
assert.match(stateSource, /configuredOutputType === 'reasoning' \|\| configuredOutputType === 'dpo'[\s\S]*?\? configuredOutputType[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
|
||||
assert.match(stateSource, /reasoningDetail:\s*configValue\(config, 'reasoning_detail', defaults\.reasoningDetail\) === 'detailed'[\s\S]*?\? 'detailed'[\s\S]*?: 'normal'/, '推理详细程度没有从任务配置安全回填')
|
||||
assert.match(stateSource, /isBuiltInGenerationPrompt\(configuredPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)/, '旧版内置提示语没有按输出类型迁移')
|
||||
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
|
||||
@@ -1412,7 +1412,7 @@ assert.match(
|
||||
)
|
||||
assert.match(
|
||||
sourceUploadSource,
|
||||
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
|
||||
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
|
||||
'无文件时未保留原有大拖拽上传区或上传配置',
|
||||
)
|
||||
assert.match(
|
||||
@@ -1432,7 +1432,7 @@ assert.ok(sourceUploadSource.includes('旧版 DOC/PPT 请先转换'), '非结构
|
||||
assert.ok(sourceUploadSource.includes('旧版 XLS 请先转换'), '结构化格式提示没有说明旧版 XLS 需转换')
|
||||
assert.match(
|
||||
sourceUploadSource,
|
||||
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
|
||||
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding \|\| externalPulling">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
|
||||
'有文件时缺少标题右侧的继续上传触发器或上传配置',
|
||||
)
|
||||
assert.match(
|
||||
@@ -1476,4 +1476,13 @@ assert.match(
|
||||
)
|
||||
assert.match(previewSource, /height:\s*clamp\(560px,\s*calc\(100vh - 370px\),\s*720px\)/, '对照预览高度不足以展示切片正文')
|
||||
|
||||
assert.doesNotMatch(taskSetupSource, /外来数据源拉取/, '外部数据源仍错误地放在第一步处理类型中')
|
||||
assert.match(sourceUploadSource, /<h3>数据来源<\/h3>[\s\S]*?本地上传[\s\S]*?外部数据源/, '第三步缺少本地与外部数据来源选择')
|
||||
for (const field of ['地址 / URL', '鉴权方式', 'SSL 模式', '连接超时', '查询超时', '只读查询语句', '拉取条数', '落地文件名']) {
|
||||
assert.ok(sourceUploadSource.includes(field), `外部数据源标准配置缺少:${field}`)
|
||||
}
|
||||
assert.match(generationControlSource, /label="DPO 偏好对" value="dpo"/, '输出类型缺少 DPO 偏好对')
|
||||
assert.match(resultEditorSource, /Chosen[\s\S]*?Rejected/, '结果编辑器缺少 DPO 成对字段')
|
||||
assert.match(viewSource, /sourceConfigForBackend\(sourceMode\.value, externalSource\)/, '任务配置没有保存第三步数据来源模式')
|
||||
|
||||
console.log('数据处理六步向导回归检查通过')
|
||||
|
||||
@@ -4,7 +4,7 @@ export type DataProcessStatus = 'pending' | 'running' | 'completed' | 'failed' |
|
||||
export type DataProcessType = 'structured' | 'unstructured' | 'external'
|
||||
export type DataProcessResultStatus = 'valid' | 'modified' | 'invalid'
|
||||
export type DataProcessSplit = 'train' | 'validation' | 'test'
|
||||
export type DataProcessOutputType = 'standard' | 'reasoning'
|
||||
export type DataProcessOutputType = 'standard' | 'reasoning' | 'dpo'
|
||||
export type DataProcessReasoningDetail = 'normal' | 'detailed'
|
||||
export type DataProcessWorkflowStep = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||
export type DataProcessPreviewStatus = 'idle' | 'queued' | 'running' | 'completed' | 'failed' | 'cancelled'
|
||||
@@ -221,6 +221,9 @@ export interface DataProcessExternalSourcePayload {
|
||||
username?: string
|
||||
password?: string
|
||||
limit: number
|
||||
connect_timeout_seconds: number
|
||||
statement_timeout_seconds: number
|
||||
ssl_mode: 'disable' | 'prefer' | 'require' | 'verify-ca' | 'verify-full'
|
||||
query?: string
|
||||
file_name?: string
|
||||
}
|
||||
@@ -342,9 +345,13 @@ export interface DataProcessResult {
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
chosen?: string
|
||||
rejected?: string
|
||||
original_instruction?: string | null
|
||||
original_input?: string | null
|
||||
original_output?: string | null
|
||||
original_chosen?: string | null
|
||||
original_rejected?: string | null
|
||||
status: DataProcessResultStatus
|
||||
error?: string | null
|
||||
split?: DataProcessSplit | null
|
||||
@@ -356,6 +363,8 @@ export interface DataProcessResultUpdatePayload {
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
chosen?: string
|
||||
rejected?: string
|
||||
expected_updated_at?: string
|
||||
}
|
||||
|
||||
@@ -410,7 +419,7 @@ export interface DataProcessPublishPayload {
|
||||
dataset_type: 'train' | 'test' | 'eval' | 'val' | 'other'
|
||||
storage_type: 'local'
|
||||
split: DataProcessDatasetSplit
|
||||
format: 'alpaca_jsonl' | 'jsonl'
|
||||
format: 'alpaca_jsonl' | 'jsonl' | 'dpo'
|
||||
}
|
||||
|
||||
export interface DataProcessPublishResult {
|
||||
|
||||
@@ -20,6 +20,7 @@ import {
|
||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
||||
import { createDefaultExternalSource, externalSourcePayload, restoreExternalSourceConfig, sourceConfigForBackend } from './create/externalSourceConfig'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
@@ -39,26 +40,20 @@ import {
|
||||
updateDataProcessPreview,
|
||||
updateDataProcessTask,
|
||||
updateDataProcessWorkflowStep,
|
||||
type DataProcessExternalSourcePayload,
|
||||
type DataProcessPreviewItem,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type {
|
||||
DataProcessConfig,
|
||||
DataProcessPreviewProgress,
|
||||
DataProcessTask,
|
||||
DataProcessWorkflowStep,
|
||||
} from '@/types/dataProcess'
|
||||
import type { DataProcessConfig, DataProcessPreviewProgress, DataProcessTask, DataProcessWorkflowStep } from '@/types/dataProcess'
|
||||
import type {
|
||||
ExternalDataSource,
|
||||
GenerationControlOptions,
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
SourceMode,
|
||||
StepId,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
UploadedDataFile,
|
||||
} from './create/types'
|
||||
|
||||
const router = useRouter()
|
||||
const modelsStore = useModelsStore()
|
||||
const { list: modelList, loaded: modelsLoaded } = storeToRefs(modelsStore)
|
||||
@@ -71,7 +66,7 @@ const PREVIEW_MODEL_VERSION = 'backend-pipeline-v4'
|
||||
const WIZARD_STEPS = [
|
||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
||||
{ id: 'model', title: '大模型选择', desc: '选择生成模型并设置输出要求' },
|
||||
{ id: 'upload', title: '上传文件', desc: '上传或接入待处理的源数据' },
|
||||
{ id: 'upload', title: '数据来源', desc: '选择本地上传或外部数据源拉取' },
|
||||
{ id: 'preview', title: '数据预览', desc: '核对源文件与预览内容' },
|
||||
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
|
||||
{ id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
|
||||
@@ -81,11 +76,13 @@ const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id
|
||||
const task = reactive({ name: '', description: '' })
|
||||
const taskId = ref<string | null>(null)
|
||||
const processType = ref<ProcessType>('structured')
|
||||
const sourceMode = ref<SourceMode>('local')
|
||||
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
|
||||
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
|
||||
const modelSelectionOptions = computed<GenerationControlOptions>(() => (
|
||||
processType.value === 'unstructured' ? unstructuredOptions.value : structuredOptions.value
|
||||
))
|
||||
const activeOutputType = computed(() => modelSelectionOptions.value.outputType)
|
||||
const uploadedFiles = ref<UploadedDataFile[]>([])
|
||||
const previewBuilding = ref(false)
|
||||
const {
|
||||
@@ -93,16 +90,7 @@ const {
|
||||
startPreviewBuild,
|
||||
stopPreviewPolling,
|
||||
} = useDataProcessPreviewBuild()
|
||||
const externalSource = reactive<ExternalDataSource>({
|
||||
type: 'postgresql',
|
||||
url: '',
|
||||
authMode: 'none',
|
||||
username: '',
|
||||
password: '',
|
||||
limit: 1000,
|
||||
query: '',
|
||||
fileName: 'external-data.jsonl',
|
||||
})
|
||||
const externalSource = reactive<ExternalDataSource>(createDefaultExternalSource())
|
||||
const externalPulling = ref(false)
|
||||
const externalConnected = ref(false)
|
||||
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
|
||||
@@ -135,6 +123,7 @@ const {
|
||||
} = useDataProcessGeneration({
|
||||
taskId,
|
||||
dirty,
|
||||
outputType: activeOutputType,
|
||||
beforeGenerate: beforeStartGeneration,
|
||||
})
|
||||
const { enqueueSourceUpload, sourceUploading } = useDataProcessSourceUpload({
|
||||
@@ -174,7 +163,7 @@ const previewFiles = computed(() => uploadedFiles.value.map((file) => {
|
||||
}))
|
||||
const primaryActionLabel = computed(() => {
|
||||
if (currentStepId.value === 'create') return '继续:选择大模型'
|
||||
if (currentStepId.value === 'model') return '继续:上传文件'
|
||||
if (currentStepId.value === 'model') return '继续:选择数据来源'
|
||||
if (currentStepId.value === 'upload') {
|
||||
if (sourceUploading.value) return '正在上传'
|
||||
return previewBuilding.value ? '正在切分' : '继续:数据预览'
|
||||
@@ -198,7 +187,6 @@ function goToStep(stepId: StepId) {
|
||||
const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
|
||||
if (nextStepIndex >= 0) currentStep.value = nextStepIndex
|
||||
}
|
||||
|
||||
function updateModelSelectionOptions(value: GenerationControlOptions) {
|
||||
if (processType.value === 'unstructured') {
|
||||
unstructuredOptions.value = { ...unstructuredOptions.value, ...value }
|
||||
@@ -206,12 +194,12 @@ function updateModelSelectionOptions(value: GenerationControlOptions) {
|
||||
}
|
||||
structuredOptions.value = { ...structuredOptions.value, ...value }
|
||||
}
|
||||
|
||||
function toBackendConfig(): DataProcessConfig {
|
||||
const options = processType.value === 'unstructured'
|
||||
? unstructuredOptions.value
|
||||
: structuredOptions.value
|
||||
const common = {
|
||||
...sourceConfigForBackend(sourceMode.value, externalSource),
|
||||
preprocess_options: [...options.preprocessOptions],
|
||||
semantic_enrichment: options.semanticEnrichment,
|
||||
dataset_split: { ...options.datasetSplit },
|
||||
@@ -247,7 +235,6 @@ function toBackendConfig(): DataProcessConfig {
|
||||
qa_pairs_per_row: structuredOptions.value.qaPairsPerRow,
|
||||
}
|
||||
}
|
||||
|
||||
function taskPayload() {
|
||||
return {
|
||||
name: task.name.trim(),
|
||||
@@ -256,12 +243,10 @@ function taskPayload() {
|
||||
config: toBackendConfig(),
|
||||
}
|
||||
}
|
||||
|
||||
async function persistWorkflowStep(step: StepId) {
|
||||
if (!taskId.value) return
|
||||
await updateDataProcessWorkflowStep(taskId.value, step as DataProcessWorkflowStep)
|
||||
}
|
||||
|
||||
async function saveTaskConfiguration() {
|
||||
if (isRegeneration.value) {
|
||||
const regenerated = await prepareRegeneration(taskPayload())
|
||||
@@ -277,19 +262,6 @@ async function saveTaskConfiguration() {
|
||||
return saved
|
||||
}
|
||||
|
||||
function externalPayload(): DataProcessExternalSourcePayload {
|
||||
return {
|
||||
type: 'postgresql',
|
||||
url: externalSource.url.trim(),
|
||||
auth_mode: externalSource.authMode,
|
||||
username: externalSource.username || undefined,
|
||||
password: externalSource.password || undefined,
|
||||
limit: externalSource.limit,
|
||||
query: externalSource.query?.trim() || undefined,
|
||||
file_name: externalSource.fileName || 'external-data.jsonl',
|
||||
}
|
||||
}
|
||||
|
||||
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
const sourceLocator = item.quality_score?.source_locator
|
||||
return {
|
||||
@@ -378,7 +350,7 @@ async function beforeStartGeneration() {
|
||||
)
|
||||
}
|
||||
watch(
|
||||
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
[() => task.name, () => task.description, processType, sourceMode, structuredOptions, unstructuredOptions, externalSource],
|
||||
() => {
|
||||
if (!hydrating.value) dirty.value = true
|
||||
},
|
||||
@@ -396,6 +368,13 @@ watch(processType, (nextType, previousType) => {
|
||||
ElMessage.info('处理类型已变更,请重新上传或拉取匹配的源数据')
|
||||
})
|
||||
|
||||
watch(processType, (nextType) => {
|
||||
if (nextType === 'unstructured' && sourceMode.value === 'external') {
|
||||
sourceMode.value = 'local'
|
||||
externalConnected.value = false
|
||||
}
|
||||
})
|
||||
|
||||
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||
if (hydrating.value) return
|
||||
if (currentSignature === previousSignature) return
|
||||
@@ -453,6 +432,16 @@ function updateExternalSource(value: ExternalDataSource) {
|
||||
externalConnected.value = false
|
||||
}
|
||||
|
||||
async function updateSourceMode(value: SourceMode) {
|
||||
const previous = sourceMode.value
|
||||
sourceMode.value = value
|
||||
externalConnected.value = false
|
||||
dirty.value = true
|
||||
externalPulling.value = true
|
||||
try { await saveTaskConfiguration() } catch { sourceMode.value = previous }
|
||||
finally { externalPulling.value = false }
|
||||
}
|
||||
|
||||
async function handleTestConnection() {
|
||||
if (!taskId.value) {
|
||||
ElMessage.error('任务尚未创建,请返回模型选择步骤后重试')
|
||||
@@ -464,7 +453,8 @@ async function handleTestConnection() {
|
||||
}
|
||||
externalPulling.value = true
|
||||
try {
|
||||
const result = await testDataProcessExternalSource(taskId.value, externalPayload())
|
||||
await saveTaskConfiguration()
|
||||
const result = await testDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
|
||||
externalConnected.value = result.connected
|
||||
if (result.connected) ElMessage.success(result.message || '数据源连接测试成功')
|
||||
else ElMessage.warning(result.message || '数据源连接失败')
|
||||
@@ -490,7 +480,8 @@ async function handlePullData() {
|
||||
}
|
||||
externalPulling.value = true
|
||||
try {
|
||||
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
|
||||
await saveTaskConfiguration()
|
||||
const response = await pullDataProcessExternalSource(taskId.value, externalSourcePayload(externalSource))
|
||||
const newFiles: UploadedDataFile[] = []
|
||||
for (const file of response.files) {
|
||||
const content = await loadCanonicalSourceContent(taskId.value, file.id)
|
||||
@@ -620,6 +611,12 @@ function applyPreviewProgress(progress: DataProcessPreviewProgress) {
|
||||
|
||||
async function completePreviewWorkspace() {
|
||||
previewItems.value = await loadAllPreviewItems()
|
||||
for (const file of uploadedFiles.value) {
|
||||
if (!file.content && file.sourceFileId && file.fileFormat?.replace('.', '') === 'pdf') {
|
||||
file.content = await loadCanonicalSourceContent(taskId.value as string, file.sourceFileId)
|
||||
.catch(() => file.content)
|
||||
}
|
||||
}
|
||||
const configSignature = buildPreviewConfigSignature()
|
||||
const previewCounts = new Map<string, number>()
|
||||
for (const item of previewItems.value) {
|
||||
@@ -678,7 +675,7 @@ async function nextFromUpload() {
|
||||
return
|
||||
}
|
||||
if (uploadedFiles.value.length === 0) {
|
||||
ElMessage.warning(processType.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
|
||||
ElMessage.warning(sourceMode.value === 'external' ? '请先拉取至少一个数据源' : '请上传至少一个源数据文件')
|
||||
return
|
||||
}
|
||||
const failedUploads = uploadedFiles.value.filter((file) => file.status === 'failed')
|
||||
@@ -992,6 +989,9 @@ async function initializeExistingWorkflow() {
|
||||
const sourceTask = await loadRegenerationSource()
|
||||
if (!sourceTask) return
|
||||
taskId.value = String(sourceTask.id)
|
||||
const restoredSource = restoreExternalSourceConfig(sourceTask.config || {})
|
||||
sourceMode.value = restoredSource.mode
|
||||
Object.assign(externalSource, restoredSource.source)
|
||||
if (!isWorkflowResume.value) return
|
||||
if (sourceTask.status === 'completed' && sourceTask.results_confirmed !== false) {
|
||||
allowLeave = true
|
||||
@@ -1102,6 +1102,7 @@ onMounted(() => {
|
||||
<SourceUploadStep
|
||||
v-else-if="currentStepId === 'upload'"
|
||||
:process-type="processType"
|
||||
:source-mode="sourceMode"
|
||||
:uploaded-files="uploadedFiles"
|
||||
:external-source="externalSource"
|
||||
:external-pulling="externalPulling"
|
||||
@@ -1109,6 +1110,7 @@ onMounted(() => {
|
||||
:preview-building="previewBuilding"
|
||||
:source-uploading="sourceUploading"
|
||||
@update:external-source="updateExternalSource"
|
||||
@update:source-mode="updateSourceMode"
|
||||
@file-change="handleFileChange"
|
||||
@remove-file="handleRemoveFile"
|
||||
@use-sample="useSampleFile"
|
||||
@@ -1154,6 +1156,7 @@ onMounted(() => {
|
||||
:preview-items="previewItems"
|
||||
:regenerating-result-id="regeneratingResultId"
|
||||
:bulk-regeneration="bulkRegeneration"
|
||||
:output-type="activeOutputType"
|
||||
@update:field="updateResultField"
|
||||
@regenerate:all="regenerateAllResults"
|
||||
@regenerate:item="regenerateResult"
|
||||
@@ -1179,8 +1182,8 @@ onMounted(() => {
|
||||
<el-button
|
||||
class="wizard-primary-action"
|
||||
type="primary"
|
||||
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
:loading="externalPulling || modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="externalPulling || hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
@click="handlePrimaryAction"
|
||||
>
|
||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||
|
||||
@@ -53,7 +53,7 @@ const resultCellTooltipOptions = {
|
||||
} as const
|
||||
let resultFilterTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const editForm = reactive({ instruction: '', input: '', output: '' })
|
||||
const editForm = reactive({ instruction: '', input: '', output: '', chosen: '', rejected: '' })
|
||||
const publishForm = reactive<DataProcessPublishPayload>({
|
||||
dataset_name: '',
|
||||
dataset_type: 'train',
|
||||
@@ -69,6 +69,8 @@ const processTypeMap: Record<DataProcessType, string> = {
|
||||
}
|
||||
|
||||
const configLabelMap: Record<string, string> = {
|
||||
source_mode: '数据来源',
|
||||
external_source: '外部数据源配置',
|
||||
preprocess_options: '预处理规则',
|
||||
dataset_split: '数据集划分',
|
||||
generation_model_id: '数据生成模型',
|
||||
@@ -212,6 +214,7 @@ const outputDatasetName = computed(() => (
|
||||
))
|
||||
const outputDatasetId = computed(() => detail.value?.output_dataset_id || null)
|
||||
const hasPublishedOutputs = computed(() => outputDatasets.value.length > 0)
|
||||
const isDpoOutput = computed(() => detail.value?.config?.output_type === 'dpo')
|
||||
const hasCurrentPublishedDataset = computed(() => (
|
||||
Boolean(outputDatasetId.value)
|
||||
&& outputDatasets.value.some((dataset) => String(dataset.id) === String(outputDatasetId.value))
|
||||
@@ -294,6 +297,18 @@ const configRows = computed(() => {
|
||||
})
|
||||
|
||||
function formatConfigValue(key: string, value: unknown) {
|
||||
if (key === 'source_mode') return value === 'external' ? '外部数据源' : '本地上传'
|
||||
if (key === 'external_source' && value && typeof value === 'object') {
|
||||
const source = value as Record<string, unknown>
|
||||
return [
|
||||
source.type,
|
||||
source.url,
|
||||
source.ssl_mode ? `SSL ${source.ssl_mode}` : '',
|
||||
source.limit ? `上限 ${source.limit} 条` : '',
|
||||
source.connect_timeout_seconds ? `连接 ${source.connect_timeout_seconds}s` : '',
|
||||
source.statement_timeout_seconds ? `查询 ${source.statement_timeout_seconds}s` : '',
|
||||
].filter(Boolean).join(' · ')
|
||||
}
|
||||
if (key === 'generation_model_id') {
|
||||
const snapshot = detail.value?.config?.generation_model_snapshot
|
||||
if (snapshot && typeof snapshot === 'object' && !Array.isArray(snapshot)) {
|
||||
@@ -305,7 +320,7 @@ function formatConfigValue(key: string, value: unknown) {
|
||||
return chunkMethodLabelMap[value] || value
|
||||
}
|
||||
if (key === 'output_type') {
|
||||
return value === 'reasoning' ? '思维链回答' : '标准回答'
|
||||
return value === 'reasoning' ? '思维链回答' : value === 'dpo' ? 'DPO 偏好对' : '标准回答'
|
||||
}
|
||||
if (key === 'reasoning_detail') {
|
||||
return value === 'detailed' ? '详细推理' : '普通推理'
|
||||
@@ -461,14 +476,26 @@ function openResultEditor(result: DataProcessResult) {
|
||||
instruction: result.instruction,
|
||||
input: result.input,
|
||||
output: result.output,
|
||||
chosen: result.chosen || result.output || '',
|
||||
rejected: result.rejected || '',
|
||||
})
|
||||
editDialogVisible.value = true
|
||||
}
|
||||
|
||||
async function saveResult() {
|
||||
if (!editingResult.value) return
|
||||
if (!editForm.instruction.trim() || !editForm.output.trim()) {
|
||||
ElMessage.warning('Instruction 和 Output 不能为空')
|
||||
const invalid = isDpoOutput.value
|
||||
? !editForm.instruction.trim()
|
||||
|| !editForm.chosen.trim()
|
||||
|| !editForm.rejected.trim()
|
||||
|| editForm.chosen.trim() === editForm.rejected.trim()
|
||||
: !editForm.instruction.trim() || !editForm.output.trim()
|
||||
if (invalid) {
|
||||
ElMessage.warning(
|
||||
isDpoOutput.value
|
||||
? 'Instruction、Chosen、Rejected 均不能为空,且两个回答不能相同'
|
||||
: 'Instruction 和 Output 不能为空',
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -478,6 +505,8 @@ async function saveResult() {
|
||||
instruction: editForm.instruction,
|
||||
input: editForm.input,
|
||||
output: editForm.output,
|
||||
chosen: editForm.chosen,
|
||||
rejected: editForm.rejected,
|
||||
expected_updated_at: editingResult.value.updated_at,
|
||||
})
|
||||
replaceResult(updated)
|
||||
@@ -530,6 +559,7 @@ function openPublishDialog() {
|
||||
if (!detail.value) return
|
||||
publishForm.dataset_name = `${detail.value.name}-数据集`
|
||||
publishForm.split = configuredSplit()
|
||||
publishForm.format = isDpoOutput.value ? 'dpo' : 'alpaca_jsonl'
|
||||
publishDialogVisible.value = true
|
||||
}
|
||||
|
||||
@@ -808,7 +838,11 @@ onBeforeUnmount(() => {
|
||||
<el-table-column type="index" label="#" width="56" align="center" />
|
||||
<el-table-column label="指令" min-width="190" show-overflow-tooltip prop="instruction" />
|
||||
<el-table-column label="输入" min-width="160" show-overflow-tooltip prop="input" />
|
||||
<el-table-column label="输出" min-width="230" show-overflow-tooltip prop="output" />
|
||||
<el-table-column v-if="!isDpoOutput" label="输出" min-width="230" show-overflow-tooltip prop="output" />
|
||||
<template v-else>
|
||||
<el-table-column label="Chosen" min-width="210" show-overflow-tooltip prop="chosen" />
|
||||
<el-table-column label="Rejected" min-width="210" show-overflow-tooltip prop="rejected" />
|
||||
</template>
|
||||
<el-table-column label="质量分" width="88" align="center">
|
||||
<template #default="{ row }">
|
||||
<el-tooltip :content="qualityFlagsLabel((row as DataProcessResult).quality_score)">
|
||||
@@ -862,9 +896,17 @@ onBeforeUnmount(() => {
|
||||
<el-form-item label="Input">
|
||||
<el-input v-model="editForm.input" type="textarea" :rows="3" maxlength="10000" show-word-limit />
|
||||
</el-form-item>
|
||||
<el-form-item label="Output" required>
|
||||
<el-form-item v-if="!isDpoOutput" label="Output" required>
|
||||
<el-input v-model="editForm.output" type="textarea" :rows="6" maxlength="20000" show-word-limit />
|
||||
</el-form-item>
|
||||
<template v-else>
|
||||
<el-form-item label="Chosen(优选回答)" required>
|
||||
<el-input v-model="editForm.chosen" type="textarea" :rows="6" maxlength="20000" show-word-limit />
|
||||
</el-form-item>
|
||||
<el-form-item label="Rejected(拒选回答)" required>
|
||||
<el-input v-model="editForm.rejected" type="textarea" :rows="6" maxlength="20000" show-word-limit />
|
||||
</el-form-item>
|
||||
</template>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
<el-button @click="editDialogVisible = false">取消</el-button>
|
||||
@@ -884,7 +926,8 @@ onBeforeUnmount(() => {
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="输出格式">
|
||||
<el-select v-model="publishForm.format">
|
||||
<el-select v-model="publishForm.format" :disabled="isDpoOutput">
|
||||
<el-option v-if="isDpoOutput" label="DPO JSONL" value="dpo" />
|
||||
<el-option label="Alpaca JSONL" value="alpaca_jsonl" />
|
||||
<el-option label="JSONL" value="jsonl" />
|
||||
</el-select>
|
||||
|
||||
@@ -76,7 +76,7 @@ function updateQualityRules(value: Array<string | number>) {
|
||||
}
|
||||
|
||||
function updateOutputType(value: string | number | boolean | undefined) {
|
||||
const outputType = value === 'reasoning' ? 'reasoning' : 'standard'
|
||||
const outputType = value === 'reasoning' || value === 'dpo' ? value : 'standard'
|
||||
emit('update:options', {
|
||||
...props.options,
|
||||
outputType,
|
||||
@@ -153,6 +153,9 @@ function modelMeta(model: ModelItem) {
|
||||
<small v-if="options.outputType === 'reasoning'">
|
||||
当前使用思维链专用提示语;系统还会按所选详细程度约束推理结构
|
||||
</small>
|
||||
<small v-else-if="options.outputType === 'dpo'">
|
||||
当前使用 DPO 专用提示语,模型会同时生成 Chosen 与 Rejected 偏好回答
|
||||
</small>
|
||||
<small v-else>当前使用标准回答提示语,只要求问题和最终答案</small>
|
||||
</div>
|
||||
<div class="prompt-input-wrapper">
|
||||
@@ -237,7 +240,7 @@ function modelMeta(model: ModelItem) {
|
||||
<div class="output-type-row">
|
||||
<div class="field-copy">
|
||||
<strong>输出类型</strong>
|
||||
<small>控制答案是否包含可用于推理模型训练的思维链内容</small>
|
||||
<small>选择标准监督、思维链或 DPO 成对偏好数据</small>
|
||||
</div>
|
||||
<el-select
|
||||
class="output-type-select"
|
||||
@@ -247,6 +250,7 @@ function modelMeta(model: ModelItem) {
|
||||
>
|
||||
<el-option label="标准回答" value="standard" />
|
||||
<el-option label="思维链回答" value="reasoning" />
|
||||
<el-option label="DPO 偏好对" value="dpo" />
|
||||
</el-select>
|
||||
</div>
|
||||
<div v-if="options.outputType === 'reasoning'" class="output-type-row">
|
||||
@@ -272,6 +276,9 @@ function modelMeta(model: ModelItem) {
|
||||
<template v-else>保留关键依据与必要步骤,</template>
|
||||
最终按 <code><think>推理过程</think></code> 加最终答案保存。
|
||||
</template>
|
||||
<template v-else-if="options.outputType === 'dpo'">
|
||||
保存同一问题的 <code>chosen</code> 优选回答和 <code>rejected</code> 拒选回答,用于直接偏好优化训练。
|
||||
</template>
|
||||
<template v-else>仅保存最终答案,不包含推理过程。</template>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref } from 'vue'
|
||||
import type { BulkResultRegenerationState, PreviewItem, ResultItem } from './types'
|
||||
import type { DataProcessOutputType } from '@/types/dataProcess'
|
||||
|
||||
const props = defineProps<{
|
||||
items: ResultItem[]
|
||||
@@ -8,11 +9,12 @@ const props = defineProps<{
|
||||
selectedId: string | null
|
||||
regeneratingResultId: string | null
|
||||
bulkRegeneration: BulkResultRegenerationState
|
||||
outputType: DataProcessOutputType
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:selectedId': [value: string]
|
||||
'update:field': [id: string, field: 'instruction' | 'input' | 'output', value: string]
|
||||
'update:field': [id: string, field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected', value: string]
|
||||
'regenerate:item': [id: string]
|
||||
'regenerate:all': []
|
||||
}>()
|
||||
@@ -59,6 +61,11 @@ const selectedSourceMeta = computed(() => {
|
||||
const source = selectedSource.value
|
||||
if (!source) return ''
|
||||
const parts: string[] = []
|
||||
if (source.sourcePages?.length) {
|
||||
const first = source.sourcePages[0]
|
||||
const last = source.sourcePages[source.sourcePages.length - 1]
|
||||
parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`)
|
||||
}
|
||||
if (source.sourceStartLine != null) {
|
||||
parts.push(
|
||||
source.sourceEndLine != null && source.sourceEndLine !== source.sourceStartLine
|
||||
@@ -75,6 +82,8 @@ const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const matchesSearch = !keyword
|
||||
|| item.instruction.toLowerCase().includes(keyword)
|
||||
|| item.output.toLowerCase().includes(keyword)
|
||||
|| item.chosen.toLowerCase().includes(keyword)
|
||||
|| item.rejected.toLowerCase().includes(keyword)
|
||||
|| String(index + 1).includes(keyword)
|
||||
return matchesSearch && (!invalidOnly.value || item.status === 'invalid')
|
||||
}))
|
||||
@@ -135,7 +144,7 @@ function selectRelative(offset: number) {
|
||||
<span class="result-index">#{{ String(items.findIndex((entry) => entry.id === item.id) + 1).padStart(3, '0') }}</span>
|
||||
<span class="result-copy">
|
||||
<strong>{{ item.instruction || '未填写指令' }}</strong>
|
||||
<small>{{ item.output || '未填写输出' }}</small>
|
||||
<small>{{ outputType === 'dpo' ? (item.chosen || '未填写 Chosen') : (item.output || '未填写输出') }}</small>
|
||||
</span>
|
||||
<i v-if="itemRegenerating(item.id)" class="css-spinner" />
|
||||
<i
|
||||
@@ -207,7 +216,7 @@ function selectRelative(offset: number) {
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'input', $event)"
|
||||
/>
|
||||
</div>
|
||||
<div class="field-editor">
|
||||
<div v-if="outputType !== 'dpo'" class="field-editor">
|
||||
<label>Output <em>必填</em></label>
|
||||
<el-input
|
||||
:model-value="selectedItem.output"
|
||||
@@ -217,6 +226,28 @@ function selectRelative(offset: number) {
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'output', $event)"
|
||||
/>
|
||||
</div>
|
||||
<template v-else>
|
||||
<div class="field-editor dpo-field is-chosen">
|
||||
<label>Chosen <em>优选回答,必填</em></label>
|
||||
<el-input
|
||||
:model-value="selectedItem.chosen"
|
||||
:disabled="selectedItemRegenerating"
|
||||
type="textarea"
|
||||
:rows="6"
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'chosen', $event)"
|
||||
/>
|
||||
</div>
|
||||
<div class="field-editor dpo-field is-rejected">
|
||||
<label>Rejected <em>拒选回答,必填</em></label>
|
||||
<el-input
|
||||
:model-value="selectedItem.rejected"
|
||||
:disabled="selectedItemRegenerating"
|
||||
type="textarea"
|
||||
:rows="6"
|
||||
@update:model-value="emit('update:field', selectedItem.id, 'rejected', $event)"
|
||||
/>
|
||||
</div>
|
||||
</template>
|
||||
<div v-if="selectedItem.error" class="validation-error">
|
||||
<i class="fa fa-exclamation-circle" /> {{ selectedItem.error }}
|
||||
</div>
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import type { UploadFile } from 'element-plus'
|
||||
import type { ExternalDataSource, ProcessType, UploadedDataFile } from './types'
|
||||
import type { ExternalDataSource, ProcessType, SourceMode, UploadedDataFile } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
processType: ProcessType
|
||||
sourceMode: SourceMode
|
||||
uploadedFiles: UploadedDataFile[]
|
||||
externalSource: ExternalDataSource
|
||||
externalPulling: boolean
|
||||
@@ -14,6 +15,7 @@ const props = defineProps<{
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:sourceMode': [value: SourceMode]
|
||||
'update:externalSource': [value: ExternalDataSource]
|
||||
'file-change': [file: UploadFile]
|
||||
'remove-file': [uid: string | number]
|
||||
@@ -31,6 +33,14 @@ const AUTH_MODES = [
|
||||
{ value: 'basic', label: '账号密码' },
|
||||
]
|
||||
|
||||
const SSL_MODES = [
|
||||
{ value: 'disable', label: '关闭' },
|
||||
{ value: 'prefer', label: '优先使用(默认)' },
|
||||
{ value: 'require', label: '必须加密' },
|
||||
{ value: 'verify-ca', label: '验证 CA' },
|
||||
{ value: 'verify-full', label: '完整验证' },
|
||||
]
|
||||
|
||||
const FILE_PAGE_SIZE = 10
|
||||
const currentFilePage = ref(1)
|
||||
type FileStage = 'queued' | 'uploading' | 'waiting' | 'processing' | 'success' | 'upload-failed' | 'preview-failed'
|
||||
@@ -45,7 +55,10 @@ const FILE_STAGE_META: Record<FileStage, { label: string; icon: string }> = {
|
||||
'preview-failed': { label: '切分失败', icon: 'fa-exclamation-circle' },
|
||||
}
|
||||
|
||||
const isExternal = computed(() => props.processType === 'external')
|
||||
const isExternal = computed(() => props.sourceMode === 'external')
|
||||
const sourceModeLocked = computed(() => (
|
||||
props.uploadedFiles.length > 0 || props.sourceUploading || props.previewBuilding || props.externalPulling
|
||||
))
|
||||
|
||||
const uploadAccept = computed(() => props.processType === 'unstructured'
|
||||
? '.txt,.md,.markdown,.pdf,.docx,.pptx,.json,.jsonl,.ndjson'
|
||||
@@ -124,6 +137,44 @@ function getFileError(file: UploadedDataFile) {
|
||||
|
||||
<template>
|
||||
<section class="source-upload-step" aria-labelledby="source-upload-title">
|
||||
<div class="source-mode-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>数据来源</h3>
|
||||
<p>在当前步骤选择本地文件上传或从外部系统拉取,处理类型保持不变</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="source-mode-options">
|
||||
<button
|
||||
type="button"
|
||||
class="source-mode-option"
|
||||
:class="{ 'is-active': sourceMode === 'local' }"
|
||||
:disabled="sourceModeLocked"
|
||||
@click="emit('update:sourceMode', 'local')"
|
||||
>
|
||||
<i class="fa fa-upload" aria-hidden="true" />
|
||||
<span><strong>本地上传</strong><small>从当前设备选择文件</small></span>
|
||||
<i class="fa fa-check-circle selection-mark" aria-hidden="true" />
|
||||
</button>
|
||||
<el-tooltip
|
||||
:content="processType === 'structured' ? '从标准化外部数据源配置拉取' : '当前仅支持把外部数据库拉取为结构化数据'"
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
class="source-mode-option"
|
||||
:class="{ 'is-active': sourceMode === 'external' }"
|
||||
:disabled="sourceModeLocked || processType !== 'structured'"
|
||||
@click="emit('update:sourceMode', 'external')"
|
||||
>
|
||||
<i class="fa fa-cloud-download" aria-hidden="true" />
|
||||
<span><strong>外部数据源</strong><small>连接数据库并执行只读拉取</small></span>
|
||||
<i class="fa fa-check-circle selection-mark" aria-hidden="true" />
|
||||
</button>
|
||||
</el-tooltip>
|
||||
</div>
|
||||
<small v-if="sourceModeLocked" class="source-mode-lock-hint">已有来源文件时不可切换;删除现有文件后可重新选择。</small>
|
||||
</div>
|
||||
|
||||
<div v-if="isExternal" class="form-section external-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
@@ -202,6 +253,45 @@ function getFileError(file: UploadedDataFile) {
|
||||
@update:model-value="updateExternalField('limit', Number($event) || 0)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item label="SSL 模式">
|
||||
<el-select
|
||||
:model-value="externalSource.sslMode"
|
||||
aria-label="SSL 模式"
|
||||
@update:model-value="updateExternalField('sslMode', $event)"
|
||||
>
|
||||
<el-option v-for="item in SSL_MODES" :key="item.value" :label="item.label" :value="item.value" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="连接超时">
|
||||
<el-input-number
|
||||
:model-value="externalSource.connectTimeoutSeconds"
|
||||
:min="1"
|
||||
:max="30"
|
||||
controls-position="right"
|
||||
aria-label="连接超时秒数"
|
||||
@update:model-value="updateExternalField('connectTimeoutSeconds', Number($event) || 5)"
|
||||
/>
|
||||
<small>单位:秒</small>
|
||||
</el-form-item>
|
||||
<el-form-item label="查询超时">
|
||||
<el-input-number
|
||||
:model-value="externalSource.statementTimeoutSeconds"
|
||||
:min="1"
|
||||
:max="300"
|
||||
controls-position="right"
|
||||
aria-label="查询超时秒数"
|
||||
@update:model-value="updateExternalField('statementTimeoutSeconds', Number($event) || 30)"
|
||||
/>
|
||||
<small>单位:秒</small>
|
||||
</el-form-item>
|
||||
<el-form-item label="落地文件名">
|
||||
<el-input
|
||||
:model-value="externalSource.fileName"
|
||||
placeholder="external-data.jsonl"
|
||||
aria-label="外部数据落地文件名"
|
||||
@update:model-value="updateExternalField('fileName', $event)"
|
||||
/>
|
||||
</el-form-item>
|
||||
<el-form-item label="只读查询语句" class="external-query-field">
|
||||
<el-input
|
||||
:model-value="externalSource.query"
|
||||
@@ -314,7 +404,7 @@ function getFileError(file: UploadedDataFile) {
|
||||
v-if="uploadedFiles.length === 0"
|
||||
link
|
||||
type="primary"
|
||||
:disabled="previewBuilding"
|
||||
:disabled="previewBuilding || externalPulling"
|
||||
@click="emit('use-sample')"
|
||||
>
|
||||
使用示例数据
|
||||
@@ -326,7 +416,7 @@ function getFileError(file: UploadedDataFile) {
|
||||
drag
|
||||
multiple
|
||||
:accept="uploadAccept"
|
||||
:disabled="previewBuilding"
|
||||
:disabled="previewBuilding || externalPulling"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||
@@ -353,13 +443,13 @@ function getFileError(file: UploadedDataFile) {
|
||||
<el-upload
|
||||
multiple
|
||||
:accept="uploadAccept"
|
||||
:disabled="previewBuilding"
|
||||
:disabled="previewBuilding || externalPulling"
|
||||
:auto-upload="false"
|
||||
:show-file-list="false"
|
||||
:on-change="(file: UploadFile) => emit('file-change', file)"
|
||||
aria-label="继续添加源数据文件"
|
||||
>
|
||||
<el-button size="small" type="primary" :disabled="previewBuilding">继续上传</el-button>
|
||||
<el-button size="small" type="primary" :disabled="previewBuilding || externalPulling">继续上传</el-button>
|
||||
</el-upload>
|
||||
</div>
|
||||
</div>
|
||||
@@ -430,6 +520,68 @@ function getFileError(file: UploadedDataFile) {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.source-mode-section {
|
||||
padding-bottom: 22px;
|
||||
margin-bottom: 22px;
|
||||
border-bottom: 1px solid #edf0f5;
|
||||
}
|
||||
|
||||
.source-mode-options {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.source-mode-option {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
min-height: 76px;
|
||||
padding: 14px 16px;
|
||||
color: #667085;
|
||||
text-align: left;
|
||||
background: #fff;
|
||||
border: 1px solid #dfe3ea;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
|
||||
> .fa:first-child {
|
||||
width: 28px;
|
||||
color: #5b50f2;
|
||||
font-size: 20px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
span {
|
||||
display: flex;
|
||||
flex: 1;
|
||||
flex-direction: column;
|
||||
gap: 5px;
|
||||
}
|
||||
|
||||
strong { color: #344054; font-size: 14px; }
|
||||
small { color: #8a93a3; font-size: 12px; }
|
||||
.selection-mark { color: transparent; }
|
||||
|
||||
&.is-active {
|
||||
background: #fafaff;
|
||||
border-color: #5b50f2;
|
||||
box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08);
|
||||
}
|
||||
|
||||
&.is-active .selection-mark { color: #5b50f2; }
|
||||
&:disabled { cursor: not-allowed; opacity: 0.55; }
|
||||
}
|
||||
|
||||
.source-mode-lock-hint {
|
||||
display: block;
|
||||
margin-top: 9px;
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.form-section {
|
||||
padding: 0;
|
||||
|
||||
@@ -672,6 +824,7 @@ function getFileError(file: UploadedDataFile) {
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.source-mode-options,
|
||||
.external-section .external-grid {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
|
||||
@@ -188,20 +188,6 @@ defineExpose({ validate })
|
||||
</span>
|
||||
<i class="fa fa-check-circle selection-mark" />
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'external' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'external')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-cloud-download" /></span>
|
||||
<span>
|
||||
<strong>外来数据源拉取</strong>
|
||||
<small>适用于数据库、API 接口等需远程拉取的外部数据</small>
|
||||
</span>
|
||||
<i class="fa fa-check-circle selection-mark" />
|
||||
</button>
|
||||
</div>
|
||||
</el-form-item>
|
||||
</div>
|
||||
@@ -274,7 +260,7 @@ defineExpose({ validate })
|
||||
|
||||
.type-options {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
@@ -73,10 +73,21 @@ export const DEFAULT_REASONING_GENERATION_PROMPT = `你是一名具有深厚专
|
||||
3. 信息不足时如实处理:遇到来源信息缺失或相互矛盾时,需在推理中明确指出,并基于来源做出合理假设,但不得编造与来源无关的内容。
|
||||
4. 最终答案须能从前面的推理过程中自然得出,语言简洁、结构清晰。`
|
||||
|
||||
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning') {
|
||||
return outputType === 'reasoning'
|
||||
? DEFAULT_REASONING_GENERATION_PROMPT
|
||||
: DEFAULT_STANDARD_GENERATION_PROMPT
|
||||
export const DEFAULT_DPO_GENERATION_PROMPT = `你是一名偏好数据构造专家。请基于下方来源内容,生成可用于直接偏好优化(DPO)的成对问答数据。
|
||||
|
||||
来源内容:
|
||||
{{ content }}
|
||||
|
||||
任务要求:
|
||||
1. 问题应聚焦来源内容的核心信息,并具有实际训练价值。
|
||||
2. Chosen 必须准确、完整、清晰且严格忠于来源内容。
|
||||
3. Rejected 应当表面合理但包含可辨认的缺陷,例如遗漏关键条件、事实偏差、逻辑不完整或表达含混;不得用乱码、空话或危险内容凑数。
|
||||
4. Chosen 与 Rejected 必须明显不同,且都直接回答同一个问题。`
|
||||
|
||||
export function defaultGenerationPrompt(outputType: 'standard' | 'reasoning' | 'dpo') {
|
||||
if (outputType === 'reasoning') return DEFAULT_REASONING_GENERATION_PROMPT
|
||||
if (outputType === 'dpo') return DEFAULT_DPO_GENERATION_PROMPT
|
||||
return DEFAULT_STANDARD_GENERATION_PROMPT
|
||||
}
|
||||
|
||||
export function isBuiltInGenerationPrompt(value: string) {
|
||||
@@ -92,6 +103,7 @@ export function isBuiltInGenerationPrompt(value: string) {
|
||||
PREVIOUS_DEFAULT_REASONING_GENERATION_PROMPT_3,
|
||||
DEFAULT_STANDARD_GENERATION_PROMPT,
|
||||
DEFAULT_REASONING_GENERATION_PROMPT,
|
||||
DEFAULT_DPO_GENERATION_PROMPT,
|
||||
].map(normalize).includes(normalizedValue)
|
||||
}
|
||||
|
||||
@@ -171,8 +183,9 @@ function generationOptionsFromConfig(
|
||||
config: DataProcessConfig,
|
||||
defaults: GenerationControlOptions,
|
||||
): GenerationControlOptions {
|
||||
const outputType = configValue(config, 'output_type', defaults.outputType) === 'reasoning'
|
||||
? 'reasoning'
|
||||
const configuredOutputType = configValue(config, 'output_type', defaults.outputType)
|
||||
const outputType = configuredOutputType === 'reasoning' || configuredOutputType === 'dpo'
|
||||
? configuredOutputType
|
||||
: 'standard'
|
||||
const configuredPrompt = String(
|
||||
configValue(config, 'generation_prompt', defaults.generationPrompt),
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { DataProcessExternalSourcePayload } from '@/api/modules/dataProcess'
|
||||
import type { DataProcessConfig } from '@/types/dataProcess'
|
||||
import type { ExternalDataSource, SourceMode } from './types'
|
||||
|
||||
export function createDefaultExternalSource(): ExternalDataSource {
|
||||
return {
|
||||
type: 'postgresql',
|
||||
url: '',
|
||||
authMode: 'none',
|
||||
username: '',
|
||||
password: '',
|
||||
limit: 1000,
|
||||
connectTimeoutSeconds: 5,
|
||||
statementTimeoutSeconds: 30,
|
||||
sslMode: 'prefer',
|
||||
query: '',
|
||||
fileName: 'external-data.jsonl',
|
||||
}
|
||||
}
|
||||
|
||||
export function externalSourcePayload(source: ExternalDataSource): DataProcessExternalSourcePayload {
|
||||
return {
|
||||
type: source.type,
|
||||
url: source.url.trim(),
|
||||
auth_mode: source.authMode,
|
||||
username: source.username?.trim() || undefined,
|
||||
password: source.password || undefined,
|
||||
limit: source.limit,
|
||||
connect_timeout_seconds: source.connectTimeoutSeconds,
|
||||
statement_timeout_seconds: source.statementTimeoutSeconds,
|
||||
ssl_mode: source.sslMode,
|
||||
query: source.query?.trim() || undefined,
|
||||
file_name: source.fileName || 'external-data.jsonl',
|
||||
}
|
||||
}
|
||||
|
||||
export function sourceConfigForBackend(mode: SourceMode, source: ExternalDataSource) {
|
||||
const payload = externalSourcePayload(source)
|
||||
const { password: _password, ...safeSource } = payload
|
||||
return {
|
||||
source_mode: mode,
|
||||
external_source: mode === 'external' ? safeSource : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export function restoreExternalSourceConfig(config: DataProcessConfig): {
|
||||
mode: SourceMode
|
||||
source: ExternalDataSource
|
||||
} {
|
||||
const defaults = createDefaultExternalSource()
|
||||
const value = config.external_source
|
||||
if (!value || typeof value !== 'object') {
|
||||
return {
|
||||
mode: config.source_mode === 'external' ? 'external' : 'local',
|
||||
source: defaults,
|
||||
}
|
||||
}
|
||||
const source = value as Record<string, unknown>
|
||||
const sslMode = String(source.ssl_mode || '')
|
||||
return {
|
||||
mode: config.source_mode === 'external' ? 'external' : 'local',
|
||||
source: {
|
||||
...defaults,
|
||||
url: String(source.url || ''),
|
||||
authMode: source.auth_mode === 'basic' ? 'basic' : 'none',
|
||||
username: String(source.username || ''),
|
||||
limit: Number(source.limit) || defaults.limit,
|
||||
connectTimeoutSeconds: Number(source.connect_timeout_seconds) || defaults.connectTimeoutSeconds,
|
||||
statementTimeoutSeconds: Number(source.statement_timeout_seconds) || defaults.statementTimeoutSeconds,
|
||||
sslMode: ['disable', 'prefer', 'require', 'verify-ca', 'verify-full'].includes(sslMode)
|
||||
? sslMode as ExternalDataSource['sslMode']
|
||||
: defaults.sslMode,
|
||||
query: String(source.query || ''),
|
||||
fileName: String(source.file_name || defaults.fileName),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import type {
|
||||
} from '@/types/dataProcess'
|
||||
|
||||
export type ProcessType = 'structured' | 'unstructured' | 'external'
|
||||
export type SourceMode = 'local' | 'external'
|
||||
|
||||
export type StepId = 'create' | 'model' | 'upload' | 'preview' | 'generate' | 'results'
|
||||
|
||||
@@ -88,6 +89,9 @@ export interface ExternalDataSource {
|
||||
username?: string
|
||||
password?: string
|
||||
limit: number
|
||||
connectTimeoutSeconds: number
|
||||
statementTimeoutSeconds: number
|
||||
sslMode: 'disable' | 'prefer' | 'require' | 'verify-ca' | 'verify-full'
|
||||
query?: string
|
||||
fileName?: string
|
||||
}
|
||||
@@ -167,12 +171,18 @@ export interface ResultItem {
|
||||
instruction: string
|
||||
input: string
|
||||
output: string
|
||||
chosen: string
|
||||
rejected: string
|
||||
originalInstruction: string
|
||||
originalInput: string
|
||||
originalOutput: string
|
||||
originalChosen: string
|
||||
originalRejected: string
|
||||
savedInstruction: string
|
||||
savedInput: string
|
||||
savedOutput: string
|
||||
savedChosen: string
|
||||
savedRejected: string
|
||||
savedStatus: 'valid' | 'modified' | 'invalid'
|
||||
status: 'valid' | 'modified' | 'invalid'
|
||||
error?: string
|
||||
|
||||
@@ -12,10 +12,12 @@ import {
|
||||
type DataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { BulkResultRegenerationState, GenerationState, ResultItem } from './types'
|
||||
import type { DataProcessOutputType } from '@/types/dataProcess'
|
||||
|
||||
interface GenerationBindings {
|
||||
taskId: Ref<string | null>
|
||||
dirty: Ref<boolean>
|
||||
outputType: Ref<DataProcessOutputType>
|
||||
beforeGenerate?: () => Promise<boolean | void>
|
||||
}
|
||||
|
||||
@@ -32,12 +34,18 @@ function mapResult(item: DataProcessResult): ResultItem {
|
||||
instruction: item.instruction,
|
||||
input: item.input || '',
|
||||
output: item.output,
|
||||
chosen: item.chosen || item.output || '',
|
||||
rejected: item.rejected || '',
|
||||
originalInstruction: item.original_instruction ?? item.instruction,
|
||||
originalInput: item.original_input ?? item.input ?? '',
|
||||
originalOutput: item.original_output ?? item.output,
|
||||
originalChosen: item.original_chosen ?? item.chosen ?? item.output ?? '',
|
||||
originalRejected: item.original_rejected ?? item.rejected ?? '',
|
||||
savedInstruction: item.instruction,
|
||||
savedInput: item.input || '',
|
||||
savedOutput: item.output,
|
||||
savedChosen: item.chosen || item.output || '',
|
||||
savedRejected: item.rejected || '',
|
||||
savedStatus: item.status,
|
||||
status: item.status,
|
||||
error: item.error || undefined,
|
||||
@@ -246,15 +254,34 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
}
|
||||
}
|
||||
|
||||
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
|
||||
function updateResultField(
|
||||
id: string,
|
||||
field: 'instruction' | 'input' | 'output' | 'chosen' | 'rejected',
|
||||
value: string,
|
||||
) {
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item[field] = value
|
||||
const valid = item.instruction.trim() && item.output.trim()
|
||||
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
|
||||
if (field === 'chosen') item.output = value
|
||||
const isDpo = bindings.outputType.value === 'dpo'
|
||||
const valid = isDpo
|
||||
? Boolean(
|
||||
item.instruction.trim()
|
||||
&& item.chosen.trim()
|
||||
&& item.rejected.trim()
|
||||
&& item.chosen.trim() !== item.rejected.trim()
|
||||
)
|
||||
: Boolean(item.instruction.trim() && item.output.trim())
|
||||
item.error = valid
|
||||
? undefined
|
||||
: isDpo
|
||||
? 'Instruction、Chosen、Rejected 均不能为空,且两个回答不能相同'
|
||||
: 'Instruction 和 Output 不能为空'
|
||||
const changed = item.instruction !== item.originalInstruction
|
||||
|| item.input !== item.originalInput
|
||||
|| item.output !== item.originalOutput
|
||||
|| item.chosen !== item.originalChosen
|
||||
|| item.rejected !== item.originalRejected
|
||||
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
|
||||
bindings.dirty.value = true
|
||||
}
|
||||
@@ -311,6 +338,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
item.instruction !== item.savedInstruction
|
||||
|| item.input !== item.savedInput
|
||||
|| item.output !== item.savedOutput
|
||||
|| item.chosen !== item.savedChosen
|
||||
|| item.rejected !== item.savedRejected
|
||||
))
|
||||
if (unsaved) {
|
||||
selectedResultId.value = unsaved.id
|
||||
@@ -409,12 +438,16 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
item.instruction !== item.savedInstruction
|
||||
|| item.input !== item.savedInput
|
||||
|| item.output !== item.savedOutput
|
||||
|| item.chosen !== item.savedChosen
|
||||
|| item.rejected !== item.savedRejected
|
||||
))
|
||||
for (const item of changed) {
|
||||
const saved = await updateDataProcessResult(taskId, item.id, {
|
||||
instruction: item.instruction,
|
||||
input: item.input,
|
||||
output: item.output,
|
||||
chosen: item.chosen,
|
||||
rejected: item.rejected,
|
||||
expected_updated_at: item.updatedAt,
|
||||
})
|
||||
const index = results.value.findIndex((entry) => entry.id === item.id)
|
||||
@@ -425,7 +458,13 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
function validateResults() {
|
||||
let firstInvalidId: string | null = null
|
||||
for (const item of results.value) {
|
||||
if (!item.instruction.trim() || !item.output.trim() || item.status === 'invalid') {
|
||||
const requiredFieldsInvalid = bindings.outputType.value === 'dpo'
|
||||
? !item.instruction.trim()
|
||||
|| !item.chosen.trim()
|
||||
|| !item.rejected.trim()
|
||||
|| item.chosen.trim() === item.rejected.trim()
|
||||
: !item.instruction.trim() || !item.output.trim()
|
||||
if (requiredFieldsInvalid || item.status === 'invalid') {
|
||||
item.error ||= '结果未通过后端质量校验,请修改后重新保存'
|
||||
item.status = 'invalid'
|
||||
firstInvalidId ??= item.id
|
||||
|
||||
Reference in New Issue
Block a user