feat: 新增外部数据源拉取与 DPO 输出格式支持
- 支持从 PostgreSQL 数据库拉取结构化数据作为训练来源 - 新增 DPO (Direct Preference Optimization) 输出类型 - 支持 chosen/rejected 字段的编辑、校验和发布 - 完善数据预处理切分逻辑和元数据管理 - 移除 OCR 扫描 PDF 功能,保持基础文本解析能力 Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -122,13 +122,13 @@ assert.match(regenerationSource, /route\.name === 'data-process-regenerate'[\s\S
|
||||
assert.match(viewSource, /const currentStep = ref\(0\)/, '重新生成必须从向导第一步开始')
|
||||
|
||||
assert.match(viewSource, /const WIZARD_STEPS = \[/, '向导步骤尚未改为固定常量')
|
||||
for (const title of ['创建任务', '大模型选择', '上传文件', '数据预览', '开始生成', '结果编辑与保存']) {
|
||||
for (const title of ['创建任务', '大模型选择', '数据来源', '数据预览', '开始生成', '结果编辑与保存']) {
|
||||
assert.ok(viewSource.includes(`title: '${title}'`), `缺少固定步骤:${title}`)
|
||||
}
|
||||
assert.match(
|
||||
viewSource,
|
||||
/\{ id: 'create',[\s\S]*?\{ id: 'model',[\s\S]*?\{ id: 'upload',[\s\S]*?\{ id: 'preview',[\s\S]*?\{ id: 'generate',[\s\S]*?\{ id: 'results'/,
|
||||
'六步向导顺序必须为创建任务、大模型选择、上传文件、数据预览、开始生成、结果编辑与保存',
|
||||
'六步向导顺序必须为创建任务、大模型选择、数据来源、数据预览、开始生成、结果编辑与保存',
|
||||
)
|
||||
assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减')
|
||||
assert.match(
|
||||
@@ -380,7 +380,7 @@ for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploa
|
||||
assert.match(viewSource, /<ModelSelectionStep\s+[\s\S]*?v-else-if="currentStepId === 'model'"/, '第二步没有挂载独立大模型选择组件')
|
||||
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第三步没有挂载独立上传组件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续:选择大模型'/, '第一步主按钮没有指向大模型选择')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续:上传文件'/, '第二步主按钮没有指向上传文件')
|
||||
assert.match(viewSource, /if \(currentStepId\.value === 'model'\) return '继续:选择数据来源'/, '第二步主按钮没有指向数据来源选择')
|
||||
assert.match(
|
||||
viewSource,
|
||||
/if \(currentStepId\.value === 'upload'\) \{[\s\S]*?sourceUploading\.value[\s\S]*?'正在上传'[\s\S]*?previewBuilding\.value \? '正在切分' : '继续:数据预览'/,
|
||||
@@ -1032,7 +1032,7 @@ assert.match(stateSource, /Object\.prototype\.hasOwnProperty\.call\(config, key\
|
||||
assert.match(stateSource, /Number\.isFinite\(value\) \? value : fallback/, '配置反向映射没有保留合法数字 0')
|
||||
assert.match(stateSource, /qaPairsPerRow:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_row[\s\S]*?defaults\.qaPairsPerRow/, '结构化生成数量回填没有按 1 到 50 归一化')
|
||||
assert.match(stateSource, /qaPairsPerChunk:\s*normalizeQaPairsGenerationCount\([\s\S]*?qa_pairs_per_chunk[\s\S]*?defaults\.qaPairsPerChunk/, '非结构化生成数量回填没有按 1 到 50 归一化')
|
||||
assert.match(stateSource, /const outputType = configValue\(config, 'output_type', defaults\.outputType\) === 'reasoning'[\s\S]*?\? 'reasoning'[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
|
||||
assert.match(stateSource, /configuredOutputType === 'reasoning' \|\| configuredOutputType === 'dpo'[\s\S]*?\? configuredOutputType[\s\S]*?: 'standard'/, '输出类型没有从任务配置安全回填')
|
||||
assert.match(stateSource, /reasoningDetail:\s*configValue\(config, 'reasoning_detail', defaults\.reasoningDetail\) === 'detailed'[\s\S]*?\? 'detailed'[\s\S]*?: 'normal'/, '推理详细程度没有从任务配置安全回填')
|
||||
assert.match(stateSource, /isBuiltInGenerationPrompt\(configuredPrompt\)[\s\S]*?defaultGenerationPrompt\(outputType\)/, '旧版内置提示语没有按输出类型迁移')
|
||||
assert.match(stateSource, /createStructuredOptionsFromConfig/, '结构化配置缺少后端到表单的反向映射')
|
||||
@@ -1412,7 +1412,7 @@ assert.match(
|
||||
)
|
||||
assert.match(
|
||||
sourceUploadSource,
|
||||
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
|
||||
/<el-upload\s+v-if="uploadedFiles\.length === 0"\s+drag\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>/,
|
||||
'无文件时未保留原有大拖拽上传区或上传配置',
|
||||
)
|
||||
assert.match(
|
||||
@@ -1432,7 +1432,7 @@ assert.ok(sourceUploadSource.includes('旧版 DOC/PPT 请先转换'), '非结构
|
||||
assert.ok(sourceUploadSource.includes('旧版 XLS 请先转换'), '结构化格式提示没有说明旧版 XLS 需转换')
|
||||
assert.match(
|
||||
sourceUploadSource,
|
||||
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
|
||||
/<div\s+class="uploaded-file-list-header">[\s\S]*?已选择 \{\{ uploadedFiles\.length \}\} 个文件[\s\S]*?<div\s+class="continue-upload">\s*<el-upload\s+multiple\s+:accept="uploadAccept"\s+:disabled="previewBuilding \|\| externalPulling"\s+:auto-upload="false"\s+:show-file-list="false"\s+:on-change="\(file: UploadFile\) => emit\('file-change', file\)"[^>]*>\s*<el-button\s+size="small"\s+type="primary"\s+:disabled="previewBuilding \|\| externalPulling">继续上传<\/el-button>\s*<\/el-upload>\s*<\/div>\s*<\/div>/,
|
||||
'有文件时缺少标题右侧的继续上传触发器或上传配置',
|
||||
)
|
||||
assert.match(
|
||||
@@ -1476,4 +1476,13 @@ assert.match(
|
||||
)
|
||||
assert.match(previewSource, /height:\s*clamp\(560px,\s*calc\(100vh - 370px\),\s*720px\)/, '对照预览高度不足以展示切片正文')
|
||||
|
||||
assert.doesNotMatch(taskSetupSource, /外来数据源拉取/, '外部数据源仍错误地放在第一步处理类型中')
|
||||
assert.match(sourceUploadSource, /<h3>数据来源<\/h3>[\s\S]*?本地上传[\s\S]*?外部数据源/, '第三步缺少本地与外部数据来源选择')
|
||||
for (const field of ['地址 / URL', '鉴权方式', 'SSL 模式', '连接超时', '查询超时', '只读查询语句', '拉取条数', '落地文件名']) {
|
||||
assert.ok(sourceUploadSource.includes(field), `外部数据源标准配置缺少:${field}`)
|
||||
}
|
||||
assert.match(generationControlSource, /label="DPO 偏好对" value="dpo"/, '输出类型缺少 DPO 偏好对')
|
||||
assert.match(resultEditorSource, /Chosen[\s\S]*?Rejected/, '结果编辑器缺少 DPO 成对字段')
|
||||
assert.match(viewSource, /sourceConfigForBackend\(sourceMode\.value, externalSource\)/, '任务配置没有保存第三步数据来源模式')
|
||||
|
||||
console.log('数据处理六步向导回归检查通过')
|
||||
|
||||
Reference in New Issue
Block a user