YG-Soft/YG_FT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 数据处理向导拆分组合式函数与子面板

Browse Source 
提取 useDataProcessDraft、useDataProcessGeneration 与 dataProcessCreateState 管理向导状态，新增 StructuredOptionsPanel、UnstructuredOptionsPanel、DatasetSplitEditor 子面板组件，样式抽离为独立 scss，DataProcessCreateView 与 TaskSetupStep 大幅瘦身，回归脚本适配。
This commit is contained in:
caoxiaozhu
2026-07-13 15:28:48 +08:00
parent 735a8a71f5
commit e580ec4791
13 changed files with 1801 additions and 1577 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
frontend/scripts/regression-data-process-wizard.mjs
View File

@@ -13,6 +13,13 @@ const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmD
const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue') const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue')
const viewSource = await readFile(viewPath, 'utf8') const viewSource = await readFile(viewPath, 'utf8')
const layoutSource = await readFile(layoutPath, 'utf8') const layoutSource = await readFile(layoutPath, 'utf8')
const [draftSource, stateSource, generationSource, viewStyleSource] = await Promise.all([
readFile(path.join(createDir, 'useDataProcessDraft.ts'), 'utf8'),
readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'),
readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'),
readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'),
])
const implementationSource = [viewSource, draftSource, stateSource, generationSource].join('\n')
assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog') assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog')
const confirmDialogSource = await readFile(confirmDialogPath, 'utf8') const confirmDialogSource = await readFile(confirmDialogPath, 'utf8')
@@ -43,9 +50,10 @@ assert.match(
'五步向导顺序必须为创建任务、上传文件、数据预览、开始生成、结果编辑与保存', '五步向导顺序必须为创建任务、上传文件、数据预览、开始生成、结果编辑与保存',
) )
assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减') assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减')
assert.match(viewSource, /localStorage\.setItem\(DRAFT_STORAGE_KEY/, '草稿没有持久化') assert.match(draftSource, /localStorage\.setItem\(DATA_PROCESS_DRAFT_STORAGE_KEY/, '草稿没有持久化')
assert.match(viewSource, /localStorage\.getItem\(DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取') assert.match(draftSource, /localStorage\.getItem\(DATA_PROCESS_DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取')
assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿') assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿')
assert.ok(viewSource.split('\n').length < 800, 'DataProcessCreateView 拆分后仍超过 800 行')
const expectedComponents = [ const expectedComponents = [
'TaskSetupStep.vue', 'TaskSetupStep.vue',
@@ -125,19 +133,49 @@ assert.match(previewSource, /\.preview-editor\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]
assert.match(previewSource, /@media \(max-width: 900px\)/, '第三步缺少窄屏上下布局') assert.match(previewSource, /@media \(max-width: 900px\)/, '第三步缺少窄屏上下布局')
const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue') const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue')
const taskSetupSource = await readFile(taskSetupPath, 'utf8') const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue')
const unstructuredOptionsPath = path.join(createDir, 'UnstructuredOptionsPanel.vue')
const datasetSplitEditorPath = path.join(createDir, 'DatasetSplitEditor.vue')
const generationOptionsPath = path.join(createDir, 'GenerationOptionsPanel.vue') const generationOptionsPath = path.join(createDir, 'GenerationOptionsPanel.vue')
const generationControlSource = await readFile(generationOptionsPath, 'utf8')
const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue') const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue')
const sourceUploadSource = await readFile(sourceUploadPath, 'utf8') const [
taskSetupSource,
structuredOptionsSource,
unstructuredOptionsSource,
datasetSplitEditorSource,
generationControlSource,
sourceUploadSource,
] = await Promise.all([
readFile(taskSetupPath, 'utf8'),
readFile(structuredOptionsPath, 'utf8'),
readFile(unstructuredOptionsPath, 'utf8'),
readFile(datasetSplitEditorPath, 'utf8'),
readFile(generationOptionsPath, 'utf8'),
readFile(sourceUploadPath, 'utf8'),
])
const taskSetupFeatureSource = [
taskSetupSource,
structuredOptionsSource,
unstructuredOptionsSource,
datasetSplitEditorSource,
].join('\n')
for (const componentPath of [structuredOptionsPath, unstructuredOptionsPath, datasetSplitEditorPath]) {
assert.ok(existsSync(componentPath), `缺少任务配置拆分组件：${path.basename(componentPath)}`)
}
assert.ok(taskSetupSource.split('\n').length < 800, 'TaskSetupStep 拆分后仍超过 800 行')
assert.match(taskSetupSource, /<StructuredOptionsPanel/, '任务配置没有挂载结构化选项面板')
assert.match(taskSetupSource, /<UnstructuredOptionsPanel/, '任务配置没有挂载非结构化选项面板')
assert.match(structuredOptionsSource, /<DatasetSplitEditor/, '结构化选项没有复用数据集划分编辑器')
assert.match(unstructuredOptionsSource, /<DatasetSplitEditor/, '非结构化选项没有复用数据集划分编辑器')
for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploadedFiles']) { for (const marker of ['<el-upload', '源数据上传', '数据源配置', 'uploadedFiles']) {
assert.ok(!taskSetupSource.includes(marker), `第一步仍包含上传职责：${marker}`) assert.ok(!taskSetupFeatureSource.includes(marker), `第一步仍包含上传职责：${marker}`)
} }
assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第二步没有挂载独立上传组件') assert.match(viewSource, /<SourceUploadStep\s+[\s\S]*?v-else-if="currentStepId === 'upload'"/, '第二步没有挂载独立上传组件')
assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：上传文件'/, '第一步主按钮没有指向上传文件') assert.match(viewSource, /if \(currentStepId\.value === 'create'\) return '继续：上传文件'/, '第一步主按钮没有指向上传文件')
assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续：数据预览'/, '第二步主按钮没有指向数据预览') assert.match(viewSource, /if \(currentStepId\.value === 'upload'\) return '继续：数据预览'/, '第二步主按钮没有指向数据预览')
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = 5/, '生成配置扩展后必须升级草稿版本') assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6/, '安全草稿格式必须升级到 v6')
const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()') const nextFromCreateStart = viewSource.indexOf('async function nextFromCreate()')
const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromCreateStart) const nextFromUploadStart = viewSource.indexOf('function nextFromUpload()', nextFromCreateStart)
@@ -154,9 +192,15 @@ assert.match(nextFromUploadSource, /goToStep\('preview'\)/, '上传步骤完成
assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识') assert.match(viewSource, /function goToStep\(stepId: StepId\)[\s\S]*?WIZARD_STEPS\.findIndex/, '向导跳转没有使用稳定步骤标识')
assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成') assert.match(viewSource, /currentStepId\.value === 'preview'[\s\S]*?goToStep\('generate'\)/, '数据预览步骤没有进入开始生成')
assert.match(viewSource, /generation\.status === 'success'[\s\S]*?goToStep\('results'\)/, '生成成功后没有进入结果编辑与保存') assert.match(viewSource, /generation\.status === 'success'[\s\S]*?goToStep\('results'\)/, '生成成功后没有进入结果编辑与保存')
assert.match(viewSource, /currentStepId:\s*currentStepId\.value/, '草稿没有保存稳定步骤标识') assert.match(draftSource, /currentStepId:\s*bindings\.currentStepId\.value/, '草稿没有保存稳定步骤标识')
assert.match(viewSource, /const LEGACY_V3_STEP_IDS:[^=]*= \['create', 'preview', 'generate', 'results'\]/, 'v3 草稿缺少旧步骤索引映射') const draftSnapshotSource = draftSource.slice(
assert.match(viewSource, /snapshotSchemaVersion === 3[\s\S]*?LEGACY_V3_STEP_IDS/, 'v3 草稿没有迁移到五步索引') draftSource.indexOf('function draftSnapshot()'),
draftSource.indexOf('function writeDraft'),
)
for (const forbiddenField of ['uploadedFiles', 'previewItems', 'results', 'password', 'token']) {
assert.ok(!draftSnapshotSource.includes(forbiddenField), `安全草稿不应持久化：${forbiddenField}`)
}
assert.match(draftSource, /writeDraft\(false\)/, '恢复旧草稿后没有立即覆盖潜在敏感数据')
assert.match(viewSource, /<ResultEditorStep\s+[\s\S]*?v-else-if="currentStepId === 'results'"/, '结果编辑器必须只在结果步骤渲染') assert.match(viewSource, /<ResultEditorStep\s+[\s\S]*?v-else-if="currentStepId === 'results'"/, '结果编辑器必须只在结果步骤渲染')
assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTypeChange\(\)/, '切换处理类型后没有失效旧源数据') assert.match(viewSource, /watch\(processType,[\s\S]*?resetSourceDataForProcessTypeChange\(\)/, '切换处理类型后没有失效旧源数据')
assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择') assert.match(viewSource, /function resetSourceDataForProcessTypeChange\(\)[\s\S]*?uploadedFiles\.value = \[\][\s\S]*?selectedPreviewFileId\.value = null/, '旧源数据失效没有同步清理文件与预览选择')
@@ -170,32 +214,32 @@ for (const option of [
'异常数据过滤', '异常数据过滤',
'敏感信息脱敏', '敏感信息脱敏',
]) { ]) {
assert.ok(taskSetupSource.includes(option), `结构化预处理缺少选项：${option}`) assert.ok(structuredOptionsSource.includes(option), `结构化预处理缺少选项：${option}`)
} }
assert.ok(taskSetupSource.includes('生成选项'), '结构化配置缺少生成选项分类') assert.ok(structuredOptionsSource.includes('生成选项'), '结构化配置缺少生成选项分类')
assert.ok(taskSetupSource.includes('语义丰富表达'), '生成选项缺少语义丰富表达开关') assert.ok(structuredOptionsSource.includes('语义丰富表达'), '生成选项缺少语义丰富表达开关')
assert.ok(taskSetupSource.includes('使用大模型将问答表述得更自然、柔和'), '语义丰富表达缺少辅助说明') assert.ok(structuredOptionsSource.includes('使用大模型将问答表述得更自然、柔和'), '语义丰富表达缺少辅助说明')
for (const splitName of ['训练集', '验证集', '测试集']) { for (const splitName of ['训练集', '验证集', '测试集']) {
assert.ok(taskSetupSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`) assert.ok(datasetSplitEditorSource.includes(splitName), `生成选项缺少数据集划分：${splitName}`)
} }
assert.match(taskSetupSource, /const splitTotal = computed/, '数据集划分缺少比例总和计算') assert.match(datasetSplitEditorSource, /const splitTotal = computed/, '数据集划分缺少比例总和计算')
assert.match(taskSetupSource, /splitTotal\.value !== 100/, '数据集划分缺少总和 100% 校验') assert.match(taskSetupSource, /splitTotal\.value !== 100/, '数据集划分缺少总和 100% 校验')
assert.ok(taskSetupSource.includes('训练集、验证集和测试集比例总和必须为 100%'), '数据集划分缺少就地错误提示') assert.ok(datasetSplitEditorSource.includes('训练集、验证集和测试集比例总和必须为 100%'), '数据集划分缺少就地错误提示')
for (const splitField of ['train', 'validation', 'test']) { for (const splitField of ['train', 'validation', 'test']) {
assert.match( assert.match(
taskSetupSource, datasetSplitEditorSource,
new RegExp(`structuredOptions\\.datasetSplit\\.${splitField}[\\s\\S]*?:min="0"[\\s\\S]*?:max="100"[\\s\\S]*?:step="1"[\\s\\S]*?:precision="0"`), new RegExp(`modelValue\\.${splitField}[\\s\\S]*?:min="0"[\\s\\S]*?:max="100"[\\s\\S]*?:step="1"[\\s\\S]*?:precision="0"`),
`数据集划分字段 ${splitField} 缺少 0～100 的整数限制`, `数据集划分字段 ${splitField} 缺少 0～100 的整数限制`,
) )
} }
assert.match(taskSetupSource, /<el-switch[\s\S]*structuredOptions\.semanticEnrichment/, '语义丰富表达必须使用开关控件') assert.match(structuredOptionsSource, /<el-switch[\s\S]*options\.semanticEnrichment/, '语义丰富表达必须使用开关控件')
assert.match(taskSetupSource, /<el-input-number[\s\S]*structuredOptions\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5') assert.match(structuredOptionsSource, /<el-input-number[\s\S]*options\.qaPairsPerRow[\s\S]*:min="1"[\s\S]*:max="5"/, '每行生成数量必须限制在 1 到 5')
assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态') assert.match(viewSource, /const structuredOptions = ref<StructuredProcessOptions>/, '父页面缺少结构化配置状态')
assert.match(viewSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10') assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10')
assert.match(viewSource, /structuredOptions:\s*\{[\s\S]*\.\.\.structuredOptions\.value/, '结构化配置没有写入草稿') assert.match(draftSource, /structuredOptions:\s*\{[\s\S]*\.\.\.bindings\.structuredOptions\.value/, '结构化配置没有写入草稿')
assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复') assert.match(draftSource, /bindings\.structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复')
assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置') assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置')
assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑') assert.match(generationSource, /createResults\([\s\S]*bindings\.structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑')
for (const field of [ for (const field of [
'generationModelId', 'generationModelId',
@@ -206,23 +250,25 @@ for (const field of [
'minOutputLength', 'minOutputLength',
]) { ]) {
assert.ok(typesSource.includes(field), `生成控制配置缺少字段：${field}`) assert.ok(typesSource.includes(field), `生成控制配置缺少字段：${field}`)
assert.ok(viewSource.includes(field), `父页面默认值或草稿状态缺少字段：${field}`) assert.ok(implementationSource.includes(field), `父页面默认值或草稿状态缺少字段：${field}`)
} }
assert.match(taskSetupSource, /GenerationOptionsPanel/, '生成选项没有复用统一的大模型与质量筛选组件') assert.match(structuredOptionsSource, /GenerationOptionsPanel/, '结构化生成选项没有复用统一的大模型与质量筛选组件')
assert.match(taskSetupSource, /:options="structuredOptions"/, '结构化生成选项未接入统一配置组件') assert.match(unstructuredOptionsSource, /GenerationOptionsPanel/, '非结构化生成选项没有复用统一的大模型与质量筛选组件')
assert.match(taskSetupSource, /:options="unstructuredOptions"/, '非结构化生成选项未接入统一配置组件') assert.match(structuredOptionsSource, /:options="options"/, '结构化生成选项未接入统一配置组件')
assert.match(taskSetupSource, /<h3>大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类') assert.match(unstructuredOptionsSource, /:options="options"/, '非结构化生成选项未接入统一配置组件')
assert.match(taskSetupSource, /section="model"/, '独立大模型分类没有挂载模型配置') assert.match(taskSetupFeatureSource, /<h3>大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类')
assert.match(taskSetupSource, /section="quality"/, '质量筛选没有保留在生成选项分类中') assert.match(taskSetupFeatureSource, /section="model"/, '独立大模型分类没有挂载模型配置')
assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中')
assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类') assert.doesNotMatch(generationControlSource, /<h4>大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类')
for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) { for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) {
assert.ok(generationControlSource.includes(label), `生成控制界面缺少：${label}`) assert.ok(generationControlSource.includes(label), `生成控制界面缺少：${label}`)
} }
assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索') assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索')
assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制') assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制')
assert.match(generationControlSource, /\.model-option-row\s*\{[\s\S]*?display:\s*flex[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有与上方生成选项使用一致的配置行样式') assert.match(generationControlSource, /\.model-field\s*\{[\s\S]*?display:\s*flex[\s\S]*?flex-direction:\s*column/, '大模型字段没有使用稳定的纵向表单布局')
assert.match(viewSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语') assert.match(generationControlSource, /\.generation-config-group\s*\{[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有保留统一配置面板边框')
assert.equal((viewSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语') assert.match(stateSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语')
assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语')
assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示') assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示')
assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示') assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示')
assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制') assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制')
@@ -253,60 +299,55 @@ for (const removedField of ['contextScope', 'generationTypes', 'skipUnanswerable
} }
assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示') assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示')
assert.ok(taskSetupSource.includes('智能预处理'), '简化后缺少智能预处理总开关') assert.ok(unstructuredOptionsSource.includes('智能预处理'), '简化后缺少智能预处理总开关')
assert.ok(taskSetupSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关') assert.ok(unstructuredOptionsSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关')
assert.match(taskSetupSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项') assert.match(unstructuredOptionsSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项')
assert.match(taskSetupSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑') assert.match(unstructuredOptionsSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑')
assert.match(taskSetupSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑') assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑')
assert.ok(taskSetupSource.includes('切分选项'), '非结构化配置缺少切分选项分类') assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类')
for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) { for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) {
assert.ok(taskSetupSource.includes(method), `切分方式缺少选项：${method}`) assert.ok(unstructuredOptionsSource.includes(method), `切分方式缺少选项：${method}`)
} }
for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) { for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) {
assert.ok(taskSetupSource.includes(label), `切分选项缺少配置：${label}`) assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`)
} }
assert.ok(taskSetupSource.includes('高级设置'), '切分选项缺少渐进式高级设置入口') assert.ok(unstructuredOptionsSource.includes('高级设置'), '切分选项缺少渐进式高级设置入口')
assert.match(taskSetupSource, /:aria-expanded="advancedChunkSettingsOpen"/, '高级设置入口缺少展开状态的可访问性标记') assert.match(unstructuredOptionsSource, /:aria-expanded="advancedChunkSettingsOpen"/, '高级设置入口缺少展开状态的可访问性标记')
assert.match(taskSetupSource, /v-if="advancedChunkSettingsOpen"/, '高级设置内容没有默认收起') assert.match(unstructuredOptionsSource, /v-if="advancedChunkSettingsOpen"/, '高级设置内容没有默认收起')
assert.match(taskSetupSource, /watch\(\(\) => props\.unstructuredOptions\.chunkMethod,[\s\S]*?method === 'custom'[\s\S]*?advancedChunkSettingsOpen\.value = true/, '选择自定义分隔符时没有自动展开高级设置') assert.match(unstructuredOptionsSource, /watch\(\(\) => props\.options\.chunkMethod,[\s\S]*?method === 'custom'[\s\S]*?advancedChunkSettingsOpen\.value = true/, '选择自定义分隔符时没有自动展开高级设置')
assert.match(taskSetupSource, /if \(chunkValidationMessage\.value\) \{[\s\S]*?advancedChunkSettingsOpen\.value = true[\s\S]*?return false/, '高级切分配置校验失败时没有重新展开定位') assert.match(unstructuredOptionsSource, /function revealValidation\(\)[\s\S]*?advancedChunkSettingsOpen\.value = true/, '非结构化面板没有暴露高级校验定位能力')
assert.match(taskSetupSource, /const preserveSpecialContentEnabled = computed/, '特殊内容保护没有合并为单一开关') assert.match(taskSetupSource, /if \(chunkValidationMessage\.value\) \{[\s\S]*?revealValidation\(\)[\s\S]*?return false/, '高级切分配置校验失败时没有重新展开定位')
assert.match(taskSetupSource, /function updateSpecialContentProtection/, '特殊内容保护开关缺少更新逻辑') assert.match(unstructuredOptionsSource, /const preserveSpecialContentEnabled = computed/, '特殊内容保护没有合并为单一开关')
assert.match(taskSetupSource, /unstructuredOptions\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token') assert.match(unstructuredOptionsSource, /function updateSpecialContentProtection/, '特殊内容保护开关缺少更新逻辑')
assert.match(taskSetupSource, /unstructuredOptions\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token') assert.match(unstructuredOptionsSource, /options\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token')
assert.match(taskSetupSource, /unstructuredOptions\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token') assert.match(unstructuredOptionsSource, /options\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token')
assert.match(unstructuredOptionsSource, /options\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token')
for (const label of ['语义丰富表达', '每个切片生成数量', '数据集划分']) { for (const label of ['语义丰富表达', '每个切片生成数量', '数据集划分']) {
assert.ok(taskSetupSource.includes(label), `非结构化生成选项缺少：${label}`) assert.ok(taskSetupFeatureSource.includes(label), `非结构化生成选项缺少：${label}`)
} }
for (const removedLabel of ['上下文范围', '问题类型', '跳过无法回答的内容']) { for (const removedLabel of ['上下文范围', '问题类型', '跳过无法回答的内容']) {
assert.ok(!taskSetupSource.includes(removedLabel), `简化后仍显示低频选项：${removedLabel}`) assert.ok(!taskSetupFeatureSource.includes(removedLabel), `简化后仍显示低频选项：${removedLabel}`)
} }
assert.match(taskSetupSource, /unstructuredOptions\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3') assert.match(unstructuredOptionsSource, /options\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3')
assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验') assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验')
assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界') assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界')
assert.ok(taskSetupSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明') assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明')
assert.match(taskSetupSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局') assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局')
assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态') assert.match(viewSource, /const unstructuredOptions = ref<UnstructuredProcessOptions>/, '父页面缺少非结构化配置状态')
assert.match(viewSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分') assert.match(stateSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分')
assert.match(viewSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token') assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token')
assert.match(viewSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token') assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token')
assert.match(viewSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token') assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token')
assert.match(viewSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对') assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对')
assert.match(viewSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.unstructuredOptions\.value/, '非结构化配置没有写入草稿') assert.match(draftSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.bindings\.unstructuredOptions\.value/, '非结构化配置没有写入草稿')
assert.match(viewSource, /unstructuredOptions\.value = \{[\s\S]*restoredUnstructuredOptions\.chunkSize/, '非结构化配置没有从草稿恢复') assert.match(draftSource, /bindings\.unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复')
assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置') assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置')
assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识') assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识')
assert.match(viewSource, /schemaVersion:\s*DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识') assert.match(draftSource, /schemaVersion:\s*DATA_PROCESS_DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识')
assert.match(viewSource, /requiresPreviewMigration/, '旧草稿没有失效旧切片预览') assert.match(draftSource, /bindings\.goToStep\('create'\)/, '恢复配置后应回到安全的创建步骤')
assert.doesNotMatch(
viewSource,
/snapshot\.unstructuredOptions\s*&&\s*!requiresPreviewMigration/,
'草稿版本迁移不应丢失仍受支持的非结构化配置',
)
assert.match(viewSource, /const restoredUnstructuredOptions = snapshot\.unstructuredOptions/, '旧草稿没有通过白名单恢复仍受支持的配置')
assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断') assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断')
const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()') const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()')
@@ -810,10 +851,10 @@ assert.match(sourceUploadSource, /@click="emit\('remove-file', file\.uid\)"/, '
const { descriptor } = parseSfc(viewSource, { filename: viewPath }) const { descriptor } = parseSfc(viewSource, { filename: viewPath })
const template = descriptor.template?.content || '' const template = descriptor.template?.content || ''
assert.equal((template.match(/class="wizard-primary-action"/g) || []).length, 1, '页面必须只有一个主操作入口') assert.equal((template.match(/class="wizard-primary-action"/g) || []).length, 1, '页面必须只有一个主操作入口')
assert.match(viewSource, /onBeforeUnmount\(stopGenerationTimer\)/, '生成计时器没有在卸载时清理') assert.match(viewSource, /onBeforeUnmount\(\(\) => \{[\s\S]*?stopGenerationTimer\(\)[\s\S]*?clearTimeout\(connectionTimer\)[\s\S]*?clearTimeout\(pullTimer\)/, '生成与外部数据源计时器没有在卸载时清理')
assert.match(viewSource, /function scrollToStepTop/, '步骤切换后没有恢复页面顶部上下文') assert.match(viewSource, /function scrollToStepTop/, '步骤切换后没有恢复页面顶部上下文')
assert.match(viewSource, /nextTick\(scrollToStepTop\)/, '步骤切换没有触发页面滚动复位') assert.match(viewSource, /nextTick\(scrollToStepTop\)/, '步骤切换没有触发页面滚动复位')
assert.match(viewSource, /\.wizard-content\s*\{[\s\S]*min-height:\s*400px/, '第一步内容区必须保留足够高度以显示底部操作栏') assert.match(viewStyleSource, /\.wizard-content\s*\{[\s\S]*min-height:\s*400px/, '第一步内容区必须保留足够高度以显示底部操作栏')
assert.match( assert.match(
layoutSource, layoutSource,
/&:has\(\.create-wizard-layout\)\s*\{[\s\S]*?overflow-y:\s*hidden[\s\S]*?\.page-canvas\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]*?min-height:\s*0/, /&:has\(\.create-wizard-layout\)\s*\{[\s\S]*?overflow-y:\s*hidden[\s\S]*?\.page-canvas\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]*?min-height:\s*0/,

642
frontend/src/views/data-process/DataProcessCreateView.vue
View File

@@ -9,17 +9,25 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue' import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue' import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue' import ResultEditorStep from './create/ResultEditorStep.vue'
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel' import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel'
import {
createDefaultStructuredOptions,
createDefaultUnstructuredOptions,
} from './create/dataProcessCreateState'
import {
DATA_PROCESS_DRAFT_STORAGE_KEY,
useDataProcessDraft,
} from './create/useDataProcessDraft'
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useModelsStore } from '@/stores/models' import { useModelsStore } from '@/stores/models'
import type { import type {
ExternalDataSource, ExternalDataSource,
GenerationState,
PreviewItem, PreviewItem,
ProcessType, ProcessType,
ResultItem,
StepId, StepId,
StructuredProcessOptions, StructuredProcessOptions,
UnstructuredProcessOptions, UnstructuredProcessOptions,
UploadedDataFile,
} from './create/types' } from './create/types'
const router = useRouter() const router = useRouter()
@@ -28,10 +36,7 @@ const { list: modelList } = storeToRefs(modelsStore)
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM')) const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>() const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>() const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
const DRAFT_SCHEMA_VERSION = 5
const PREVIEW_MODEL_VERSION = 'document-chunk-v2' const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
const WIZARD_STEPS = [ const WIZARD_STEPS = [
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' }, { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
@@ -40,65 +45,12 @@ const WIZARD_STEPS = [
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' }, { id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
{ id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' }, { id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }> ] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }>
const LEGACY_V3_STEP_IDS: ReadonlyArray<StepId> = ['create', 'preview', 'generate', 'results']
const currentStep = ref(0) const currentStep = ref(0)
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create') const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
const task = reactive({ name: '', description: '' }) const task = reactive({ name: '', description: '' })
const processType = ref<ProcessType>('unstructured') const processType = ref<ProcessType>('unstructured')
const structuredOptions = ref<StructuredProcessOptions>({ const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
})
const unstructuredOptions = ref<UnstructuredProcessOptions>({
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
chunkMethod: 'semantic',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
customDelimiter: '',
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
})
interface UploadedDataFile {
uid: number | string
name: string
size: number
count: number
content: string
}
const uploadedFiles = ref<UploadedDataFile[]>([]) const uploadedFiles = ref<UploadedDataFile[]>([])
const externalSource = reactive<ExternalDataSource>({ const externalSource = reactive<ExternalDataSource>({
@@ -112,35 +64,53 @@ const externalSource = reactive<ExternalDataSource>({
}) })
const externalPulling = ref(false) const externalPulling = ref(false)
const externalConnected = ref(false) const externalConnected = ref(false)
let connectionTimer: ReturnType<typeof setTimeout> | null = null
let pullTimer: ReturnType<typeof setTimeout> | null = null
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', ')) const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
const fileSize = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.size, 0))
const fileCount = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.count, 0))
const previewSignature = ref('') const previewSignature = ref('')
const previewItems = ref<PreviewItem[]>([]) const previewItems = ref<PreviewItem[]>([])
const selectedPreviewFileId = ref<string | null>(null) const selectedPreviewFileId = ref<string | null>(null)
const selectedPreviewId = ref<string | null>(null) const selectedPreviewId = ref<string | null>(null)
const selectedPreviewIdsByFile = ref<Record<string, string>>({}) const selectedPreviewIdsByFile = ref<Record<string, string>>({})
const results = ref<ResultItem[]>([])
const selectedResultId = ref<string | null>(null)
const dirty = ref(false) const dirty = ref(false)
const restoringDraft = ref(false) const restoringDraft = ref(false)
let allowLeave = false let allowLeave = false
const generation = reactive<GenerationState>({ const {
status: 'idle', generation,
progress: 0, results,
message: '确认摘要后即可开始生成，过程中可查看实时进度。', selectedResultId,
resetDownstream,
restoreResult,
startGeneration,
stopGeneration,
stopGenerationTimer,
updateResultField,
validateResults,
} = useDataProcessGeneration({
previewItems,
processType,
structuredOptions,
unstructuredOptions,
dirty,
}) })
let generationTimer: ReturnType<typeof setInterval> | null = null
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length) const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
const previewItemsByFile = computed(() => {
const grouped = new Map<string, PreviewItem[]>()
for (const item of previewItems.value) {
const items = grouped.get(item.sourceFileId) || []
items.push(item)
grouped.set(item.sourceFileId, items)
}
return grouped
})
const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value)) const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value))
const activePreviewItems = computed(() => previewItems.value.filter((item) => item.sourceFileId === selectedPreviewFileId.value)) const activePreviewItems = computed(() => previewItemsByFile.value.get(selectedPreviewFileId.value || '') || [])
const activeSourceText = computed(() => activePreviewFile.value?.content ?? '') const activeSourceText = computed(() => activePreviewFile.value?.content ?? '')
const previewFiles = computed(() => uploadedFiles.value.map((file) => { const previewFiles = computed(() => uploadedFiles.value.map((file) => {
const items = previewItems.value.filter((item) => item.sourceFileId === String(file.uid)) const items = previewItemsByFile.value.get(String(file.uid)) || []
return { return {
id: String(file.uid), id: String(file.uid),
name: file.name, name: file.name,
@@ -169,44 +139,22 @@ const previousStepLabel = computed(() => currentStep.value > 0
? WIZARD_STEPS[currentStep.value - 1].title ? WIZARD_STEPS[currentStep.value - 1].title
: '') : '')
function isStepId(value: unknown): value is StepId {
return typeof value === 'string' && WIZARD_STEPS.some((step) => step.id === value)
}
function goToStep(stepId: StepId) { function goToStep(stepId: StepId) {
const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId) const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
if (nextStepIndex >= 0) currentStep.value = nextStepIndex if (nextStepIndex >= 0) currentStep.value = nextStepIndex
} }
function draftSnapshot() { const { persistDraft, restoreDraft } = useDataProcessDraft({
return { currentStepId,
schemaVersion: DRAFT_SCHEMA_VERSION, task,
currentStep: currentStep.value, processType,
currentStepId: currentStepId.value, structuredOptions,
task: { ...task }, unstructuredOptions,
processType: processType.value, externalSource,
structuredOptions: { restoringDraft,
...structuredOptions.value, dirty,
preprocessOptions: [...structuredOptions.value.preprocessOptions], goToStep,
datasetSplit: { ...structuredOptions.value.datasetSplit }, })
},
unstructuredOptions: {
...unstructuredOptions.value,
preprocessOptions: [...unstructuredOptions.value.preprocessOptions],
datasetSplit: { ...unstructuredOptions.value.datasetSplit },
},
uploadedFiles: uploadedFiles.value,
externalSource: { ...externalSource },
previewSignature: previewSignature.value,
previewItems: previewItems.value,
selectedPreviewFileId: selectedPreviewFileId.value,
selectedPreviewId: selectedPreviewId.value,
selectedPreviewIdsByFile: selectedPreviewIdsByFile.value,
results: results.value,
selectedResultId: selectedResultId.value,
generation: { ...generation },
}
}
function previewAffectingOptions() { function previewAffectingOptions() {
if (processType.value === 'structured') { if (processType.value === 'structured') {
@@ -291,191 +239,11 @@ const generationOptionsSignature = computed(() => JSON.stringify(generationAffec
function buildPreviewSignature() { function buildPreviewSignature() {
const filesSignature = uploadedFiles.value const filesSignature = uploadedFiles.value
.map((file) => `${file.uid}:${file.content}`) .map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`)
.join('|') .join('|')
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}` return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
} }
function persistDraft() {
if (restoringDraft.value) return
try {
localStorage.setItem(DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
} catch {
ElMessage.warning('草稿保存失败，请检查浏览器存储空间')
}
}
type DraftSnapshot = Partial<ReturnType<typeof draftSnapshot>> & {
fileName?: string
fileSize?: number
fileCount?: number
sourceText?: string
}
function restoreDraft() {
try {
const raw = localStorage.getItem(DRAFT_STORAGE_KEY)
if (!raw) return
const snapshot = JSON.parse(raw) as DraftSnapshot
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
const snapshotSchemaVersion = Number(snapshot.schemaVersion) || 0
const requiresPreviewMigration = snapshotSchemaVersion < 3
const legacyStepId = snapshotSchemaVersion === 3
? LEGACY_V3_STEP_IDS[Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), LEGACY_V3_STEP_IDS.length - 1)]
: undefined
const indexedStepId = WIZARD_STEPS[Math.min(
Math.max(Number(snapshot.currentStep) || 0, 0),
WIZARD_STEPS.length - 1,
)]?.id
const restoredStepId = isStepId(snapshot.currentStepId)
? snapshot.currentStepId
: legacyStepId || indexedStepId || 'create'
restoringDraft.value = true
goToStep(requiresPreviewMigration ? 'create' : restoredStepId)
task.name = snapshot.task?.name || ''
task.description = snapshot.task?.description || ''
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
? snapshot.processType
: 'unstructured'
if (snapshot.structuredOptions) {
structuredOptions.value = {
...structuredOptions.value,
...snapshot.structuredOptions,
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
? snapshot.structuredOptions.preprocessOptions
: structuredOptions.value.preprocessOptions,
datasetSplit: {
...structuredOptions.value.datasetSplit,
...snapshot.structuredOptions.datasetSplit,
},
}
}
if (snapshot.unstructuredOptions) {
const restoredUnstructuredOptions = snapshot.unstructuredOptions
const restoredDatasetSplit = restoredUnstructuredOptions.datasetSplit
unstructuredOptions.value = {
...unstructuredOptions.value,
preprocessOptions: Array.isArray(restoredUnstructuredOptions.preprocessOptions)
? restoredUnstructuredOptions.preprocessOptions
: unstructuredOptions.value.preprocessOptions,
chunkMethod: restoredUnstructuredOptions.chunkMethod || unstructuredOptions.value.chunkMethod,
chunkSize: Number.isFinite(restoredUnstructuredOptions.chunkSize)
? restoredUnstructuredOptions.chunkSize
: unstructuredOptions.value.chunkSize,
chunkOverlap: Number.isFinite(restoredUnstructuredOptions.chunkOverlap)
? restoredUnstructuredOptions.chunkOverlap
: unstructuredOptions.value.chunkOverlap,
minChunkSize: Number.isFinite(restoredUnstructuredOptions.minChunkSize)
? restoredUnstructuredOptions.minChunkSize
: unstructuredOptions.value.minChunkSize,
customDelimiter: typeof restoredUnstructuredOptions.customDelimiter === 'string'
? restoredUnstructuredOptions.customDelimiter
: unstructuredOptions.value.customDelimiter,
preserveTables: typeof restoredUnstructuredOptions.preserveTables === 'boolean'
? restoredUnstructuredOptions.preserveTables
: unstructuredOptions.value.preserveTables,
preserveCodeBlocks: typeof restoredUnstructuredOptions.preserveCodeBlocks === 'boolean'
? restoredUnstructuredOptions.preserveCodeBlocks
: unstructuredOptions.value.preserveCodeBlocks,
preserveLists: typeof restoredUnstructuredOptions.preserveLists === 'boolean'
? restoredUnstructuredOptions.preserveLists
: unstructuredOptions.value.preserveLists,
semanticEnrichment: typeof restoredUnstructuredOptions.semanticEnrichment === 'boolean'
? restoredUnstructuredOptions.semanticEnrichment
: unstructuredOptions.value.semanticEnrichment,
qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk)
? restoredUnstructuredOptions.qaPairsPerChunk
: unstructuredOptions.value.qaPairsPerChunk,
generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId,
generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string'
? restoredUnstructuredOptions.generationPrompt
: unstructuredOptions.value.generationPrompt,
temperature: Number.isFinite(restoredUnstructuredOptions.temperature)
? restoredUnstructuredOptions.temperature
: unstructuredOptions.value.temperature,
maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens)
? restoredUnstructuredOptions.maxTokens
: unstructuredOptions.value.maxTokens,
jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean'
? restoredUnstructuredOptions.jsonMode
: unstructuredOptions.value.jsonMode,
qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean'
? restoredUnstructuredOptions.qualityFilterEnabled
: unstructuredOptions.value.qualityFilterEnabled,
filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean'
? restoredUnstructuredOptions.filterLowQuality
: unstructuredOptions.value.filterLowQuality,
filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean'
? restoredUnstructuredOptions.filterShortContent
: unstructuredOptions.value.filterShortContent,
minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength)
? restoredUnstructuredOptions.minOutputLength
: unstructuredOptions.value.minOutputLength,
datasetSplit: {
train: Number.isFinite(restoredDatasetSplit?.train)
? restoredDatasetSplit.train
: unstructuredOptions.value.datasetSplit.train,
validation: Number.isFinite(restoredDatasetSplit?.validation)
? restoredDatasetSplit.validation
: unstructuredOptions.value.datasetSplit.validation,
test: Number.isFinite(restoredDatasetSplit?.test)
? restoredDatasetSplit.test
: unstructuredOptions.value.datasetSplit.test,
},
}
}
if (snapshot.uploadedFiles) {
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
} else if (snapshot.fileName && snapshot.sourceText) {
// Migrate old draft
uploadedFiles.value = [{
uid: 'migrated-draft',
name: snapshot.fileName,
size: Number(snapshot.fileSize) || 0,
count: Number(snapshot.fileCount) || 0,
content: snapshot.sourceText
}]
}
previewSignature.value = requiresPreviewMigration ? '' : snapshot.previewSignature || ''
if (snapshot.externalSource) {
Object.assign(externalSource, snapshot.externalSource)
}
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
previewItems.value = !requiresPreviewMigration && Array.isArray(snapshot.previewItems)
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
: []
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
selectedPreviewIdsByFile.value = requiresPreviewMigration ? {} : snapshot.selectedPreviewIdsByFile || {}
selectedPreviewId.value = requiresPreviewMigration
? null
: snapshot.selectedPreviewId
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|| activePreviewItems.value[0]?.id
|| null
results.value = !requiresPreviewMigration && Array.isArray(snapshot.results) ? snapshot.results : []
selectedResultId.value = requiresPreviewMigration ? null : snapshot.selectedResultId || results.value[0]?.id || null
if (!requiresPreviewMigration) {
const restoredGeneration = snapshot.generation || {}
Object.assign(generation, restoredGeneration)
if (generation.status === 'running') {
generation.status = 'idle'
generation.progress = 0
generation.message = '草稿已恢复，请重新开始生成。'
}
}
dirty.value = false
nextTick(() => { restoringDraft.value = false })
ElMessage.info(requiresPreviewMigration
? '已恢复任务信息，切分规则已更新，请重新生成预览'
: '已恢复上次保存的草稿')
} catch {
localStorage.removeItem(DRAFT_STORAGE_KEY)
}
}
watch( watch(
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource], [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
() => { () => {
@@ -496,9 +264,7 @@ watch(generationOptionsSignature, (currentSignature, previousSignature) => {
}) })
watch( watch(
[currentStep, task, processType, structuredOptions, unstructuredOptions, uploadedFiles, externalSource, previewSignature, [task, processType, structuredOptions, unstructuredOptions, externalSource],
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
results, selectedResultId, generation],
persistDraft, persistDraft,
{ deep: true }, { deep: true },
) )
@@ -566,8 +332,10 @@ function handleTestConnection() {
ElMessage.warning('请先填写数据源地址') ElMessage.warning('请先填写数据源地址')
return return
} }
if (connectionTimer) clearTimeout(connectionTimer)
externalPulling.value = true externalPulling.value = true
setTimeout(() => { connectionTimer = setTimeout(() => {
connectionTimer = null
externalPulling.value = false externalPulling.value = false
externalConnected.value = true externalConnected.value = true
ElMessage.success('数据源连接测试成功') ElMessage.success('数据源连接测试成功')
@@ -579,8 +347,10 @@ function handlePullData() {
ElMessage.warning('请先填写数据源地址') ElMessage.warning('请先填写数据源地址')
return return
} }
if (pullTimer) clearTimeout(pullTimer)
externalPulling.value = true externalPulling.value = true
setTimeout(() => { pullTimer = setTimeout(() => {
pullTimer = null
externalPulling.value = false externalPulling.value = false
externalConnected.value = true externalConnected.value = true
const typeName = externalSource.type.toUpperCase() const typeName = externalSource.type.toUpperCase()
@@ -620,15 +390,6 @@ function resetSourceDataForProcessTypeChange() {
resetDownstream() resetDownstream()
} }
function resetDownstream() {
stopGenerationTimer()
generation.status = 'idle'
generation.progress = 0
generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。'
results.value = []
selectedResultId.value = null
}
async function nextFromCreate() { async function nextFromCreate() {
const valid = await taskSetupRef.value?.validate() const valid = await taskSetupRef.value?.validate()
if (!valid) return if (!valid) return
@@ -738,81 +499,6 @@ async function removePreviewItem(id: string) {
dirty.value = true dirty.value = true
} }
function stopGenerationTimer() {
if (generationTimer) clearInterval(generationTimer)
generationTimer = null
}
function startGeneration() {
stopGenerationTimer()
generation.status = 'running'
generation.progress = 0
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
generationTimer = setInterval(() => {
generation.progress = Math.min(100, generation.progress + 8)
if (generation.progress < 100) return
stopGenerationTimer()
generation.status = 'success'
results.value = createResults(
previewItems.value,
processType.value === 'structured'
? structuredOptions.value
: processType.value === 'unstructured'
? unstructuredOptions.value
: undefined,
)
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
selectedResultId.value = results.value[0]?.id ?? null
dirty.value = true
ElMessage.success('数据处理完成')
}, 180)
}
function stopGeneration() {
stopGenerationTimer()
generation.status = 'failed'
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
}
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item[field] = value
const valid = item.instruction.trim() && item.output.trim()
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
const changed = item.instruction !== item.originalInstruction
|| item.input !== item.originalInput
|| item.output !== item.originalOutput
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
dirty.value = true
}
function restoreResult(id: string) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item.instruction = item.originalInstruction
item.input = item.originalInput
item.output = item.originalOutput
item.error = undefined
item.status = 'valid'
dirty.value = true
}
function validateResults() {
let firstInvalidId: string | null = null
for (const item of results.value) {
if (!item.instruction.trim() || !item.output.trim()) {
item.error = 'Instruction 和 Output 不能为空'
item.status = 'invalid'
firstInvalidId ??= item.id
}
}
if (firstInvalidId) selectedResultId.value = firstInvalidId
return firstInvalidId == null
}
async function handlePrimaryAction() { async function handlePrimaryAction() {
if (currentStepId.value === 'create') { if (currentStepId.value === 'create') {
await nextFromCreate() await nextFromCreate()
@@ -856,7 +542,7 @@ async function saveTask() {
return return
} }
dirty.value = false dirty.value = false
localStorage.removeItem(DRAFT_STORAGE_KEY) localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
allowLeave = true allowLeave = true
ElMessage.success('数据处理任务已保存') ElMessage.success('数据处理任务已保存')
await router.push('/data-process') await router.push('/data-process')
@@ -893,7 +579,11 @@ onBeforeRouteLeave(async () => {
return Boolean(confirmed) return Boolean(confirmed)
}) })
onBeforeUnmount(stopGenerationTimer) onBeforeUnmount(() => {
stopGenerationTimer()
if (connectionTimer) clearTimeout(connectionTimer)
if (pullTimer) clearTimeout(pullTimer)
})
onMounted(() => { onMounted(() => {
restoreDraft() restoreDraft()
modelsStore.load() modelsStore.load()
@@ -1023,198 +713,4 @@ onMounted(() => {
<AppConfirmDialog ref="confirmDialogRef" /> <AppConfirmDialog ref="confirmDialogRef" />
</template> </template>
<style scoped lang="scss"> <style scoped lang="scss" src="./create/data-process-create.scss"></style>
.create-wizard-layout {
display: flex;
flex-direction: column;
height: 100%;
border: 1px solid #eef0f5;
border-radius: 8px;
background: #fff;
}
.wizard-main {
flex: 1;
overflow-y: auto;
padding: 32px;
/* 自定义滚动条 */
&::-webkit-scrollbar {
width: 6px;
}
&::-webkit-scrollbar-thumb {
background: #cbd5e1;
border-radius: 3px;
}
}
.wizard-main-inner {
min-height: 100%;
box-sizing: border-box;
}
.wizard-steps-container {
margin-bottom: 32px;
padding-bottom: 24px;
border-bottom: 1px dashed #e2e8f0;
}
.custom-wizard-steps {
display: flex;
align-items: center;
justify-content: space-between;
max-width: 860px;
margin: 0 auto;
padding: 0 20px;
}
.step-item {
display: flex;
align-items: center;
flex: 1;
&:first-child {
flex: 0;
}
}
.step-connector {
flex: 1;
height: 2px;
background-color: #e2e8f0;
margin: 0 16px;
transition: background-color 0.3s;
}
.step-item.is-completed .step-connector,
.step-item.is-active .step-connector {
background-color: #5146e5;
}
.step-node {
display: flex;
align-items: center;
gap: 10px;
}
.step-icon {
display: flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
border-radius: 50%;
border: 2px solid #cbd5e1;
background-color: #fff;
color: #64748b;
font-size: 13px;
font-weight: 650;
transition: all 0.3s;
}
.step-title {
font-size: 14px;
font-weight: 600;
color: #64748b;
white-space: nowrap;
}
/* State: Active */
.step-item.is-active {
.step-icon {
border-color: #5146e5;
background-color: #eef2ff;
color: #5146e5;
}
.step-title {
color: #1e293b;
}
}
/* State: Completed */
.step-item.is-completed {
.step-icon {
background-color: #5146e5;
border-color: #5146e5;
color: #fff;
}
.step-title {
color: #1e293b;
}
}
.wizard-content {
min-height: 400px;
}
.wizard-footer {
flex-shrink: 0;
height: 64px;
background: #ffffff;
border-top: 1px solid #e2e8f0;
display: grid;
grid-template-columns: 1fr auto 1fr;
align-items: center;
padding: 0 32px;
box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.02);
.footer-left {
justify-self: start;
}
.footer-center {
justify-self: center;
}
.footer-right {
justify-self: end;
}
}
.wizard-primary-action {
min-width: 160px;
}
@media (max-width: 760px) {
.wizard-main {
padding: 24px 20px;
}
.custom-wizard-steps {
padding: 0;
}
.step-connector {
margin: 0 8px;
}
.step-node {
gap: 6px;
}
.step-title {
max-width: 72px;
font-size: 12px;
line-height: 1.35;
white-space: normal;
}
.wizard-footer {
padding: 0 20px;
}
}
@media (max-width: 600px) {
.step-title {
display: none;
}
.step-connector {
margin: 0 6px;
}
.wizard-primary-action {
min-width: 0;
}
}
</style>

185
frontend/src/views/data-process/create/DatasetSplitEditor.vue Normal file
View File

@@ -0,0 +1,185 @@
<script setup lang="ts">
import { computed } from 'vue'
import type { DatasetSplitOptions } from './types'
const props = withDefaults(defineProps<{
modelValue: DatasetSplitOptions
ariaLabelPrefix?: string
}>(), {
ariaLabelPrefix: '',
})
const emit = defineEmits<{
'update:modelValue': [value: DatasetSplitOptions]
}>()
const splitTotal = computed(() => {
const { train, validation, test } = props.modelValue
return train + validation + test
})
function updateField(field: keyof DatasetSplitOptions, value: number | undefined) {
emit('update:modelValue', {
...props.modelValue,
[field]: Math.min(100, Math.max(0, Number(value) || 0)),
})
}
function ariaLabel(label: string) {
return `${props.ariaLabelPrefix}${label}比例`
}
</script>
<template>
<div class="generation-option-row dataset-split-row">
<div class="generation-option-copy">
<strong>数据集划分</strong>
<small>按比例将生成后的问答对随机划分为训练集、验证集和测试集</small>
</div>
<div class="dataset-split-config">
<div class="dataset-split-grid">
<label class="dataset-split-field">
<span>训练集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="modelValue.train"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
:aria-label="ariaLabel('训练集')"
@update:model-value="updateField('train', $event)"
/>
<span>%</span>
</span>
</label>
<label class="dataset-split-field">
<span>验证集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="modelValue.validation"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
:aria-label="ariaLabel('验证集')"
@update:model-value="updateField('validation', $event)"
/>
<span>%</span>
</span>
</label>
<label class="dataset-split-field">
<span>测试集</span>
<span class="dataset-split-input">
<el-input-number
:model-value="modelValue.test"
:min="0"
:max="100"
:step="1"
:precision="0"
controls-position="right"
:aria-label="ariaLabel('测试集')"
@update:model-value="updateField('test', $event)"
/>
<span>%</span>
</span>
</label>
</div>
<div class="dataset-split-summary" :class="{ 'is-invalid': splitTotal !== 100 }">
<span>总计 {{ splitTotal }}%</span>
<span v-if="splitTotal !== 100" role="alert">
训练集、验证集和测试集比例总和必须为 100%
</span>
</div>
</div>
</div>
</template>
<style scoped lang="scss">
.generation-option-row {
display: flex;
min-height: 64px;
align-items: center;
justify-content: space-between;
gap: 24px;
padding: 12px 14px;
box-sizing: border-box;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.generation-option-copy {
display: grid;
gap: 4px;
strong {
color: #344054;
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
}
.dataset-split-row {
align-items: stretch;
flex-direction: column;
}
.dataset-split-config {
display: grid;
gap: 10px;
}
.dataset-split-grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 16px;
}
.dataset-split-field {
display: grid;
gap: 7px;
color: #5f6878;
font-size: 12px;
}
.dataset-split-input {
display: flex;
align-items: center;
gap: 8px;
:deep(.el-input-number) {
width: 100%;
}
}
.dataset-split-summary {
display: flex;
justify-content: space-between;
gap: 12px;
color: #2ca66a;
font-size: 12px;
&.is-invalid {
color: #dc2626;
}
}
@media (max-width: 900px) {
.dataset-split-grid {
grid-template-columns: minmax(0, 1fr);
}
.dataset-split-summary {
align-items: flex-start;
flex-direction: column;
}
}
</style>

2
frontend/src/views/data-process/create/GenerationOptionsPanel.vue
View File

@@ -95,7 +95,7 @@ function sectionValidationMessage() {
@update:model-value="updateField('generationPrompt', $event)" @update:model-value="updateField('generationPrompt', $event)"
/> />
<div class="prompt-variables-hint"> <div class="prompt-variables-hint">
提示：可在文本中通过 <code>{{ content }}</code> 引用当前正在处理的数据内容。 提示：可在文本中通过 <code v-text="'{{ content }}'" /> 引用当前正在处理的数据内容。
</div> </div>
</div> </div>

11
frontend/src/views/data-process/create/SourceUploadStep.vue
View File

@@ -1,18 +1,11 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, ref, watch } from 'vue' import { computed, ref, watch } from 'vue'
import type { UploadFile } from 'element-plus' import type { UploadFile } from 'element-plus'
import type { ExternalDataSource, ProcessType } from './types' import type { ExternalDataSource, ProcessType, UploadedDataFile } from './types'
interface UploadedSourceFile {
uid: string | number
name: string
size: number
count: number
}
const props = defineProps<{ const props = defineProps<{
processType: ProcessType processType: ProcessType
uploadedFiles: UploadedSourceFile[] uploadedFiles: UploadedDataFile[]
externalSource: ExternalDataSource externalSource: ExternalDataSource
externalPulling: boolean externalPulling: boolean
externalConnected: boolean externalConnected: boolean

270
frontend/src/views/data-process/create/StructuredOptionsPanel.vue Normal file
View File

@@ -0,0 +1,270 @@
<script setup lang="ts">
import type { ModelItem } from '@/types'
import type {
GenerationControlOptions,
PreprocessOption,
StructuredProcessOptions,
} from './types'
import DatasetSplitEditor from './DatasetSplitEditor.vue'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
const props = defineProps<{
options: StructuredProcessOptions
generationModels: ModelItem[]
validationAttempted: boolean
generationValidationMessage: string
}>()
const emit = defineEmits<{
'update:options': [value: StructuredProcessOptions]
}>()
const PREPROCESS_OPTIONS: Array<{
value: PreprocessOption
label: string
description: string
}> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
]
function updateField<K extends keyof StructuredProcessOptions>(
field: K,
value: StructuredProcessOptions[K],
) {
emit('update:options', { ...props.options, [field]: value })
}
function updateGenerationOptions(value: GenerationControlOptions) {
emit('update:options', { ...props.options, ...value })
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = value.filter(
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
)
updateField('preprocessOptions', preprocessOptions)
}
</script>
<template>
<div class="form-section structured-options-section">
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>选择在生成问答对之前需要执行的数据处理方式</p>
</div>
</div>
<el-checkbox-group
:model-value="options.preprocessOptions"
class="preprocess-option-grid"
@update:model-value="updatePreprocessOptions"
>
<el-checkbox
v-for="option in PREPROCESS_OPTIONS"
:key="option.value"
:value="option.value"
class="preprocess-option"
>
<span class="preprocess-option-copy">
<strong>{{ option.label }}</strong>
<small>{{ option.description }}</small>
</span>
</el-checkbox>
</el-checkbox-group>
</div>
<div class="form-section generation-options-section">
<div class="section-title-row">
<div>
<h3>生成选项</h3>
<p>配置每条结构化记录生成问答对的方式和数量</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="options"
:models="generationModels"
section="quality"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateGenerationOptions"
/>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>语义丰富表达</strong>
<small>使用大模型将问答表述得更自然、柔和</small>
</div>
<el-switch
:model-value="options.semanticEnrichment"
inline-prompt
active-text="开"
inactive-text="关"
@update:model-value="updateField('semanticEnrichment', Boolean($event))"
/>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每行生成数量</strong>
<small>每行结构化数据生成的问答对数量</small>
</div>
<el-input-number
:model-value="options.qaPairsPerRow"
:min="1"
:max="5"
:step="1"
controls-position="right"
@update:model-value="updateField('qaPairsPerRow', Number($event) || 1)"
/>
</div>
<DatasetSplitEditor
:model-value="options.datasetSplit"
@update:model-value="updateField('datasetSplit', $event)"
/>
</div>
</div>
<div class="form-section model-options-section">
<div class="section-title-row">
<div>
<h3>大模型</h3>
<p>选择数据生成模型，并设置模型输出内容的要求</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="options"
:models="generationModels"
section="model"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateGenerationOptions"
/>
</div>
</div>
</template>
<style scoped lang="scss">
.form-section {
padding: 0 0 26px;
margin-bottom: 26px;
border-bottom: 1px solid #edf0f5;
&:last-child {
margin-bottom: 0;
border-bottom: 0;
}
h3 {
margin: 0 0 16px;
color: #2f3747;
font-size: 15px;
font-weight: 650;
}
}
.section-title-row {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
h3 {
margin-bottom: 5px;
}
p {
margin: 0;
color: #8a93a3;
font-size: 12px;
}
}
.preprocess-option-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
margin-top: 16px;
}
.preprocess-option {
width: 100%;
min-height: 64px;
margin-right: 0;
padding: 12px 14px;
box-sizing: border-box;
align-items: flex-start;
border: 1px solid #e2e5ec;
border-radius: 8px;
transition: border-color 0.18s ease, background-color 0.18s ease;
&:hover {
border-color: #b7b2f7;
}
&.is-checked {
background: #fafaff;
border-color: #8b82f4;
}
:deep(.el-checkbox__input) {
margin-top: 3px;
}
:deep(.el-checkbox__label) {
min-width: 0;
padding-left: 10px;
white-space: normal;
}
}
.preprocess-option-copy,
.generation-option-copy {
display: grid;
gap: 4px;
strong {
color: #344054;
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
}
.generation-option-list {
display: grid;
gap: 12px;
margin-top: 16px;
}
.generation-option-row {
display: flex;
min-height: 64px;
align-items: center;
justify-content: space-between;
gap: 24px;
padding: 12px 14px;
box-sizing: border-box;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
@media (max-width: 900px) {
.preprocess-option-grid {
grid-template-columns: minmax(0, 1fr);
}
.generation-option-row {
align-items: flex-start;
flex-direction: column;
}
}
</style>

934
frontend/src/views/data-process/create/TaskSetupStep.vue
View File

File diff suppressed because it is too large Load Diff

604
frontend/src/views/data-process/create/UnstructuredOptionsPanel.vue Normal file
View File

@@ -0,0 +1,604 @@
<script setup lang="ts">
import { computed, ref, watch } from 'vue'
import type { ModelItem } from '@/types'
import type {
ChunkMethod,
GenerationControlOptions,
UnstructuredPreprocessOption,
UnstructuredProcessOptions,
} from './types'
import DatasetSplitEditor from './DatasetSplitEditor.vue'
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
const props = defineProps<{
options: UnstructuredProcessOptions
generationModels: ModelItem[]
validationAttempted: boolean
generationValidationMessage: string
chunkValidationMessage: string
}>()
const emit = defineEmits<{
'update:options': [value: UnstructuredProcessOptions]
}>()
const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
]
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
{ value: 'semantic', label: '自动语义切分' },
{ value: 'heading', label: '按标题和段落' },
{ value: 'fixed', label: '按固定长度' },
{ value: 'custom', label: '自定义分隔符' },
]
const UNSTRUCTURED_NUMBER_LIMITS = {
chunkSize: { min: 200, max: 2000 },
chunkOverlap: { min: 0, max: 500 },
minChunkSize: { min: 20, max: 500 },
qaPairsPerChunk: { min: 1, max: 3 },
} as const
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
const advancedChunkSettingsOpen = ref(props.options.chunkMethod === 'custom')
const smartPreprocessEnabled = computed(() => SMART_PREPROCESS_OPTIONS.every(
(option) => props.options.preprocessOptions.includes(option),
))
const desensitizeEnabled = computed(() => props.options.preprocessOptions.includes('desensitize'))
const preserveSpecialContentEnabled = computed(() => (
props.options.preserveTables
&& props.options.preserveCodeBlocks
&& props.options.preserveLists
))
watch(() => props.options.chunkMethod, (method) => {
if (method === 'custom') advancedChunkSettingsOpen.value = true
})
watch(() => props.chunkValidationMessage, (message) => {
if (message) advancedChunkSettingsOpen.value = true
})
function updateField<K extends keyof UnstructuredProcessOptions>(
field: K,
value: UnstructuredProcessOptions[K],
) {
emit('update:options', { ...props.options, [field]: value })
}
function updateGenerationOptions(value: GenerationControlOptions) {
emit('update:options', { ...props.options, ...value })
}
function updateSmartPreprocess(value: string | number | boolean) {
const enabled = Boolean(value)
const remainingOptions = props.options.preprocessOptions.filter(
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
)
updateField(
'preprocessOptions',
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
)
}
function updateDesensitize(value: string | number | boolean) {
const preprocessOptions: UnstructuredPreprocessOption[] = props.options.preprocessOptions.filter(
(option) => option !== 'desensitize',
)
if (Boolean(value)) preprocessOptions.push('desensitize')
updateField('preprocessOptions', preprocessOptions)
}
function updateSpecialContentProtection(value: string | number | boolean) {
const enabled = Boolean(value)
emit('update:options', {
...props.options,
preserveTables: enabled,
preserveCodeBlocks: enabled,
preserveLists: enabled,
})
}
function updateUnstructuredNumber(field: UnstructuredNumberField, value: number | undefined) {
const limits = UNSTRUCTURED_NUMBER_LIMITS[field]
const parsedValue = Number(value)
const nextValue = Number.isFinite(parsedValue) ? parsedValue : limits.min
updateField(field, Math.min(limits.max, Math.max(limits.min, nextValue)))
}
function revealValidation() {
advancedChunkSettingsOpen.value = true
}
defineExpose({ revealValidation })
</script>
<template>
<div class="form-section unstructured-options-section">
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>默认使用推荐策略，只需决定是否需要脱敏</p>
</div>
</div>
<div class="generation-option-list compact-option-list">
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>智能预处理</strong>
<small>自动完成内容清理、结构解析、短段合并、质量过滤、去重及上下文保留</small>
</div>
<el-switch
:model-value="smartPreprocessEnabled"
aria-label="智能预处理"
inline-prompt
active-text="开"
inactive-text="关"
@update:model-value="updateSmartPreprocess"
/>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>敏感信息脱敏</strong>
<small>处理姓名、手机号、邮箱和证件号等信息</small>
</div>
<el-switch
:model-value="desensitizeEnabled"
aria-label="敏感信息脱敏"
inline-prompt
active-text="开"
inactive-text="关"
@update:model-value="updateDesensitize"
/>
</div>
</div>
</div>
<div class="form-section chunk-options-section">
<div class="section-title-row">
<div>
<h3>切分选项</h3>
<p>以语义完整为优先，将长文档拆成可独立生成问答的内容块</p>
</div>
</div>
<div class="chunk-settings-grid is-basic">
<label class="config-field">
<span class="config-field-label">切分方式</span>
<el-select
:model-value="options.chunkMethod"
aria-label="切分方式"
@update:model-value="updateField('chunkMethod', $event)"
>
<el-option
v-for="method in CHUNK_METHODS"
:key="method.value"
:label="method.label"
:value="method.value"
/>
</el-select>
<small>推荐使用自动语义切分，在长度限制内优先保留完整句段</small>
</label>
<label class="config-field">
<span class="config-field-label">切片长度</span>
<span class="unit-input">
<el-input-number
:model-value="options.chunkSize"
:min="200"
:max="2000"
:step="50"
:precision="0"
controls-position="right"
aria-label="切片长度"
@update:model-value="updateUnstructuredNumber('chunkSize', $event)"
/>
<span>Token</span>
</span>
<small>单个切片的目标上限，默认 800 Token</small>
</label>
<label class="config-field">
<span class="config-field-label">重叠长度</span>
<span class="unit-input">
<el-input-number
:model-value="options.chunkOverlap"
:min="0"
:max="500"
:step="10"
:precision="0"
controls-position="right"
aria-label="重叠长度"
@update:model-value="updateUnstructuredNumber('chunkOverlap', $event)"
/>
<span>Token</span>
</span>
<small>在相邻切片中最多重复保留的上下文，默认 100 Token</small>
</label>
</div>
<p class="chunk-estimation-note">
Token 数为轻量估算值，实际长度以训练使用的模型分词器为准。
</p>
<p v-if="chunkValidationMessage" class="option-validation-message" role="alert">
{{ chunkValidationMessage }}
</p>
<button
type="button"
class="advanced-settings-toggle"
:aria-expanded="advancedChunkSettingsOpen"
aria-controls="unstructured-advanced-settings"
@click="advancedChunkSettingsOpen = !advancedChunkSettingsOpen"
>
<span>
<strong>高级设置</strong>
<small>最小切片长度、自定义分隔符和特殊内容保护</small>
</span>
<i class="fa" :class="advancedChunkSettingsOpen ? 'fa-chevron-up' : 'fa-chevron-down'" />
</button>
<div
v-if="advancedChunkSettingsOpen"
id="unstructured-advanced-settings"
class="advanced-settings-panel"
>
<div class="advanced-settings-grid">
<label class="config-field">
<span class="config-field-label">最小切片长度</span>
<span class="unit-input">
<el-input-number
:model-value="options.minChunkSize"
:min="20"
:max="500"
:step="10"
:precision="0"
controls-position="right"
aria-label="最小切片长度"
@update:model-value="updateUnstructuredNumber('minChunkSize', $event)"
/>
<span>Token</span>
</span>
<small>过短的尾部内容会尽量并入前一个切片</small>
</label>
<label v-if="options.chunkMethod === 'custom'" class="config-field">
<span class="config-field-label">自定义分隔符</span>
<el-input
:model-value="options.customDelimiter"
maxlength="40"
show-word-limit
placeholder="例如：--- 或 ###"
aria-label="自定义分隔符"
@update:model-value="updateField('customDelimiter', $event)"
/>
<small>系统会优先在分隔符位置结束当前切片</small>
</label>
</div>
<div class="generation-option-row advanced-protection-row">
<div class="generation-option-copy">
<strong>保护表格、代码和列表</strong>
<small>避免切分点破坏特殊内容块的完整性</small>
</div>
<el-switch
:model-value="preserveSpecialContentEnabled"
aria-label="保护表格、代码和列表"
inline-prompt
active-text="开"
inactive-text="关"
@update:model-value="updateSpecialContentProtection"
/>
</div>
</div>
</div>
<div class="form-section generation-options-section">
<div class="section-title-row">
<div>
<h3>生成选项</h3>
<p>只保留会直接影响输出的核心设置</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="options"
:models="generationModels"
section="quality"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateGenerationOptions"
/>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>语义丰富表达</strong>
<small>使用大模型将问答表述得更自然、柔和</small>
</div>
<el-switch
:model-value="options.semanticEnrichment"
aria-label="非结构化语义丰富表达"
inline-prompt
active-text="开"
inactive-text="关"
@update:model-value="updateField('semanticEnrichment', Boolean($event))"
/>
</div>
<div class="generation-option-row">
<div class="generation-option-copy">
<strong>每个切片生成数量</strong>
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
</div>
<el-input-number
:model-value="options.qaPairsPerChunk"
:min="1"
:max="3"
:step="1"
:precision="0"
controls-position="right"
aria-label="每个切片生成数量"
@update:model-value="updateUnstructuredNumber('qaPairsPerChunk', $event)"
/>
</div>
<DatasetSplitEditor
:model-value="options.datasetSplit"
aria-label-prefix="非结构化"
@update:model-value="updateField('datasetSplit', $event)"
/>
</div>
</div>
<div class="form-section model-options-section">
<div class="section-title-row">
<div>
<h3>大模型</h3>
<p>选择数据生成模型，并设置模型输出内容的要求</p>
</div>
</div>
<div class="generation-option-list">
<GenerationOptionsPanel
:options="options"
:models="generationModels"
section="model"
:validation-message="validationAttempted ? generationValidationMessage : ''"
@update:options="updateGenerationOptions"
/>
</div>
</div>
</template>
<style scoped lang="scss">
.form-section {
padding: 0 0 26px;
margin-bottom: 26px;
border-bottom: 1px solid #edf0f5;
&:last-child {
margin-bottom: 0;
border-bottom: 0;
}
h3 {
margin: 0 0 16px;
color: #2f3747;
font-size: 15px;
font-weight: 650;
}
}
.section-title-row {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 16px;
h3 {
margin-bottom: 5px;
}
p {
margin: 0;
color: #8a93a3;
font-size: 12px;
}
}
.generation-option-list {
display: grid;
gap: 12px;
margin-top: 16px;
}
.compact-option-list {
gap: 8px;
.generation-option-row {
min-height: 58px;
}
}
.generation-option-row {
display: flex;
min-height: 64px;
align-items: center;
justify-content: space-between;
gap: 24px;
padding: 12px 14px;
box-sizing: border-box;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.generation-option-copy {
display: grid;
gap: 4px;
strong {
color: #344054;
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
}
.chunk-settings-grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 12px;
margin-top: 16px;
}
.config-field {
display: grid;
align-content: start;
gap: 8px;
min-width: 0;
padding: 14px;
color: #344054;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
> small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
:deep(.el-select),
:deep(.el-input) {
width: 100%;
}
}
.config-field-label {
font-size: 13px;
font-weight: 600;
}
.unit-input {
display: flex;
align-items: center;
gap: 8px;
color: #7b8495;
font-size: 12px;
:deep(.el-input-number) {
width: 100%;
}
}
.option-validation-message {
display: block;
margin: 8px 0 0;
color: #dc2626;
font-size: 12px;
line-height: 1.5;
}
.chunk-estimation-note {
margin: 9px 0 0;
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
.advanced-settings-toggle {
display: flex;
width: 100%;
min-height: 48px;
align-items: center;
justify-content: space-between;
gap: 16px;
margin-top: 12px;
padding: 10px 14px;
color: #344054;
text-align: left;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
cursor: pointer;
transition: border-color 0.18s ease, background-color 0.18s ease;
&:hover {
background: #f8f7ff;
border-color: #b7b2f7;
}
&:focus-visible {
outline: 2px solid #5b50f2;
outline-offset: 2px;
}
> span {
display: grid;
gap: 3px;
}
strong {
font-size: 13px;
font-weight: 600;
}
small {
color: #8a93a3;
font-size: 12px;
line-height: 1.5;
}
i {
flex: 0 0 auto;
color: #7b8495;
font-size: 12px;
}
}
.advanced-settings-panel {
display: grid;
gap: 12px;
margin-top: 8px;
padding: 12px;
background: #fff;
border: 1px solid #e2e5ec;
border-radius: 8px;
}
.advanced-settings-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 12px;
}
@media (max-width: 900px) {
.chunk-settings-grid,
.advanced-settings-grid {
grid-template-columns: minmax(0, 1fr);
}
.generation-option-row {
align-items: flex-start;
flex-direction: column;
}
.compact-option-list .generation-option-row,
.advanced-protection-row {
align-items: center;
flex-direction: row;
}
}
</style>

193
frontend/src/views/data-process/create/data-process-create.scss Normal file
View File

@@ -0,0 +1,193 @@
.create-wizard-layout {
display: flex;
flex-direction: column;
height: 100%;
border: 1px solid #eef0f5;
border-radius: 8px;
background: #fff;
}
.wizard-main {
flex: 1;
overflow-y: auto;
padding: 32px;
&::-webkit-scrollbar {
width: 6px;
}
&::-webkit-scrollbar-thumb {
background: #cbd5e1;
border-radius: 3px;
}
}
.wizard-main-inner {
min-height: 100%;
box-sizing: border-box;
}
.wizard-steps-container {
margin-bottom: 32px;
padding-bottom: 24px;
border-bottom: 1px dashed #e2e8f0;
}
.custom-wizard-steps {
display: flex;
align-items: center;
justify-content: space-between;
max-width: 860px;
margin: 0 auto;
padding: 0 20px;
}
.step-item {
display: flex;
align-items: center;
flex: 1;
&:first-child {
flex: 0;
}
}
.step-connector {
flex: 1;
height: 2px;
margin: 0 16px;
background-color: #e2e8f0;
transition: background-color 0.3s;
}
.step-item.is-completed .step-connector,
.step-item.is-active .step-connector {
background-color: #5146e5;
}
.step-node {
display: flex;
align-items: center;
gap: 10px;
}
.step-icon {
display: flex;
align-items: center;
justify-content: center;
width: 28px;
height: 28px;
color: #64748b;
font-size: 13px;
font-weight: 650;
background-color: #fff;
border: 2px solid #cbd5e1;
border-radius: 50%;
transition: all 0.3s;
}
.step-title {
color: #64748b;
font-size: 14px;
font-weight: 600;
white-space: nowrap;
}
.step-item.is-active {
.step-icon {
color: #5146e5;
background-color: #eef2ff;
border-color: #5146e5;
}
.step-title {
color: #1e293b;
}
}
.step-item.is-completed {
.step-icon {
color: #fff;
background-color: #5146e5;
border-color: #5146e5;
}
.step-title {
color: #1e293b;
}
}
.wizard-content {
min-height: 400px;
}
.wizard-footer {
display: grid;
flex-shrink: 0;
grid-template-columns: 1fr auto 1fr;
align-items: center;
height: 64px;
padding: 0 32px;
background: #fff;
border-top: 1px solid #e2e8f0;
box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.02);
.footer-left {
justify-self: start;
}
.footer-center {
justify-self: center;
}
.footer-right {
justify-self: end;
}
}
.wizard-primary-action {
min-width: 160px;
}
@media (max-width: 760px) {
.wizard-main {
padding: 24px 20px;
}
.custom-wizard-steps {
padding: 0;
}
.step-connector {
margin: 0 8px;
}
.step-node {
gap: 6px;
}
.step-title {
max-width: 72px;
font-size: 12px;
line-height: 1.35;
white-space: normal;
}
.wizard-footer {
padding: 0 20px;
}
}
@media (max-width: 600px) {
.step-title {
display: none;
}
.step-connector {
margin: 0 6px;
}
.wizard-primary-action {
min-width: 0;
}
}

54
frontend/src/views/data-process/create/dataProcessCreateState.ts Normal file
View File

@@ -0,0 +1,54 @@
import type { StructuredProcessOptions, UnstructuredProcessOptions } from './types'
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
export function createDefaultStructuredOptions(): StructuredProcessOptions {
return {
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
}
}
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
return {
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
chunkMethod: 'semantic',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
customDelimiter: '',
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
generationModelId: '',
generationPrompt: DEFAULT_GENERATION_PROMPT,
temperature: 0.7,
maxTokens: 1024,
jsonMode: false,
qualityFilterEnabled: false,
filterLowQuality: true,
filterShortContent: true,
minOutputLength: 20,
}
}

8
frontend/src/views/data-process/create/types.ts
View File

@@ -71,6 +71,14 @@ export interface ExternalDataSource {
limit: number limit: number
} }
export interface UploadedDataFile {
uid: string | number
name: string
size: number
count: number
content: string
}
export interface SourceLine { export interface SourceLine {
number: number number: number
content: string content: string

147
frontend/src/views/data-process/create/useDataProcessDraft.ts Normal file
View File

@@ -0,0 +1,147 @@
import { nextTick, type Reactive, type Ref } from 'vue'
import { ElMessage } from 'element-plus'
import type {
ExternalDataSource,
ProcessType,
StepId,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6
interface DraftSnapshot {
schemaVersion?: number
currentStepId?: StepId
task?: { name?: string; description?: string }
processType?: ProcessType
structuredOptions?: Partial<StructuredProcessOptions>
unstructuredOptions?: Partial<UnstructuredProcessOptions>
externalSource?: Partial<ExternalDataSource>
}
interface DraftBindings {
currentStepId: Readonly<Ref<StepId>>
task: Reactive<{ name: string; description: string }>
processType: Ref<ProcessType>
structuredOptions: Ref<StructuredProcessOptions>
unstructuredOptions: Ref<UnstructuredProcessOptions>
externalSource: Reactive<ExternalDataSource>
restoringDraft: Ref<boolean>
dirty: Ref<boolean>
goToStep: (stepId: StepId) => void
}
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
return {
type: typeof source.type === 'string' ? source.type : 'mysql',
url: typeof source.url === 'string' ? source.url : '',
authMode: typeof source.authMode === 'string' ? source.authMode : 'none',
username: typeof source.username === 'string' ? source.username : '',
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
}
}
function isDraftSnapshot(value: unknown): value is DraftSnapshot {
return Boolean(value && typeof value === 'object')
}
/**
* 只持久化可重建的配置。密码、令牌、文件正文、预览与结果都不进入 localStorage。
*/
export function useDataProcessDraft(bindings: DraftBindings) {
function draftSnapshot(): DraftSnapshot {
return {
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
currentStepId: bindings.currentStepId.value,
task: { ...bindings.task },
processType: bindings.processType.value,
structuredOptions: {
...bindings.structuredOptions.value,
preprocessOptions: [...bindings.structuredOptions.value.preprocessOptions],
datasetSplit: { ...bindings.structuredOptions.value.datasetSplit },
},
unstructuredOptions: {
...bindings.unstructuredOptions.value,
preprocessOptions: [...bindings.unstructuredOptions.value.preprocessOptions],
datasetSplit: { ...bindings.unstructuredOptions.value.datasetSplit },
},
externalSource: sanitizeExternalSource(bindings.externalSource),
}
}
function writeDraft(showWarning: boolean) {
try {
localStorage.setItem(DATA_PROCESS_DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
} catch {
if (showWarning) ElMessage.warning('草稿保存失败，请检查浏览器存储空间')
}
}
function persistDraft() {
if (!bindings.restoringDraft.value) writeDraft(true)
}
function restoreDraft() {
try {
const raw = localStorage.getItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
if (!raw) return
const snapshot: unknown = JSON.parse(raw)
if (!isDraftSnapshot(snapshot)) return
bindings.restoringDraft.value = true
bindings.goToStep('create')
bindings.task.name = snapshot.task?.name || ''
bindings.task.description = snapshot.task?.description || ''
bindings.processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
? snapshot.processType
: 'unstructured'
if (snapshot.structuredOptions) {
bindings.structuredOptions.value = {
...bindings.structuredOptions.value,
...snapshot.structuredOptions,
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
? snapshot.structuredOptions.preprocessOptions
: bindings.structuredOptions.value.preprocessOptions,
datasetSplit: {
...bindings.structuredOptions.value.datasetSplit,
...snapshot.structuredOptions.datasetSplit,
},
}
}
if (snapshot.unstructuredOptions) {
bindings.unstructuredOptions.value = {
...bindings.unstructuredOptions.value,
...snapshot.unstructuredOptions,
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
? snapshot.unstructuredOptions.preprocessOptions
: bindings.unstructuredOptions.value.preprocessOptions,
datasetSplit: {
...bindings.unstructuredOptions.value.datasetSplit,
...snapshot.unstructuredOptions.datasetSplit,
},
}
}
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
password: '',
token: '',
})
bindings.dirty.value = false
nextTick(() => {
bindings.restoringDraft.value = false
// 立即覆盖 v5 及更早草稿，清除其中可能存在的敏感值和大段正文。
writeDraft(false)
})
ElMessage.info('已恢复上次的任务配置，请重新上传或拉取源数据')
} catch {
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
}
}
return { draftSnapshot, persistDraft, restoreDraft }
}

127
frontend/src/views/data-process/create/useDataProcessGeneration.ts Normal file
View File

@@ -0,0 +1,127 @@
import { reactive, ref, type Ref } from 'vue'
import { ElMessage } from 'element-plus'
import { createResults } from './previewModel'
import type {
GenerationState,
PreviewItem,
ProcessType,
ResultItem,
StructuredProcessOptions,
UnstructuredProcessOptions,
} from './types'
interface GenerationBindings {
previewItems: Ref<PreviewItem[]>
processType: Ref<ProcessType>
structuredOptions: Ref<StructuredProcessOptions>
unstructuredOptions: Ref<UnstructuredProcessOptions>
dirty: Ref<boolean>
}
export function useDataProcessGeneration(bindings: GenerationBindings) {
const results = ref<ResultItem[]>([])
const selectedResultId = ref<string | null>(null)
const generation = reactive<GenerationState>({
status: 'idle',
progress: 0,
message: '确认摘要后即可开始生成，过程中可查看实时进度。',
})
let generationTimer: ReturnType<typeof setInterval> | null = null
function stopGenerationTimer() {
if (generationTimer) clearInterval(generationTimer)
generationTimer = null
}
function resetDownstream() {
stopGenerationTimer()
generation.status = 'idle'
generation.progress = 0
generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。'
results.value = []
selectedResultId.value = null
}
function startGeneration() {
stopGenerationTimer()
generation.status = 'running'
generation.progress = 0
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
generationTimer = setInterval(() => {
generation.progress = Math.min(100, generation.progress + 8)
if (generation.progress < 100) return
stopGenerationTimer()
generation.status = 'success'
results.value = createResults(
bindings.previewItems.value,
bindings.processType.value === 'structured'
? bindings.structuredOptions.value
: bindings.processType.value === 'unstructured'
? bindings.unstructuredOptions.value
: undefined,
)
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
selectedResultId.value = results.value[0]?.id ?? null
bindings.dirty.value = true
ElMessage.success('数据处理完成')
}, 180)
}
function stopGeneration() {
stopGenerationTimer()
generation.status = 'failed'
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
}
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item[field] = value
const valid = item.instruction.trim() && item.output.trim()
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
const changed = item.instruction !== item.originalInstruction
|| item.input !== item.originalInput
|| item.output !== item.originalOutput
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
bindings.dirty.value = true
}
function restoreResult(id: string) {
const item = results.value.find((entry) => entry.id === id)
if (!item) return
item.instruction = item.originalInstruction
item.input = item.originalInput
item.output = item.originalOutput
item.error = undefined
item.status = 'valid'
bindings.dirty.value = true
}
function validateResults() {
let firstInvalidId: string | null = null
for (const item of results.value) {
if (!item.instruction.trim() || !item.output.trim()) {
item.error = 'Instruction 和 Output 不能为空'
item.status = 'invalid'
firstInvalidId ??= item.id
}
}
if (firstInvalidId) selectedResultId.value = firstInvalidId
return firstInvalidId == null
}
return {
generation,
results,
selectedResultId,
resetDownstream,
restoreResult,
startGeneration,
stopGeneration,
stopGenerationTimer,
updateResultField,
validateResults,
}
}