diff --git a/frontend/scripts/regression-data-process-wizard.mjs b/frontend/scripts/regression-data-process-wizard.mjs index f5300c9..5593ea0 100644 --- a/frontend/scripts/regression-data-process-wizard.mjs +++ b/frontend/scripts/regression-data-process-wizard.mjs @@ -13,6 +13,13 @@ const confirmDialogPath = path.resolve(scriptDir, '../src/components/AppConfirmD const layoutPath = path.resolve(scriptDir, '../src/layouts/MainLayout.vue') const viewSource = await readFile(viewPath, 'utf8') const layoutSource = await readFile(layoutPath, 'utf8') +const [draftSource, stateSource, generationSource, viewStyleSource] = await Promise.all([ + readFile(path.join(createDir, 'useDataProcessDraft.ts'), 'utf8'), + readFile(path.join(createDir, 'dataProcessCreateState.ts'), 'utf8'), + readFile(path.join(createDir, 'useDataProcessGeneration.ts'), 'utf8'), + readFile(path.join(createDir, 'data-process-create.scss'), 'utf8'), +]) +const implementationSource = [viewSource, draftSource, stateSource, generationSource].join('\n') assert.ok(existsSync(confirmDialogPath), '缺少公共确认弹窗组件 AppConfirmDialog') const confirmDialogSource = await readFile(confirmDialogPath, 'utf8') @@ -43,9 +50,10 @@ assert.match( '五步向导顺序必须为创建任务、上传文件、数据预览、开始生成、结果编辑与保存', ) assert.doesNotMatch(viewSource, /steps\s*=\s*computed|all\.filter/, '步骤仍根据处理类型动态增减') -assert.match(viewSource, /localStorage\.setItem\(DRAFT_STORAGE_KEY/, '草稿没有持久化') -assert.match(viewSource, /localStorage\.getItem\(DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取') +assert.match(draftSource, /localStorage\.setItem\(DATA_PROCESS_DRAFT_STORAGE_KEY/, '草稿没有持久化') +assert.match(draftSource, /localStorage\.getItem\(DATA_PROCESS_DRAFT_STORAGE_KEY\)/, '草稿没有恢复读取') assert.match(viewSource, /restoreDraft\(\)/, '页面没有恢复草稿') +assert.ok(viewSource.split('\n').length < 800, 'DataProcessCreateView 拆分后仍超过 800 行') const expectedComponents = [ 'TaskSetupStep.vue', @@ -125,19 +133,49 @@ assert.match(previewSource, /\.preview-editor\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S] assert.match(previewSource, /@media \(max-width: 900px\)/, '第三步缺少窄屏上下布局') const taskSetupPath = path.join(createDir, 'TaskSetupStep.vue') -const taskSetupSource = await readFile(taskSetupPath, 'utf8') +const structuredOptionsPath = path.join(createDir, 'StructuredOptionsPanel.vue') +const unstructuredOptionsPath = path.join(createDir, 'UnstructuredOptionsPanel.vue') +const datasetSplitEditorPath = path.join(createDir, 'DatasetSplitEditor.vue') const generationOptionsPath = path.join(createDir, 'GenerationOptionsPanel.vue') -const generationControlSource = await readFile(generationOptionsPath, 'utf8') const sourceUploadPath = path.join(createDir, 'SourceUploadStep.vue') -const sourceUploadSource = await readFile(sourceUploadPath, 'utf8') +const [ + taskSetupSource, + structuredOptionsSource, + unstructuredOptionsSource, + datasetSplitEditorSource, + generationControlSource, + sourceUploadSource, +] = await Promise.all([ + readFile(taskSetupPath, 'utf8'), + readFile(structuredOptionsPath, 'utf8'), + readFile(unstructuredOptionsPath, 'utf8'), + readFile(datasetSplitEditorPath, 'utf8'), + readFile(generationOptionsPath, 'utf8'), + readFile(sourceUploadPath, 'utf8'), +]) +const taskSetupFeatureSource = [ + taskSetupSource, + structuredOptionsSource, + unstructuredOptionsSource, + datasetSplitEditorSource, +].join('\n') + +for (const componentPath of [structuredOptionsPath, unstructuredOptionsPath, datasetSplitEditorPath]) { + assert.ok(existsSync(componentPath), `缺少任务配置拆分组件：${path.basename(componentPath)}`) +} +assert.ok(taskSetupSource.split('\n').length < 800, 'TaskSetupStep 拆分后仍超过 800 行') +assert.match(taskSetupSource, //, '父页面缺少结构化配置状态') -assert.match(viewSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10') -assert.match(viewSource, /structuredOptions:\s*\{[\s\S]*\.\.\.structuredOptions\.value/, '结构化配置没有写入草稿') -assert.match(viewSource, /structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复') +assert.match(stateSource, /datasetSplit:\s*\{ train: 80, validation: 10, test: 10 \}/, '数据集划分默认值必须为 80/10/10') +assert.match(draftSource, /structuredOptions:\s*\{[\s\S]*\.\.\.bindings\.structuredOptions\.value/, '结构化配置没有写入草稿') +assert.match(draftSource, /bindings\.structuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.structuredOptions/, '结构化配置没有从草稿恢复') assert.match(viewSource, /v-model:structured-options="structuredOptions"/, '父页面没有双向绑定结构化配置') -assert.match(viewSource, /createResults\([\s\S]*structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑') +assert.match(generationSource, /createResults\([\s\S]*bindings\.structuredOptions\.value/, '每行生成数量没有接入结果生成逻辑') for (const field of [ 'generationModelId', @@ -206,23 +250,25 @@ for (const field of [ 'minOutputLength', ]) { assert.ok(typesSource.includes(field), `生成控制配置缺少字段：${field}`) - assert.ok(viewSource.includes(field), `父页面默认值或草稿状态缺少字段：${field}`) + assert.ok(implementationSource.includes(field), `父页面默认值或草稿状态缺少字段：${field}`) } -assert.match(taskSetupSource, /GenerationOptionsPanel/, '生成选项没有复用统一的大模型与质量筛选组件') -assert.match(taskSetupSource, /:options="structuredOptions"/, '结构化生成选项未接入统一配置组件') -assert.match(taskSetupSource, /:options="unstructuredOptions"/, '非结构化生成选项未接入统一配置组件') -assert.match(taskSetupSource, /

大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类') -assert.match(taskSetupSource, /section="model"/, '独立大模型分类没有挂载模型配置') -assert.match(taskSetupSource, /section="quality"/, '质量筛选没有保留在生成选项分类中') +assert.match(structuredOptionsSource, /GenerationOptionsPanel/, '结构化生成选项没有复用统一的大模型与质量筛选组件') +assert.match(unstructuredOptionsSource, /GenerationOptionsPanel/, '非结构化生成选项没有复用统一的大模型与质量筛选组件') +assert.match(structuredOptionsSource, /:options="options"/, '结构化生成选项未接入统一配置组件') +assert.match(unstructuredOptionsSource, /:options="options"/, '非结构化生成选项未接入统一配置组件') +assert.match(taskSetupFeatureSource, /

大模型<\/h3>/, '大模型必须作为与生成选项平级的独立分类') +assert.match(taskSetupFeatureSource, /section="model"/, '独立大模型分类没有挂载模型配置') +assert.match(taskSetupFeatureSource, /section="quality"/, '质量筛选没有保留在生成选项分类中') assert.doesNotMatch(generationControlSource, /

大模型<\/h4>/, '大模型不应继续作为生成选项内部子分类') for (const label of ['大模型', '数据生成模型', '默认提示语', '质量筛选', '过滤低质量内容', '过滤过短内容', '最少字数']) { assert.ok(generationControlSource.includes(label), `生成控制界面缺少：${label}`) } assert.match(generationControlSource, /filterable/, '数据生成模型下拉必须支持搜索') assert.match(generationControlSource, /maxlength="500"/, '默认提示语缺少合理的长度限制') -assert.match(generationControlSource, /\.model-option-row\s*\{[\s\S]*?display:\s*flex[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有与上方生成选项使用一致的配置行样式') -assert.match(viewSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语') -assert.equal((viewSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语') +assert.match(generationControlSource, /\.model-field\s*\{[\s\S]*?display:\s*flex[\s\S]*?flex-direction:\s*column/, '大模型字段没有使用稳定的纵向表单布局') +assert.match(generationControlSource, /\.generation-config-group\s*\{[\s\S]*?border:\s*1px solid #e2e5ec/, '大模型配置没有保留统一配置面板边框') +assert.match(stateSource, /const DEFAULT_GENERATION_PROMPT\s*=\s*['"][^'"]{40,}['"]/, '大模型配置缺少可直接使用的默认提示语') +assert.equal((stateSource.match(/generationPrompt:\s*DEFAULT_GENERATION_PROMPT/g) || []).length, 2, '结构化与非结构化任务必须共用默认提示语') assert.match(generationControlSource, /v-if="options\.qualityFilterEnabled"/, '质量规则没有随总开关渐进显示') assert.match(generationControlSource, /v-if="options\.filterShortContent"/, '最少字数没有随短内容规则显示') assert.match(generationControlSource, /:min="1"[\s\S]*:max="1000"/, '最少字数缺少 1 到 1000 的边界限制') @@ -253,60 +299,55 @@ for (const removedField of ['contextScope', 'generationTypes', 'skipUnanswerable } assert.match(taskSetupSource, /v-if="processType === 'unstructured'"/, '非结构化配置必须仅在非结构化数据类型下显示') -assert.ok(taskSetupSource.includes('智能预处理'), '简化后缺少智能预处理总开关') -assert.ok(taskSetupSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关') -assert.match(taskSetupSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项') -assert.match(taskSetupSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑') -assert.match(taskSetupSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑') +assert.ok(unstructuredOptionsSource.includes('智能预处理'), '简化后缺少智能预处理总开关') +assert.ok(unstructuredOptionsSource.includes('敏感信息脱敏'), '简化后缺少脱敏开关') +assert.match(unstructuredOptionsSource, /const smartPreprocessEnabled = computed/, '智能预处理没有映射到内部处理项') +assert.match(unstructuredOptionsSource, /function updateSmartPreprocess/, '智能预处理开关缺少更新逻辑') +assert.match(unstructuredOptionsSource, /function updateDesensitize/, '脱敏开关缺少更新逻辑') -assert.ok(taskSetupSource.includes('切分选项'), '非结构化配置缺少切分选项分类') +assert.ok(unstructuredOptionsSource.includes('切分选项'), '非结构化配置缺少切分选项分类') for (const method of ['自动语义切分', '按标题和段落', '按固定长度', '自定义分隔符']) { - assert.ok(taskSetupSource.includes(method), `切分方式缺少选项：${method}`) + assert.ok(unstructuredOptionsSource.includes(method), `切分方式缺少选项：${method}`) } for (const label of ['切片长度', '重叠长度', '最小切片长度', '保护表格、代码和列表']) { - assert.ok(taskSetupSource.includes(label), `切分选项缺少配置：${label}`) + assert.ok(unstructuredOptionsSource.includes(label), `切分选项缺少配置：${label}`) } -assert.ok(taskSetupSource.includes('高级设置'), '切分选项缺少渐进式高级设置入口') -assert.match(taskSetupSource, /:aria-expanded="advancedChunkSettingsOpen"/, '高级设置入口缺少展开状态的可访问性标记') -assert.match(taskSetupSource, /v-if="advancedChunkSettingsOpen"/, '高级设置内容没有默认收起') -assert.match(taskSetupSource, /watch\(\(\) => props\.unstructuredOptions\.chunkMethod,[\s\S]*?method === 'custom'[\s\S]*?advancedChunkSettingsOpen\.value = true/, '选择自定义分隔符时没有自动展开高级设置') -assert.match(taskSetupSource, /if \(chunkValidationMessage\.value\) \{[\s\S]*?advancedChunkSettingsOpen\.value = true[\s\S]*?return false/, '高级切分配置校验失败时没有重新展开定位') -assert.match(taskSetupSource, /const preserveSpecialContentEnabled = computed/, '特殊内容保护没有合并为单一开关') -assert.match(taskSetupSource, /function updateSpecialContentProtection/, '特殊内容保护开关缺少更新逻辑') -assert.match(taskSetupSource, /unstructuredOptions\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token') -assert.match(taskSetupSource, /unstructuredOptions\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token') -assert.match(taskSetupSource, /unstructuredOptions\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token') +assert.ok(unstructuredOptionsSource.includes('高级设置'), '切分选项缺少渐进式高级设置入口') +assert.match(unstructuredOptionsSource, /:aria-expanded="advancedChunkSettingsOpen"/, '高级设置入口缺少展开状态的可访问性标记') +assert.match(unstructuredOptionsSource, /v-if="advancedChunkSettingsOpen"/, '高级设置内容没有默认收起') +assert.match(unstructuredOptionsSource, /watch\(\(\) => props\.options\.chunkMethod,[\s\S]*?method === 'custom'[\s\S]*?advancedChunkSettingsOpen\.value = true/, '选择自定义分隔符时没有自动展开高级设置') +assert.match(unstructuredOptionsSource, /function revealValidation\(\)[\s\S]*?advancedChunkSettingsOpen\.value = true/, '非结构化面板没有暴露高级校验定位能力') +assert.match(taskSetupSource, /if \(chunkValidationMessage\.value\) \{[\s\S]*?revealValidation\(\)[\s\S]*?return false/, '高级切分配置校验失败时没有重新展开定位') +assert.match(unstructuredOptionsSource, /const preserveSpecialContentEnabled = computed/, '特殊内容保护没有合并为单一开关') +assert.match(unstructuredOptionsSource, /function updateSpecialContentProtection/, '特殊内容保护开关缺少更新逻辑') +assert.match(unstructuredOptionsSource, /options\.chunkSize[\s\S]*?:min="200"[\s\S]*?:max="2000"/, '切片长度必须限制在 200 到 2000 Token') +assert.match(unstructuredOptionsSource, /options\.chunkOverlap[\s\S]*?:min="0"[\s\S]*?:max="500"/, '重叠长度必须限制在 0 到 500 Token') +assert.match(unstructuredOptionsSource, /options\.minChunkSize[\s\S]*?:min="20"[\s\S]*?:max="500"/, '最小切片长度必须限制在 20 到 500 Token') for (const label of ['语义丰富表达', '每个切片生成数量', '数据集划分']) { - assert.ok(taskSetupSource.includes(label), `非结构化生成选项缺少：${label}`) + assert.ok(taskSetupFeatureSource.includes(label), `非结构化生成选项缺少：${label}`) } for (const removedLabel of ['上下文范围', '问题类型', '跳过无法回答的内容']) { - assert.ok(!taskSetupSource.includes(removedLabel), `简化后仍显示低频选项：${removedLabel}`) + assert.ok(!taskSetupFeatureSource.includes(removedLabel), `简化后仍显示低频选项：${removedLabel}`) } -assert.match(taskSetupSource, /unstructuredOptions\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3') +assert.match(unstructuredOptionsSource, /options\.qaPairsPerChunk[\s\S]*?:min="1"[\s\S]*?:max="3"/, '每个切片生成数量必须限制在 1 到 3') assert.match(taskSetupSource, /unstructuredSplitTotal\.value !== 100/, '非结构化数据集划分缺少总和 100% 校验') assert.match(taskSetupSource, /chunkOverlap \+ props\.unstructuredOptions\.minChunkSize[\s\S]*?> props\.unstructuredOptions\.chunkSize/, '切分配置未校验重叠长度与最小切片长度的组合边界') -assert.ok(taskSetupSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明') -assert.match(taskSetupSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局') +assert.ok(unstructuredOptionsSource.includes('Token 数为轻量估算值'), '切片长度缺少 Token 估算说明') +assert.match(unstructuredOptionsSource, /\.chunk-settings-grid\s*\{[\s\S]*?grid-template-columns:\s*repeat\(3,\s*minmax\(0,\s*1fr\)\)/, '核心切分参数没有收紧为三列布局') assert.match(viewSource, /const unstructuredOptions = ref/, '父页面缺少非结构化配置状态') -assert.match(viewSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分') -assert.match(viewSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token') -assert.match(viewSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token') -assert.match(viewSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token') -assert.match(viewSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对') -assert.match(viewSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.unstructuredOptions\.value/, '非结构化配置没有写入草稿') -assert.match(viewSource, /unstructuredOptions\.value = \{[\s\S]*restoredUnstructuredOptions\.chunkSize/, '非结构化配置没有从草稿恢复') +assert.match(stateSource, /chunkMethod:\s*'semantic'/, '非结构化默认切分方式必须为自动语义切分') +assert.match(stateSource, /chunkSize:\s*800/, '默认切片长度必须为 800 Token') +assert.match(stateSource, /chunkOverlap:\s*100/, '默认重叠长度必须为 100 Token') +assert.match(stateSource, /minChunkSize:\s*100/, '默认最小切片长度必须为 100 Token') +assert.match(stateSource, /qaPairsPerChunk:\s*1/, '默认每个切片必须生成 1 个问答对') +assert.match(draftSource, /unstructuredOptions:\s*\{[\s\S]*\.\.\.bindings\.unstructuredOptions\.value/, '非结构化配置没有写入草稿') +assert.match(draftSource, /bindings\.unstructuredOptions\.value = \{[\s\S]*\.\.\.snapshot\.unstructuredOptions/, '非结构化配置没有从草稿恢复') assert.match(viewSource, /v-model:unstructured-options="unstructuredOptions"/, '父页面没有双向绑定非结构化配置') -assert.match(viewSource, /const DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识') -assert.match(viewSource, /schemaVersion:\s*DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识') -assert.match(viewSource, /requiresPreviewMigration/, '旧草稿没有失效旧切片预览') -assert.doesNotMatch( - viewSource, - /snapshot\.unstructuredOptions\s*&&\s*!requiresPreviewMigration/, - '草稿版本迁移不应丢失仍受支持的非结构化配置', -) -assert.match(viewSource, /const restoredUnstructuredOptions = snapshot\.unstructuredOptions/, '旧草稿没有通过白名单恢复仍受支持的配置') +assert.match(draftSource, /DATA_PROCESS_DRAFT_SCHEMA_VERSION = \d+/, '草稿缺少版本标识') +assert.match(draftSource, /schemaVersion:\s*DATA_PROCESS_DRAFT_SCHEMA_VERSION/, '草稿快照没有写入版本标识') +assert.match(draftSource, /bindings\.goToStep\('create'\)/, '恢复配置后应回到安全的创建步骤') assert.match(viewSource, /JSON\.stringify\(previewAffectingOptions\(\)\)/, '影响切分的非结构化配置没有纳入预览失效判断') const previewOptionsStart = viewSource.indexOf('function previewAffectingOptions()') @@ -810,10 +851,10 @@ assert.match(sourceUploadSource, /@click="emit\('remove-file', file\.uid\)"/, ' const { descriptor } = parseSfc(viewSource, { filename: viewPath }) const template = descriptor.template?.content || '' assert.equal((template.match(/class="wizard-primary-action"/g) || []).length, 1, '页面必须只有一个主操作入口') -assert.match(viewSource, /onBeforeUnmount\(stopGenerationTimer\)/, '生成计时器没有在卸载时清理') +assert.match(viewSource, /onBeforeUnmount\(\(\) => \{[\s\S]*?stopGenerationTimer\(\)[\s\S]*?clearTimeout\(connectionTimer\)[\s\S]*?clearTimeout\(pullTimer\)/, '生成与外部数据源计时器没有在卸载时清理') assert.match(viewSource, /function scrollToStepTop/, '步骤切换后没有恢复页面顶部上下文') assert.match(viewSource, /nextTick\(scrollToStepTop\)/, '步骤切换没有触发页面滚动复位') -assert.match(viewSource, /\.wizard-content\s*\{[\s\S]*min-height:\s*400px/, '第一步内容区必须保留足够高度以显示底部操作栏') +assert.match(viewStyleSource, /\.wizard-content\s*\{[\s\S]*min-height:\s*400px/, '第一步内容区必须保留足够高度以显示底部操作栏') assert.match( layoutSource, /&:has\(\.create-wizard-layout\)\s*\{[\s\S]*?overflow-y:\s*hidden[\s\S]*?\.page-canvas\s*\{[\s\S]*?flex:\s*1 1 auto[\s\S]*?min-height:\s*0/, diff --git a/frontend/src/views/data-process/DataProcessCreateView.vue b/frontend/src/views/data-process/DataProcessCreateView.vue index 379b582..a8eb7c9 100644 --- a/frontend/src/views/data-process/DataProcessCreateView.vue +++ b/frontend/src/views/data-process/DataProcessCreateView.vue @@ -9,17 +9,25 @@ import SourceUploadStep from './create/SourceUploadStep.vue' import PreviewCompareStep from './create/PreviewCompareStep.vue' import GenerationStep from './create/GenerationStep.vue' import ResultEditorStep from './create/ResultEditorStep.vue' -import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel' +import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel' +import { + createDefaultStructuredOptions, + createDefaultUnstructuredOptions, +} from './create/dataProcessCreateState' +import { + DATA_PROCESS_DRAFT_STORAGE_KEY, + useDataProcessDraft, +} from './create/useDataProcessDraft' +import { useDataProcessGeneration } from './create/useDataProcessGeneration' import { useModelsStore } from '@/stores/models' import type { ExternalDataSource, - GenerationState, PreviewItem, ProcessType, - ResultItem, StepId, StructuredProcessOptions, UnstructuredProcessOptions, + UploadedDataFile, } from './create/types' const router = useRouter() @@ -28,10 +36,7 @@ const { list: modelList } = storeToRefs(modelsStore) const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM')) const taskSetupRef = ref>() const confirmDialogRef = ref>() -const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft' -const DRAFT_SCHEMA_VERSION = 5 const PREVIEW_MODEL_VERSION = 'document-chunk-v2' -const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。' const WIZARD_STEPS = [ { id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' }, @@ -40,65 +45,12 @@ const WIZARD_STEPS = [ { id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' }, { id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' }, ] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }> -const LEGACY_V3_STEP_IDS: ReadonlyArray = ['create', 'preview', 'generate', 'results'] - const currentStep = ref(0) const currentStepId = computed(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create') const task = reactive({ name: '', description: '' }) const processType = ref('unstructured') -const structuredOptions = ref({ - preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], - semanticEnrichment: false, - qaPairsPerRow: 1, - datasetSplit: { train: 80, validation: 10, test: 10 }, - generationModelId: '', - generationPrompt: DEFAULT_GENERATION_PROMPT, - temperature: 0.7, - maxTokens: 1024, - jsonMode: false, - qualityFilterEnabled: false, - filterLowQuality: true, - filterShortContent: true, - minOutputLength: 20, -}) -const unstructuredOptions = ref({ - preprocessOptions: [ - 'clean_invalid_content', - 'detect_document_structure', - 'merge_short_content', - 'filter_low_quality', - 'deduplicate_content', - 'preserve_context', - ], - chunkMethod: 'semantic', - chunkSize: 800, - chunkOverlap: 100, - minChunkSize: 100, - customDelimiter: '', - preserveTables: true, - preserveCodeBlocks: true, - preserveLists: true, - semanticEnrichment: false, - qaPairsPerChunk: 1, - datasetSplit: { train: 80, validation: 10, test: 10 }, - generationModelId: '', - generationPrompt: DEFAULT_GENERATION_PROMPT, - temperature: 0.7, - maxTokens: 1024, - jsonMode: false, - qualityFilterEnabled: false, - filterLowQuality: true, - filterShortContent: true, - minOutputLength: 20, -}) -interface UploadedDataFile { - uid: number | string - name: string - size: number - count: number - content: string -} - +const structuredOptions = ref(createDefaultStructuredOptions()) +const unstructuredOptions = ref(createDefaultUnstructuredOptions()) const uploadedFiles = ref([]) const externalSource = reactive({ @@ -112,35 +64,53 @@ const externalSource = reactive({ }) const externalPulling = ref(false) const externalConnected = ref(false) +let connectionTimer: ReturnType | null = null +let pullTimer: ReturnType | null = null const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', ')) -const fileSize = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.size, 0)) -const fileCount = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.count, 0)) const previewSignature = ref('') const previewItems = ref([]) const selectedPreviewFileId = ref(null) const selectedPreviewId = ref(null) const selectedPreviewIdsByFile = ref>({}) -const results = ref([]) -const selectedResultId = ref(null) const dirty = ref(false) const restoringDraft = ref(false) let allowLeave = false -const generation = reactive({ - status: 'idle', - progress: 0, - message: '确认摘要后即可开始生成，过程中可查看实时进度。', +const { + generation, + results, + selectedResultId, + resetDownstream, + restoreResult, + startGeneration, + stopGeneration, + stopGenerationTimer, + updateResultField, + validateResults, +} = useDataProcessGeneration({ + previewItems, + processType, + structuredOptions, + unstructuredOptions, + dirty, }) -let generationTimer: ReturnType | null = null - const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length) +const previewItemsByFile = computed(() => { + const grouped = new Map() + for (const item of previewItems.value) { + const items = grouped.get(item.sourceFileId) || [] + items.push(item) + grouped.set(item.sourceFileId, items) + } + return grouped +}) const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value)) -const activePreviewItems = computed(() => previewItems.value.filter((item) => item.sourceFileId === selectedPreviewFileId.value)) +const activePreviewItems = computed(() => previewItemsByFile.value.get(selectedPreviewFileId.value || '') || []) const activeSourceText = computed(() => activePreviewFile.value?.content ?? '') const previewFiles = computed(() => uploadedFiles.value.map((file) => { - const items = previewItems.value.filter((item) => item.sourceFileId === String(file.uid)) + const items = previewItemsByFile.value.get(String(file.uid)) || [] return { id: String(file.uid), name: file.name, @@ -169,44 +139,22 @@ const previousStepLabel = computed(() => currentStep.value > 0 ? WIZARD_STEPS[currentStep.value - 1].title : '') -function isStepId(value: unknown): value is StepId { - return typeof value === 'string' && WIZARD_STEPS.some((step) => step.id === value) -} - function goToStep(stepId: StepId) { const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId) if (nextStepIndex >= 0) currentStep.value = nextStepIndex } -function draftSnapshot() { - return { - schemaVersion: DRAFT_SCHEMA_VERSION, - currentStep: currentStep.value, - currentStepId: currentStepId.value, - task: { ...task }, - processType: processType.value, - structuredOptions: { - ...structuredOptions.value, - preprocessOptions: [...structuredOptions.value.preprocessOptions], - datasetSplit: { ...structuredOptions.value.datasetSplit }, - }, - unstructuredOptions: { - ...unstructuredOptions.value, - preprocessOptions: [...unstructuredOptions.value.preprocessOptions], - datasetSplit: { ...unstructuredOptions.value.datasetSplit }, - }, - uploadedFiles: uploadedFiles.value, - externalSource: { ...externalSource }, - previewSignature: previewSignature.value, - previewItems: previewItems.value, - selectedPreviewFileId: selectedPreviewFileId.value, - selectedPreviewId: selectedPreviewId.value, - selectedPreviewIdsByFile: selectedPreviewIdsByFile.value, - results: results.value, - selectedResultId: selectedResultId.value, - generation: { ...generation }, - } -} +const { persistDraft, restoreDraft } = useDataProcessDraft({ + currentStepId, + task, + processType, + structuredOptions, + unstructuredOptions, + externalSource, + restoringDraft, + dirty, + goToStep, +}) function previewAffectingOptions() { if (processType.value === 'structured') { @@ -291,191 +239,11 @@ const generationOptionsSignature = computed(() => JSON.stringify(generationAffec function buildPreviewSignature() { const filesSignature = uploadedFiles.value - .map((file) => `${file.uid}:${file.content}`) + .map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`) .join('|') return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}` } -function persistDraft() { - if (restoringDraft.value) return - try { - localStorage.setItem(DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot())) - } catch { - ElMessage.warning('草稿保存失败，请检查浏览器存储空间') - } -} - -type DraftSnapshot = Partial> & { - fileName?: string - fileSize?: number - fileCount?: number - sourceText?: string -} - -function restoreDraft() { - try { - const raw = localStorage.getItem(DRAFT_STORAGE_KEY) - if (!raw) return - const snapshot = JSON.parse(raw) as DraftSnapshot - if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return - const snapshotSchemaVersion = Number(snapshot.schemaVersion) || 0 - const requiresPreviewMigration = snapshotSchemaVersion < 3 - const legacyStepId = snapshotSchemaVersion === 3 - ? LEGACY_V3_STEP_IDS[Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), LEGACY_V3_STEP_IDS.length - 1)] - : undefined - const indexedStepId = WIZARD_STEPS[Math.min( - Math.max(Number(snapshot.currentStep) || 0, 0), - WIZARD_STEPS.length - 1, - )]?.id - const restoredStepId = isStepId(snapshot.currentStepId) - ? snapshot.currentStepId - : legacyStepId || indexedStepId || 'create' - - restoringDraft.value = true - goToStep(requiresPreviewMigration ? 'create' : restoredStepId) - task.name = snapshot.task?.name || '' - task.description = snapshot.task?.description || '' - processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external' - ? snapshot.processType - : 'unstructured' - if (snapshot.structuredOptions) { - structuredOptions.value = { - ...structuredOptions.value, - ...snapshot.structuredOptions, - preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions) - ? snapshot.structuredOptions.preprocessOptions - : structuredOptions.value.preprocessOptions, - datasetSplit: { - ...structuredOptions.value.datasetSplit, - ...snapshot.structuredOptions.datasetSplit, - }, - } - } - if (snapshot.unstructuredOptions) { - const restoredUnstructuredOptions = snapshot.unstructuredOptions - const restoredDatasetSplit = restoredUnstructuredOptions.datasetSplit - unstructuredOptions.value = { - ...unstructuredOptions.value, - preprocessOptions: Array.isArray(restoredUnstructuredOptions.preprocessOptions) - ? restoredUnstructuredOptions.preprocessOptions - : unstructuredOptions.value.preprocessOptions, - chunkMethod: restoredUnstructuredOptions.chunkMethod || unstructuredOptions.value.chunkMethod, - chunkSize: Number.isFinite(restoredUnstructuredOptions.chunkSize) - ? restoredUnstructuredOptions.chunkSize - : unstructuredOptions.value.chunkSize, - chunkOverlap: Number.isFinite(restoredUnstructuredOptions.chunkOverlap) - ? restoredUnstructuredOptions.chunkOverlap - : unstructuredOptions.value.chunkOverlap, - minChunkSize: Number.isFinite(restoredUnstructuredOptions.minChunkSize) - ? restoredUnstructuredOptions.minChunkSize - : unstructuredOptions.value.minChunkSize, - customDelimiter: typeof restoredUnstructuredOptions.customDelimiter === 'string' - ? restoredUnstructuredOptions.customDelimiter - : unstructuredOptions.value.customDelimiter, - preserveTables: typeof restoredUnstructuredOptions.preserveTables === 'boolean' - ? restoredUnstructuredOptions.preserveTables - : unstructuredOptions.value.preserveTables, - preserveCodeBlocks: typeof restoredUnstructuredOptions.preserveCodeBlocks === 'boolean' - ? restoredUnstructuredOptions.preserveCodeBlocks - : unstructuredOptions.value.preserveCodeBlocks, - preserveLists: typeof restoredUnstructuredOptions.preserveLists === 'boolean' - ? restoredUnstructuredOptions.preserveLists - : unstructuredOptions.value.preserveLists, - semanticEnrichment: typeof restoredUnstructuredOptions.semanticEnrichment === 'boolean' - ? restoredUnstructuredOptions.semanticEnrichment - : unstructuredOptions.value.semanticEnrichment, - qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk) - ? restoredUnstructuredOptions.qaPairsPerChunk - : unstructuredOptions.value.qaPairsPerChunk, - generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId, - generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string' - ? restoredUnstructuredOptions.generationPrompt - : unstructuredOptions.value.generationPrompt, - temperature: Number.isFinite(restoredUnstructuredOptions.temperature) - ? restoredUnstructuredOptions.temperature - : unstructuredOptions.value.temperature, - maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens) - ? restoredUnstructuredOptions.maxTokens - : unstructuredOptions.value.maxTokens, - jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean' - ? restoredUnstructuredOptions.jsonMode - : unstructuredOptions.value.jsonMode, - qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean' - ? restoredUnstructuredOptions.qualityFilterEnabled - : unstructuredOptions.value.qualityFilterEnabled, - filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean' - ? restoredUnstructuredOptions.filterLowQuality - : unstructuredOptions.value.filterLowQuality, - filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean' - ? restoredUnstructuredOptions.filterShortContent - : unstructuredOptions.value.filterShortContent, - minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength) - ? restoredUnstructuredOptions.minOutputLength - : unstructuredOptions.value.minOutputLength, - datasetSplit: { - train: Number.isFinite(restoredDatasetSplit?.train) - ? restoredDatasetSplit.train - : unstructuredOptions.value.datasetSplit.train, - validation: Number.isFinite(restoredDatasetSplit?.validation) - ? restoredDatasetSplit.validation - : unstructuredOptions.value.datasetSplit.validation, - test: Number.isFinite(restoredDatasetSplit?.test) - ? restoredDatasetSplit.test - : unstructuredOptions.value.datasetSplit.test, - }, - } - } - - if (snapshot.uploadedFiles) { - uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : [] - } else if (snapshot.fileName && snapshot.sourceText) { - // Migrate old draft - uploadedFiles.value = [{ - uid: 'migrated-draft', - name: snapshot.fileName, - size: Number(snapshot.fileSize) || 0, - count: Number(snapshot.fileCount) || 0, - content: snapshot.sourceText - }] - } - - previewSignature.value = requiresPreviewMigration ? '' : snapshot.previewSignature || '' - if (snapshot.externalSource) { - Object.assign(externalSource, snapshot.externalSource) - } - const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '') - previewItems.value = !requiresPreviewMigration && Array.isArray(snapshot.previewItems) - ? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId })) - : [] - selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null - selectedPreviewIdsByFile.value = requiresPreviewMigration ? {} : snapshot.selectedPreviewIdsByFile || {} - selectedPreviewId.value = requiresPreviewMigration - ? null - : snapshot.selectedPreviewId - || selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? ''] - || activePreviewItems.value[0]?.id - || null - results.value = !requiresPreviewMigration && Array.isArray(snapshot.results) ? snapshot.results : [] - selectedResultId.value = requiresPreviewMigration ? null : snapshot.selectedResultId || results.value[0]?.id || null - if (!requiresPreviewMigration) { - const restoredGeneration = snapshot.generation || {} - Object.assign(generation, restoredGeneration) - if (generation.status === 'running') { - generation.status = 'idle' - generation.progress = 0 - generation.message = '草稿已恢复，请重新开始生成。' - } - } - dirty.value = false - nextTick(() => { restoringDraft.value = false }) - ElMessage.info(requiresPreviewMigration - ? '已恢复任务信息，切分规则已更新，请重新生成预览' - : '已恢复上次保存的草稿') - } catch { - localStorage.removeItem(DRAFT_STORAGE_KEY) - } -} - watch( [() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource], () => { @@ -496,9 +264,7 @@ watch(generationOptionsSignature, (currentSignature, previousSignature) => { }) watch( - [currentStep, task, processType, structuredOptions, unstructuredOptions, uploadedFiles, externalSource, previewSignature, - previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile, - results, selectedResultId, generation], + [task, processType, structuredOptions, unstructuredOptions, externalSource], persistDraft, { deep: true }, ) @@ -566,8 +332,10 @@ function handleTestConnection() { ElMessage.warning('请先填写数据源地址') return } + if (connectionTimer) clearTimeout(connectionTimer) externalPulling.value = true - setTimeout(() => { + connectionTimer = setTimeout(() => { + connectionTimer = null externalPulling.value = false externalConnected.value = true ElMessage.success('数据源连接测试成功') @@ -579,8 +347,10 @@ function handlePullData() { ElMessage.warning('请先填写数据源地址') return } + if (pullTimer) clearTimeout(pullTimer) externalPulling.value = true - setTimeout(() => { + pullTimer = setTimeout(() => { + pullTimer = null externalPulling.value = false externalConnected.value = true const typeName = externalSource.type.toUpperCase() @@ -620,15 +390,6 @@ function resetSourceDataForProcessTypeChange() { resetDownstream() } -function resetDownstream() { - stopGenerationTimer() - generation.status = 'idle' - generation.progress = 0 - generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。' - results.value = [] - selectedResultId.value = null -} - async function nextFromCreate() { const valid = await taskSetupRef.value?.validate() if (!valid) return @@ -738,81 +499,6 @@ async function removePreviewItem(id: string) { dirty.value = true } -function stopGenerationTimer() { - if (generationTimer) clearInterval(generationTimer) - generationTimer = null -} - -function startGeneration() { - stopGenerationTimer() - generation.status = 'running' - generation.progress = 0 - generation.message = '正在应用预览修改并生成标准化结果，请稍候。' - - generationTimer = setInterval(() => { - generation.progress = Math.min(100, generation.progress + 8) - if (generation.progress < 100) return - - stopGenerationTimer() - generation.status = 'success' - results.value = createResults( - previewItems.value, - processType.value === 'structured' - ? structuredOptions.value - : processType.value === 'unstructured' - ? unstructuredOptions.value - : undefined, - ) - generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。` - selectedResultId.value = results.value[0]?.id ?? null - dirty.value = true - ElMessage.success('数据处理完成') - }, 180) -} - -function stopGeneration() { - stopGenerationTimer() - generation.status = 'failed' - generation.message = '任务已停止，预览修改仍然保留，可以重新生成。' -} - -function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) { - const item = results.value.find((entry) => entry.id === id) - if (!item) return - item[field] = value - const valid = item.instruction.trim() && item.output.trim() - item.error = valid ? undefined : 'Instruction 和 Output 不能为空' - const changed = item.instruction !== item.originalInstruction - || item.input !== item.originalInput - || item.output !== item.originalOutput - item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid' - dirty.value = true -} - -function restoreResult(id: string) { - const item = results.value.find((entry) => entry.id === id) - if (!item) return - item.instruction = item.originalInstruction - item.input = item.originalInput - item.output = item.originalOutput - item.error = undefined - item.status = 'valid' - dirty.value = true -} - -function validateResults() { - let firstInvalidId: string | null = null - for (const item of results.value) { - if (!item.instruction.trim() || !item.output.trim()) { - item.error = 'Instruction 和 Output 不能为空' - item.status = 'invalid' - firstInvalidId ??= item.id - } - } - if (firstInvalidId) selectedResultId.value = firstInvalidId - return firstInvalidId == null -} - async function handlePrimaryAction() { if (currentStepId.value === 'create') { await nextFromCreate() @@ -856,7 +542,7 @@ async function saveTask() { return } dirty.value = false - localStorage.removeItem(DRAFT_STORAGE_KEY) + localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY) allowLeave = true ElMessage.success('数据处理任务已保存') await router.push('/data-process') @@ -893,7 +579,11 @@ onBeforeRouteLeave(async () => { return Boolean(confirmed) }) -onBeforeUnmount(stopGenerationTimer) +onBeforeUnmount(() => { + stopGenerationTimer() + if (connectionTimer) clearTimeout(connectionTimer) + if (pullTimer) clearTimeout(pullTimer) +}) onMounted(() => { restoreDraft() modelsStore.load() @@ -1023,198 +713,4 @@ onMounted(() => { - + diff --git a/frontend/src/views/data-process/create/DatasetSplitEditor.vue b/frontend/src/views/data-process/create/DatasetSplitEditor.vue new file mode 100644 index 0000000..d65d846 --- /dev/null +++ b/frontend/src/views/data-process/create/DatasetSplitEditor.vue @@ -0,0 +1,185 @@ + + + + + diff --git a/frontend/src/views/data-process/create/GenerationOptionsPanel.vue b/frontend/src/views/data-process/create/GenerationOptionsPanel.vue index 466906f..144bb1b 100644 --- a/frontend/src/views/data-process/create/GenerationOptionsPanel.vue +++ b/frontend/src/views/data-process/create/GenerationOptionsPanel.vue @@ -95,7 +95,7 @@ function sectionValidationMessage() { @update:model-value="updateField('generationPrompt', $event)" />
- 提示：可在文本中通过 {{ content }} 引用当前正在处理的数据内容。 + 提示：可在文本中通过 引用当前正在处理的数据内容。
diff --git a/frontend/src/views/data-process/create/SourceUploadStep.vue b/frontend/src/views/data-process/create/SourceUploadStep.vue index d5a2283..6ef9027 100644 --- a/frontend/src/views/data-process/create/SourceUploadStep.vue +++ b/frontend/src/views/data-process/create/SourceUploadStep.vue @@ -1,18 +1,11 @@ + + + + diff --git a/frontend/src/views/data-process/create/TaskSetupStep.vue b/frontend/src/views/data-process/create/TaskSetupStep.vue index ccd84db..62ec863 100644 --- a/frontend/src/views/data-process/create/TaskSetupStep.vue +++ b/frontend/src/views/data-process/create/TaskSetupStep.vue @@ -1,18 +1,15 @@ + + + + diff --git a/frontend/src/views/data-process/create/data-process-create.scss b/frontend/src/views/data-process/create/data-process-create.scss new file mode 100644 index 0000000..1da57fb --- /dev/null +++ b/frontend/src/views/data-process/create/data-process-create.scss @@ -0,0 +1,193 @@ +.create-wizard-layout { + display: flex; + flex-direction: column; + height: 100%; + border: 1px solid #eef0f5; + border-radius: 8px; + background: #fff; +} + +.wizard-main { + flex: 1; + overflow-y: auto; + padding: 32px; + + &::-webkit-scrollbar { + width: 6px; + } + + &::-webkit-scrollbar-thumb { + background: #cbd5e1; + border-radius: 3px; + } +} + +.wizard-main-inner { + min-height: 100%; + box-sizing: border-box; +} + +.wizard-steps-container { + margin-bottom: 32px; + padding-bottom: 24px; + border-bottom: 1px dashed #e2e8f0; +} + +.custom-wizard-steps { + display: flex; + align-items: center; + justify-content: space-between; + max-width: 860px; + margin: 0 auto; + padding: 0 20px; +} + +.step-item { + display: flex; + align-items: center; + flex: 1; + + &:first-child { + flex: 0; + } +} + +.step-connector { + flex: 1; + height: 2px; + margin: 0 16px; + background-color: #e2e8f0; + transition: background-color 0.3s; +} + +.step-item.is-completed .step-connector, +.step-item.is-active .step-connector { + background-color: #5146e5; +} + +.step-node { + display: flex; + align-items: center; + gap: 10px; +} + +.step-icon { + display: flex; + align-items: center; + justify-content: center; + width: 28px; + height: 28px; + color: #64748b; + font-size: 13px; + font-weight: 650; + background-color: #fff; + border: 2px solid #cbd5e1; + border-radius: 50%; + transition: all 0.3s; +} + +.step-title { + color: #64748b; + font-size: 14px; + font-weight: 600; + white-space: nowrap; +} + +.step-item.is-active { + .step-icon { + color: #5146e5; + background-color: #eef2ff; + border-color: #5146e5; + } + + .step-title { + color: #1e293b; + } +} + +.step-item.is-completed { + .step-icon { + color: #fff; + background-color: #5146e5; + border-color: #5146e5; + } + + .step-title { + color: #1e293b; + } +} + +.wizard-content { + min-height: 400px; +} + +.wizard-footer { + display: grid; + flex-shrink: 0; + grid-template-columns: 1fr auto 1fr; + align-items: center; + height: 64px; + padding: 0 32px; + background: #fff; + border-top: 1px solid #e2e8f0; + box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.02); + + .footer-left { + justify-self: start; + } + + .footer-center { + justify-self: center; + } + + .footer-right { + justify-self: end; + } +} + +.wizard-primary-action { + min-width: 160px; +} + +@media (max-width: 760px) { + .wizard-main { + padding: 24px 20px; + } + + .custom-wizard-steps { + padding: 0; + } + + .step-connector { + margin: 0 8px; + } + + .step-node { + gap: 6px; + } + + .step-title { + max-width: 72px; + font-size: 12px; + line-height: 1.35; + white-space: normal; + } + + .wizard-footer { + padding: 0 20px; + } +} + +@media (max-width: 600px) { + .step-title { + display: none; + } + + .step-connector { + margin: 0 6px; + } + + .wizard-primary-action { + min-width: 0; + } +} diff --git a/frontend/src/views/data-process/create/dataProcessCreateState.ts b/frontend/src/views/data-process/create/dataProcessCreateState.ts new file mode 100644 index 0000000..56debdc --- /dev/null +++ b/frontend/src/views/data-process/create/dataProcessCreateState.ts @@ -0,0 +1,54 @@ +import type { StructuredProcessOptions, UnstructuredProcessOptions } from './types' + +export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。' + +export function createDefaultStructuredOptions(): StructuredProcessOptions { + return { + preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'], + semanticEnrichment: false, + qaPairsPerRow: 1, + datasetSplit: { train: 80, validation: 10, test: 10 }, + generationModelId: '', + generationPrompt: DEFAULT_GENERATION_PROMPT, + temperature: 0.7, + maxTokens: 1024, + jsonMode: false, + qualityFilterEnabled: false, + filterLowQuality: true, + filterShortContent: true, + minOutputLength: 20, + } +} + +export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions { + return { + preprocessOptions: [ + 'clean_invalid_content', + 'detect_document_structure', + 'merge_short_content', + 'filter_low_quality', + 'deduplicate_content', + 'preserve_context', + ], + chunkMethod: 'semantic', + chunkSize: 800, + chunkOverlap: 100, + minChunkSize: 100, + customDelimiter: '', + preserveTables: true, + preserveCodeBlocks: true, + preserveLists: true, + semanticEnrichment: false, + qaPairsPerChunk: 1, + datasetSplit: { train: 80, validation: 10, test: 10 }, + generationModelId: '', + generationPrompt: DEFAULT_GENERATION_PROMPT, + temperature: 0.7, + maxTokens: 1024, + jsonMode: false, + qualityFilterEnabled: false, + filterLowQuality: true, + filterShortContent: true, + minOutputLength: 20, + } +} diff --git a/frontend/src/views/data-process/create/types.ts b/frontend/src/views/data-process/create/types.ts index 0e6a3d0..2a94129 100644 --- a/frontend/src/views/data-process/create/types.ts +++ b/frontend/src/views/data-process/create/types.ts @@ -71,6 +71,14 @@ export interface ExternalDataSource { limit: number } +export interface UploadedDataFile { + uid: string | number + name: string + size: number + count: number + content: string +} + export interface SourceLine { number: number content: string diff --git a/frontend/src/views/data-process/create/useDataProcessDraft.ts b/frontend/src/views/data-process/create/useDataProcessDraft.ts new file mode 100644 index 0000000..a19b6bd --- /dev/null +++ b/frontend/src/views/data-process/create/useDataProcessDraft.ts @@ -0,0 +1,147 @@ +import { nextTick, type Reactive, type Ref } from 'vue' +import { ElMessage } from 'element-plus' +import type { + ExternalDataSource, + ProcessType, + StepId, + StructuredProcessOptions, + UnstructuredProcessOptions, +} from './types' + +export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft' +export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6 + +interface DraftSnapshot { + schemaVersion?: number + currentStepId?: StepId + task?: { name?: string; description?: string } + processType?: ProcessType + structuredOptions?: Partial + unstructuredOptions?: Partial + externalSource?: Partial +} + +interface DraftBindings { + currentStepId: Readonly> + task: Reactive<{ name: string; description: string }> + processType: Ref + structuredOptions: Ref + unstructuredOptions: Ref + externalSource: Reactive + restoringDraft: Ref + dirty: Ref + goToStep: (stepId: StepId) => void +} + +function sanitizeExternalSource(source: Partial) { + return { + type: typeof source.type === 'string' ? source.type : 'mysql', + url: typeof source.url === 'string' ? source.url : '', + authMode: typeof source.authMode === 'string' ? source.authMode : 'none', + username: typeof source.username === 'string' ? source.username : '', + limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000, + } +} + +function isDraftSnapshot(value: unknown): value is DraftSnapshot { + return Boolean(value && typeof value === 'object') +} + +/** + * 只持久化可重建的配置。密码、令牌、文件正文、预览与结果都不进入 localStorage。 + */ +export function useDataProcessDraft(bindings: DraftBindings) { + function draftSnapshot(): DraftSnapshot { + return { + schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION, + currentStepId: bindings.currentStepId.value, + task: { ...bindings.task }, + processType: bindings.processType.value, + structuredOptions: { + ...bindings.structuredOptions.value, + preprocessOptions: [...bindings.structuredOptions.value.preprocessOptions], + datasetSplit: { ...bindings.structuredOptions.value.datasetSplit }, + }, + unstructuredOptions: { + ...bindings.unstructuredOptions.value, + preprocessOptions: [...bindings.unstructuredOptions.value.preprocessOptions], + datasetSplit: { ...bindings.unstructuredOptions.value.datasetSplit }, + }, + externalSource: sanitizeExternalSource(bindings.externalSource), + } + } + + function writeDraft(showWarning: boolean) { + try { + localStorage.setItem(DATA_PROCESS_DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot())) + } catch { + if (showWarning) ElMessage.warning('草稿保存失败，请检查浏览器存储空间') + } + } + + function persistDraft() { + if (!bindings.restoringDraft.value) writeDraft(true) + } + + function restoreDraft() { + try { + const raw = localStorage.getItem(DATA_PROCESS_DRAFT_STORAGE_KEY) + if (!raw) return + const snapshot: unknown = JSON.parse(raw) + if (!isDraftSnapshot(snapshot)) return + + bindings.restoringDraft.value = true + bindings.goToStep('create') + bindings.task.name = snapshot.task?.name || '' + bindings.task.description = snapshot.task?.description || '' + bindings.processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external' + ? snapshot.processType + : 'unstructured' + + if (snapshot.structuredOptions) { + bindings.structuredOptions.value = { + ...bindings.structuredOptions.value, + ...snapshot.structuredOptions, + preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions) + ? snapshot.structuredOptions.preprocessOptions + : bindings.structuredOptions.value.preprocessOptions, + datasetSplit: { + ...bindings.structuredOptions.value.datasetSplit, + ...snapshot.structuredOptions.datasetSplit, + }, + } + } + + if (snapshot.unstructuredOptions) { + bindings.unstructuredOptions.value = { + ...bindings.unstructuredOptions.value, + ...snapshot.unstructuredOptions, + preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions) + ? snapshot.unstructuredOptions.preprocessOptions + : bindings.unstructuredOptions.value.preprocessOptions, + datasetSplit: { + ...bindings.unstructuredOptions.value.datasetSplit, + ...snapshot.unstructuredOptions.datasetSplit, + }, + } + } + + Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), { + password: '', + token: '', + }) + bindings.dirty.value = false + + nextTick(() => { + bindings.restoringDraft.value = false + // 立即覆盖 v5 及更早草稿，清除其中可能存在的敏感值和大段正文。 + writeDraft(false) + }) + ElMessage.info('已恢复上次的任务配置，请重新上传或拉取源数据') + } catch { + localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY) + } + } + + return { draftSnapshot, persistDraft, restoreDraft } +} diff --git a/frontend/src/views/data-process/create/useDataProcessGeneration.ts b/frontend/src/views/data-process/create/useDataProcessGeneration.ts new file mode 100644 index 0000000..763d835 --- /dev/null +++ b/frontend/src/views/data-process/create/useDataProcessGeneration.ts @@ -0,0 +1,127 @@ +import { reactive, ref, type Ref } from 'vue' +import { ElMessage } from 'element-plus' +import { createResults } from './previewModel' +import type { + GenerationState, + PreviewItem, + ProcessType, + ResultItem, + StructuredProcessOptions, + UnstructuredProcessOptions, +} from './types' + +interface GenerationBindings { + previewItems: Ref + processType: Ref + structuredOptions: Ref + unstructuredOptions: Ref + dirty: Ref +} + +export function useDataProcessGeneration(bindings: GenerationBindings) { + const results = ref([]) + const selectedResultId = ref(null) + const generation = reactive({ + status: 'idle', + progress: 0, + message: '确认摘要后即可开始生成，过程中可查看实时进度。', + }) + let generationTimer: ReturnType | null = null + + function stopGenerationTimer() { + if (generationTimer) clearInterval(generationTimer) + generationTimer = null + } + + function resetDownstream() { + stopGenerationTimer() + generation.status = 'idle' + generation.progress = 0 + generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。' + results.value = [] + selectedResultId.value = null + } + + function startGeneration() { + stopGenerationTimer() + generation.status = 'running' + generation.progress = 0 + generation.message = '正在应用预览修改并生成标准化结果，请稍候。' + + generationTimer = setInterval(() => { + generation.progress = Math.min(100, generation.progress + 8) + if (generation.progress < 100) return + + stopGenerationTimer() + generation.status = 'success' + results.value = createResults( + bindings.previewItems.value, + bindings.processType.value === 'structured' + ? bindings.structuredOptions.value + : bindings.processType.value === 'unstructured' + ? bindings.unstructuredOptions.value + : undefined, + ) + generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。` + selectedResultId.value = results.value[0]?.id ?? null + bindings.dirty.value = true + ElMessage.success('数据处理完成') + }, 180) + } + + function stopGeneration() { + stopGenerationTimer() + generation.status = 'failed' + generation.message = '任务已停止，预览修改仍然保留，可以重新生成。' + } + + function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) { + const item = results.value.find((entry) => entry.id === id) + if (!item) return + item[field] = value + const valid = item.instruction.trim() && item.output.trim() + item.error = valid ? undefined : 'Instruction 和 Output 不能为空' + const changed = item.instruction !== item.originalInstruction + || item.input !== item.originalInput + || item.output !== item.originalOutput + item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid' + bindings.dirty.value = true + } + + function restoreResult(id: string) { + const item = results.value.find((entry) => entry.id === id) + if (!item) return + item.instruction = item.originalInstruction + item.input = item.originalInput + item.output = item.originalOutput + item.error = undefined + item.status = 'valid' + bindings.dirty.value = true + } + + function validateResults() { + let firstInvalidId: string | null = null + for (const item of results.value) { + if (!item.instruction.trim() || !item.output.trim()) { + item.error = 'Instruction 和 Output 不能为空' + item.status = 'invalid' + firstInvalidId ??= item.id + } + } + if (firstInvalidId) selectedResultId.value = firstInvalidId + return firstInvalidId == null + } + + return { + generation, + results, + selectedResultId, + resetDownstream, + restoreResult, + startGeneration, + stopGeneration, + stopGenerationTimer, + updateResultField, + validateResults, + } +}