refactor: 数据处理向导拆分组合式函数与子面板
提取 useDataProcessDraft、useDataProcessGeneration 与 dataProcessCreateState 管理向导状态，新增 StructuredOptionsPanel、UnstructuredOptionsPanel、DatasetSplitEditor 子面板组件，样式抽离为独立 scss，DataProcessCreateView 与 TaskSetupStep 大幅瘦身，回归脚本适配。
This commit is contained in:
caoxiaozhu
@@ -9,17 +9,25 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
|
||||
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||
import GenerationStep from './create/GenerationStep.vue'
|
||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||
import { buildPreviewItems, createResults, DEFAULT_SOURCE_TEXT } from './create/previewModel'
|
||||
import { buildPreviewItems, DEFAULT_SOURCE_TEXT } from './create/previewModel'
|
||||
import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
} from './create/dataProcessCreateState'
|
||||
import {
|
||||
DATA_PROCESS_DRAFT_STORAGE_KEY,
|
||||
useDataProcessDraft,
|
||||
} from './create/useDataProcessDraft'
|
||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useModelsStore } from '@/stores/models'
|
||||
import type {
|
||||
ExternalDataSource,
|
||||
GenerationState,
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
StepId,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
UploadedDataFile,
|
||||
} from './create/types'
|
||||
|
||||
const router = useRouter()
|
||||
@@ -28,10 +36,7 @@ const { list: modelList } = storeToRefs(modelsStore)
|
||||
const generationModels = computed(() => modelList.value.filter((model) => model.type === 'LLM'))
|
||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||
const confirmDialogRef = ref<InstanceType<typeof AppConfirmDialog>>()
|
||||
const DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
||||
const DRAFT_SCHEMA_VERSION = 5
|
||||
const PREVIEW_MODEL_VERSION = 'document-chunk-v2'
|
||||
const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
|
||||
|
||||
const WIZARD_STEPS = [
|
||||
{ id: 'create', title: '创建任务', desc: '填写任务信息与处理配置' },
|
||||
@@ -40,65 +45,12 @@ const WIZARD_STEPS = [
|
||||
{ id: 'generate', title: '开始生成', desc: '确认摘要并启动处理' },
|
||||
{ id: 'results', title: '结果编辑与保存', desc: '检查、修改并保存结果' },
|
||||
] as const satisfies ReadonlyArray<{ id: StepId; title: string; desc: string }>
|
||||
const LEGACY_V3_STEP_IDS: ReadonlyArray<StepId> = ['create', 'preview', 'generate', 'results']
|
||||
|
||||
const currentStep = ref(0)
|
||||
const currentStepId = computed<StepId>(() => WIZARD_STEPS[currentStep.value]?.id ?? 'create')
|
||||
const task = reactive({ name: '', description: '' })
|
||||
const processType = ref<ProcessType>('unstructured')
|
||||
const structuredOptions = ref<StructuredProcessOptions>({
|
||||
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
})
|
||||
const unstructuredOptions = ref<UnstructuredProcessOptions>({
|
||||
preprocessOptions: [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
chunkMethod: 'semantic',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
customDelimiter: '',
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
})
|
||||
interface UploadedDataFile {
|
||||
uid: number | string
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
content: string
|
||||
}
|
||||
|
||||
const structuredOptions = ref<StructuredProcessOptions>(createDefaultStructuredOptions())
|
||||
const unstructuredOptions = ref<UnstructuredProcessOptions>(createDefaultUnstructuredOptions())
|
||||
const uploadedFiles = ref<UploadedDataFile[]>([])
|
||||
|
||||
const externalSource = reactive<ExternalDataSource>({
|
||||
@@ -112,35 +64,53 @@ const externalSource = reactive<ExternalDataSource>({
|
||||
})
|
||||
const externalPulling = ref(false)
|
||||
const externalConnected = ref(false)
|
||||
let connectionTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let pullTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const fileName = computed(() => uploadedFiles.value.map(f => f.name).join(', '))
|
||||
const fileSize = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.size, 0))
|
||||
const fileCount = computed(() => uploadedFiles.value.reduce((sum, f) => sum + f.count, 0))
|
||||
const previewSignature = ref('')
|
||||
const previewItems = ref<PreviewItem[]>([])
|
||||
const selectedPreviewFileId = ref<string | null>(null)
|
||||
const selectedPreviewId = ref<string | null>(null)
|
||||
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
|
||||
const results = ref<ResultItem[]>([])
|
||||
const selectedResultId = ref<string | null>(null)
|
||||
const dirty = ref(false)
|
||||
const restoringDraft = ref(false)
|
||||
let allowLeave = false
|
||||
|
||||
const generation = reactive<GenerationState>({
|
||||
status: 'idle',
|
||||
progress: 0,
|
||||
message: '确认摘要后即可开始生成，过程中可查看实时进度。',
|
||||
const {
|
||||
generation,
|
||||
results,
|
||||
selectedResultId,
|
||||
resetDownstream,
|
||||
restoreResult,
|
||||
startGeneration,
|
||||
stopGeneration,
|
||||
stopGenerationTimer,
|
||||
updateResultField,
|
||||
validateResults,
|
||||
} = useDataProcessGeneration({
|
||||
previewItems,
|
||||
processType,
|
||||
structuredOptions,
|
||||
unstructuredOptions,
|
||||
dirty,
|
||||
})
|
||||
|
||||
let generationTimer: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
const modifiedPreviewCount = computed(() => previewItems.value.filter((item) => item.status !== 'original').length)
|
||||
const previewItemsByFile = computed(() => {
|
||||
const grouped = new Map<string, PreviewItem[]>()
|
||||
for (const item of previewItems.value) {
|
||||
const items = grouped.get(item.sourceFileId) || []
|
||||
items.push(item)
|
||||
grouped.set(item.sourceFileId, items)
|
||||
}
|
||||
return grouped
|
||||
})
|
||||
const activePreviewFile = computed(() => uploadedFiles.value.find((file) => String(file.uid) === selectedPreviewFileId.value))
|
||||
const activePreviewItems = computed(() => previewItems.value.filter((item) => item.sourceFileId === selectedPreviewFileId.value))
|
||||
const activePreviewItems = computed(() => previewItemsByFile.value.get(selectedPreviewFileId.value || '') || [])
|
||||
const activeSourceText = computed(() => activePreviewFile.value?.content ?? '')
|
||||
const previewFiles = computed(() => uploadedFiles.value.map((file) => {
|
||||
const items = previewItems.value.filter((item) => item.sourceFileId === String(file.uid))
|
||||
const items = previewItemsByFile.value.get(String(file.uid)) || []
|
||||
return {
|
||||
id: String(file.uid),
|
||||
name: file.name,
|
||||
@@ -169,44 +139,22 @@ const previousStepLabel = computed(() => currentStep.value > 0
|
||||
? WIZARD_STEPS[currentStep.value - 1].title
|
||||
: '')
|
||||
|
||||
function isStepId(value: unknown): value is StepId {
|
||||
return typeof value === 'string' && WIZARD_STEPS.some((step) => step.id === value)
|
||||
}
|
||||
|
||||
function goToStep(stepId: StepId) {
|
||||
const nextStepIndex = WIZARD_STEPS.findIndex((step) => step.id === stepId)
|
||||
if (nextStepIndex >= 0) currentStep.value = nextStepIndex
|
||||
}
|
||||
|
||||
function draftSnapshot() {
|
||||
return {
|
||||
schemaVersion: DRAFT_SCHEMA_VERSION,
|
||||
currentStep: currentStep.value,
|
||||
currentStepId: currentStepId.value,
|
||||
task: { ...task },
|
||||
processType: processType.value,
|
||||
structuredOptions: {
|
||||
...structuredOptions.value,
|
||||
preprocessOptions: [...structuredOptions.value.preprocessOptions],
|
||||
datasetSplit: { ...structuredOptions.value.datasetSplit },
|
||||
},
|
||||
unstructuredOptions: {
|
||||
...unstructuredOptions.value,
|
||||
preprocessOptions: [...unstructuredOptions.value.preprocessOptions],
|
||||
datasetSplit: { ...unstructuredOptions.value.datasetSplit },
|
||||
},
|
||||
uploadedFiles: uploadedFiles.value,
|
||||
externalSource: { ...externalSource },
|
||||
previewSignature: previewSignature.value,
|
||||
previewItems: previewItems.value,
|
||||
selectedPreviewFileId: selectedPreviewFileId.value,
|
||||
selectedPreviewId: selectedPreviewId.value,
|
||||
selectedPreviewIdsByFile: selectedPreviewIdsByFile.value,
|
||||
results: results.value,
|
||||
selectedResultId: selectedResultId.value,
|
||||
generation: { ...generation },
|
||||
}
|
||||
}
|
||||
const { persistDraft, restoreDraft } = useDataProcessDraft({
|
||||
currentStepId,
|
||||
task,
|
||||
processType,
|
||||
structuredOptions,
|
||||
unstructuredOptions,
|
||||
externalSource,
|
||||
restoringDraft,
|
||||
dirty,
|
||||
goToStep,
|
||||
})
|
||||
|
||||
function previewAffectingOptions() {
|
||||
if (processType.value === 'structured') {
|
||||
@@ -291,191 +239,11 @@ const generationOptionsSignature = computed(() => JSON.stringify(generationAffec
|
||||
|
||||
function buildPreviewSignature() {
|
||||
const filesSignature = uploadedFiles.value
|
||||
.map((file) => `${file.uid}:${file.content}`)
|
||||
.map((file) => `${file.uid}:${file.name}:${file.size}:${file.count}`)
|
||||
.join('|')
|
||||
return `${PREVIEW_MODEL_VERSION}:${processType.value}:${JSON.stringify(previewAffectingOptions())}:${filesSignature}`
|
||||
}
|
||||
|
||||
function persistDraft() {
|
||||
if (restoringDraft.value) return
|
||||
try {
|
||||
localStorage.setItem(DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
|
||||
} catch {
|
||||
ElMessage.warning('草稿保存失败，请检查浏览器存储空间')
|
||||
}
|
||||
}
|
||||
|
||||
type DraftSnapshot = Partial<ReturnType<typeof draftSnapshot>> & {
|
||||
fileName?: string
|
||||
fileSize?: number
|
||||
fileCount?: number
|
||||
sourceText?: string
|
||||
}
|
||||
|
||||
function restoreDraft() {
|
||||
try {
|
||||
const raw = localStorage.getItem(DRAFT_STORAGE_KEY)
|
||||
if (!raw) return
|
||||
const snapshot = JSON.parse(raw) as DraftSnapshot
|
||||
if (!snapshot?.uploadedFiles && (!snapshot?.fileName || !snapshot?.sourceText)) return
|
||||
const snapshotSchemaVersion = Number(snapshot.schemaVersion) || 0
|
||||
const requiresPreviewMigration = snapshotSchemaVersion < 3
|
||||
const legacyStepId = snapshotSchemaVersion === 3
|
||||
? LEGACY_V3_STEP_IDS[Math.min(Math.max(Number(snapshot.currentStep) || 0, 0), LEGACY_V3_STEP_IDS.length - 1)]
|
||||
: undefined
|
||||
const indexedStepId = WIZARD_STEPS[Math.min(
|
||||
Math.max(Number(snapshot.currentStep) || 0, 0),
|
||||
WIZARD_STEPS.length - 1,
|
||||
)]?.id
|
||||
const restoredStepId = isStepId(snapshot.currentStepId)
|
||||
? snapshot.currentStepId
|
||||
: legacyStepId || indexedStepId || 'create'
|
||||
|
||||
restoringDraft.value = true
|
||||
goToStep(requiresPreviewMigration ? 'create' : restoredStepId)
|
||||
task.name = snapshot.task?.name || ''
|
||||
task.description = snapshot.task?.description || ''
|
||||
processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
|
||||
? snapshot.processType
|
||||
: 'unstructured'
|
||||
if (snapshot.structuredOptions) {
|
||||
structuredOptions.value = {
|
||||
...structuredOptions.value,
|
||||
...snapshot.structuredOptions,
|
||||
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
|
||||
? snapshot.structuredOptions.preprocessOptions
|
||||
: structuredOptions.value.preprocessOptions,
|
||||
datasetSplit: {
|
||||
...structuredOptions.value.datasetSplit,
|
||||
...snapshot.structuredOptions.datasetSplit,
|
||||
},
|
||||
}
|
||||
}
|
||||
if (snapshot.unstructuredOptions) {
|
||||
const restoredUnstructuredOptions = snapshot.unstructuredOptions
|
||||
const restoredDatasetSplit = restoredUnstructuredOptions.datasetSplit
|
||||
unstructuredOptions.value = {
|
||||
...unstructuredOptions.value,
|
||||
preprocessOptions: Array.isArray(restoredUnstructuredOptions.preprocessOptions)
|
||||
? restoredUnstructuredOptions.preprocessOptions
|
||||
: unstructuredOptions.value.preprocessOptions,
|
||||
chunkMethod: restoredUnstructuredOptions.chunkMethod || unstructuredOptions.value.chunkMethod,
|
||||
chunkSize: Number.isFinite(restoredUnstructuredOptions.chunkSize)
|
||||
? restoredUnstructuredOptions.chunkSize
|
||||
: unstructuredOptions.value.chunkSize,
|
||||
chunkOverlap: Number.isFinite(restoredUnstructuredOptions.chunkOverlap)
|
||||
? restoredUnstructuredOptions.chunkOverlap
|
||||
: unstructuredOptions.value.chunkOverlap,
|
||||
minChunkSize: Number.isFinite(restoredUnstructuredOptions.minChunkSize)
|
||||
? restoredUnstructuredOptions.minChunkSize
|
||||
: unstructuredOptions.value.minChunkSize,
|
||||
customDelimiter: typeof restoredUnstructuredOptions.customDelimiter === 'string'
|
||||
? restoredUnstructuredOptions.customDelimiter
|
||||
: unstructuredOptions.value.customDelimiter,
|
||||
preserveTables: typeof restoredUnstructuredOptions.preserveTables === 'boolean'
|
||||
? restoredUnstructuredOptions.preserveTables
|
||||
: unstructuredOptions.value.preserveTables,
|
||||
preserveCodeBlocks: typeof restoredUnstructuredOptions.preserveCodeBlocks === 'boolean'
|
||||
? restoredUnstructuredOptions.preserveCodeBlocks
|
||||
: unstructuredOptions.value.preserveCodeBlocks,
|
||||
preserveLists: typeof restoredUnstructuredOptions.preserveLists === 'boolean'
|
||||
? restoredUnstructuredOptions.preserveLists
|
||||
: unstructuredOptions.value.preserveLists,
|
||||
semanticEnrichment: typeof restoredUnstructuredOptions.semanticEnrichment === 'boolean'
|
||||
? restoredUnstructuredOptions.semanticEnrichment
|
||||
: unstructuredOptions.value.semanticEnrichment,
|
||||
qaPairsPerChunk: Number.isFinite(restoredUnstructuredOptions.qaPairsPerChunk)
|
||||
? restoredUnstructuredOptions.qaPairsPerChunk
|
||||
: unstructuredOptions.value.qaPairsPerChunk,
|
||||
generationModelId: restoredUnstructuredOptions.generationModelId ?? unstructuredOptions.value.generationModelId,
|
||||
generationPrompt: typeof restoredUnstructuredOptions.generationPrompt === 'string'
|
||||
? restoredUnstructuredOptions.generationPrompt
|
||||
: unstructuredOptions.value.generationPrompt,
|
||||
temperature: Number.isFinite(restoredUnstructuredOptions.temperature)
|
||||
? restoredUnstructuredOptions.temperature
|
||||
: unstructuredOptions.value.temperature,
|
||||
maxTokens: Number.isFinite(restoredUnstructuredOptions.maxTokens)
|
||||
? restoredUnstructuredOptions.maxTokens
|
||||
: unstructuredOptions.value.maxTokens,
|
||||
jsonMode: typeof restoredUnstructuredOptions.jsonMode === 'boolean'
|
||||
? restoredUnstructuredOptions.jsonMode
|
||||
: unstructuredOptions.value.jsonMode,
|
||||
qualityFilterEnabled: typeof restoredUnstructuredOptions.qualityFilterEnabled === 'boolean'
|
||||
? restoredUnstructuredOptions.qualityFilterEnabled
|
||||
: unstructuredOptions.value.qualityFilterEnabled,
|
||||
filterLowQuality: typeof restoredUnstructuredOptions.filterLowQuality === 'boolean'
|
||||
? restoredUnstructuredOptions.filterLowQuality
|
||||
: unstructuredOptions.value.filterLowQuality,
|
||||
filterShortContent: typeof restoredUnstructuredOptions.filterShortContent === 'boolean'
|
||||
? restoredUnstructuredOptions.filterShortContent
|
||||
: unstructuredOptions.value.filterShortContent,
|
||||
minOutputLength: Number.isFinite(restoredUnstructuredOptions.minOutputLength)
|
||||
? restoredUnstructuredOptions.minOutputLength
|
||||
: unstructuredOptions.value.minOutputLength,
|
||||
datasetSplit: {
|
||||
train: Number.isFinite(restoredDatasetSplit?.train)
|
||||
? restoredDatasetSplit.train
|
||||
: unstructuredOptions.value.datasetSplit.train,
|
||||
validation: Number.isFinite(restoredDatasetSplit?.validation)
|
||||
? restoredDatasetSplit.validation
|
||||
: unstructuredOptions.value.datasetSplit.validation,
|
||||
test: Number.isFinite(restoredDatasetSplit?.test)
|
||||
? restoredDatasetSplit.test
|
||||
: unstructuredOptions.value.datasetSplit.test,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshot.uploadedFiles) {
|
||||
uploadedFiles.value = Array.isArray(snapshot.uploadedFiles) ? snapshot.uploadedFiles : []
|
||||
} else if (snapshot.fileName && snapshot.sourceText) {
|
||||
// Migrate old draft
|
||||
uploadedFiles.value = [{
|
||||
uid: 'migrated-draft',
|
||||
name: snapshot.fileName,
|
||||
size: Number(snapshot.fileSize) || 0,
|
||||
count: Number(snapshot.fileCount) || 0,
|
||||
content: snapshot.sourceText
|
||||
}]
|
||||
}
|
||||
|
||||
previewSignature.value = requiresPreviewMigration ? '' : snapshot.previewSignature || ''
|
||||
if (snapshot.externalSource) {
|
||||
Object.assign(externalSource, snapshot.externalSource)
|
||||
}
|
||||
const defaultSourceFileId = String(uploadedFiles.value[0]?.uid ?? '')
|
||||
previewItems.value = !requiresPreviewMigration && Array.isArray(snapshot.previewItems)
|
||||
? snapshot.previewItems.map((item) => ({ ...item, sourceFileId: item.sourceFileId ?? defaultSourceFileId }))
|
||||
: []
|
||||
selectedPreviewFileId.value = snapshot.selectedPreviewFileId || defaultSourceFileId || null
|
||||
selectedPreviewIdsByFile.value = requiresPreviewMigration ? {} : snapshot.selectedPreviewIdsByFile || {}
|
||||
selectedPreviewId.value = requiresPreviewMigration
|
||||
? null
|
||||
: snapshot.selectedPreviewId
|
||||
|| selectedPreviewIdsByFile.value[selectedPreviewFileId.value ?? '']
|
||||
|| activePreviewItems.value[0]?.id
|
||||
|| null
|
||||
results.value = !requiresPreviewMigration && Array.isArray(snapshot.results) ? snapshot.results : []
|
||||
selectedResultId.value = requiresPreviewMigration ? null : snapshot.selectedResultId || results.value[0]?.id || null
|
||||
if (!requiresPreviewMigration) {
|
||||
const restoredGeneration = snapshot.generation || {}
|
||||
Object.assign(generation, restoredGeneration)
|
||||
if (generation.status === 'running') {
|
||||
generation.status = 'idle'
|
||||
generation.progress = 0
|
||||
generation.message = '草稿已恢复，请重新开始生成。'
|
||||
}
|
||||
}
|
||||
dirty.value = false
|
||||
nextTick(() => { restoringDraft.value = false })
|
||||
ElMessage.info(requiresPreviewMigration
|
||||
? '已恢复任务信息，切分规则已更新，请重新生成预览'
|
||||
: '已恢复上次保存的草稿')
|
||||
} catch {
|
||||
localStorage.removeItem(DRAFT_STORAGE_KEY)
|
||||
}
|
||||
}
|
||||
|
||||
watch(
|
||||
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
() => {
|
||||
@@ -496,9 +264,7 @@ watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||
})
|
||||
|
||||
watch(
|
||||
[currentStep, task, processType, structuredOptions, unstructuredOptions, uploadedFiles, externalSource, previewSignature,
|
||||
previewItems, selectedPreviewFileId, selectedPreviewId, selectedPreviewIdsByFile,
|
||||
results, selectedResultId, generation],
|
||||
[task, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
persistDraft,
|
||||
{ deep: true },
|
||||
)
|
||||
@@ -566,8 +332,10 @@ function handleTestConnection() {
|
||||
ElMessage.warning('请先填写数据源地址')
|
||||
return
|
||||
}
|
||||
if (connectionTimer) clearTimeout(connectionTimer)
|
||||
externalPulling.value = true
|
||||
setTimeout(() => {
|
||||
connectionTimer = setTimeout(() => {
|
||||
connectionTimer = null
|
||||
externalPulling.value = false
|
||||
externalConnected.value = true
|
||||
ElMessage.success('数据源连接测试成功')
|
||||
@@ -579,8 +347,10 @@ function handlePullData() {
|
||||
ElMessage.warning('请先填写数据源地址')
|
||||
return
|
||||
}
|
||||
if (pullTimer) clearTimeout(pullTimer)
|
||||
externalPulling.value = true
|
||||
setTimeout(() => {
|
||||
pullTimer = setTimeout(() => {
|
||||
pullTimer = null
|
||||
externalPulling.value = false
|
||||
externalConnected.value = true
|
||||
const typeName = externalSource.type.toUpperCase()
|
||||
@@ -620,15 +390,6 @@ function resetSourceDataForProcessTypeChange() {
|
||||
resetDownstream()
|
||||
}
|
||||
|
||||
function resetDownstream() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'idle'
|
||||
generation.progress = 0
|
||||
generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。'
|
||||
results.value = []
|
||||
selectedResultId.value = null
|
||||
}
|
||||
|
||||
async function nextFromCreate() {
|
||||
const valid = await taskSetupRef.value?.validate()
|
||||
if (!valid) return
|
||||
@@ -738,81 +499,6 @@ async function removePreviewItem(id: string) {
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
function stopGenerationTimer() {
|
||||
if (generationTimer) clearInterval(generationTimer)
|
||||
generationTimer = null
|
||||
}
|
||||
|
||||
function startGeneration() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'running'
|
||||
generation.progress = 0
|
||||
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
|
||||
|
||||
generationTimer = setInterval(() => {
|
||||
generation.progress = Math.min(100, generation.progress + 8)
|
||||
if (generation.progress < 100) return
|
||||
|
||||
stopGenerationTimer()
|
||||
generation.status = 'success'
|
||||
results.value = createResults(
|
||||
previewItems.value,
|
||||
processType.value === 'structured'
|
||||
? structuredOptions.value
|
||||
: processType.value === 'unstructured'
|
||||
? unstructuredOptions.value
|
||||
: undefined,
|
||||
)
|
||||
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
|
||||
selectedResultId.value = results.value[0]?.id ?? null
|
||||
dirty.value = true
|
||||
ElMessage.success('数据处理完成')
|
||||
}, 180)
|
||||
}
|
||||
|
||||
function stopGeneration() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'failed'
|
||||
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
|
||||
}
|
||||
|
||||
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item[field] = value
|
||||
const valid = item.instruction.trim() && item.output.trim()
|
||||
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
|
||||
const changed = item.instruction !== item.originalInstruction
|
||||
|| item.input !== item.originalInput
|
||||
|| item.output !== item.originalOutput
|
||||
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
function restoreResult(id: string) {
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item.instruction = item.originalInstruction
|
||||
item.input = item.originalInput
|
||||
item.output = item.originalOutput
|
||||
item.error = undefined
|
||||
item.status = 'valid'
|
||||
dirty.value = true
|
||||
}
|
||||
|
||||
function validateResults() {
|
||||
let firstInvalidId: string | null = null
|
||||
for (const item of results.value) {
|
||||
if (!item.instruction.trim() || !item.output.trim()) {
|
||||
item.error = 'Instruction 和 Output 不能为空'
|
||||
item.status = 'invalid'
|
||||
firstInvalidId ??= item.id
|
||||
}
|
||||
}
|
||||
if (firstInvalidId) selectedResultId.value = firstInvalidId
|
||||
return firstInvalidId == null
|
||||
}
|
||||
|
||||
async function handlePrimaryAction() {
|
||||
if (currentStepId.value === 'create') {
|
||||
await nextFromCreate()
|
||||
@@ -856,7 +542,7 @@ async function saveTask() {
|
||||
return
|
||||
}
|
||||
dirty.value = false
|
||||
localStorage.removeItem(DRAFT_STORAGE_KEY)
|
||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
allowLeave = true
|
||||
ElMessage.success('数据处理任务已保存')
|
||||
await router.push('/data-process')
|
||||
@@ -893,7 +579,11 @@ onBeforeRouteLeave(async () => {
|
||||
return Boolean(confirmed)
|
||||
})
|
||||
|
||||
onBeforeUnmount(stopGenerationTimer)
|
||||
onBeforeUnmount(() => {
|
||||
stopGenerationTimer()
|
||||
if (connectionTimer) clearTimeout(connectionTimer)
|
||||
if (pullTimer) clearTimeout(pullTimer)
|
||||
})
|
||||
onMounted(() => {
|
||||
restoreDraft()
|
||||
modelsStore.load()
|
||||
@@ -1023,198 +713,4 @@ onMounted(() => {
|
||||
<AppConfirmDialog ref="confirmDialogRef" />
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.create-wizard-layout {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
border: 1px solid #eef0f5;
|
||||
border-radius: 8px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.wizard-main {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 32px;
|
||||
|
||||
/* 自定义滚动条 */
|
||||
&::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
&::-webkit-scrollbar-thumb {
|
||||
background: #cbd5e1;
|
||||
border-radius: 3px;
|
||||
}
|
||||
}
|
||||
|
||||
.wizard-main-inner {
|
||||
min-height: 100%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.wizard-steps-container {
|
||||
margin-bottom: 32px;
|
||||
padding-bottom: 24px;
|
||||
border-bottom: 1px dashed #e2e8f0;
|
||||
}
|
||||
|
||||
.custom-wizard-steps {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
max-width: 860px;
|
||||
margin: 0 auto;
|
||||
padding: 0 20px;
|
||||
}
|
||||
|
||||
.step-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex: 1;
|
||||
|
||||
&:first-child {
|
||||
flex: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.step-connector {
|
||||
flex: 1;
|
||||
height: 2px;
|
||||
background-color: #e2e8f0;
|
||||
margin: 0 16px;
|
||||
transition: background-color 0.3s;
|
||||
}
|
||||
|
||||
.step-item.is-completed .step-connector,
|
||||
.step-item.is-active .step-connector {
|
||||
background-color: #5146e5;
|
||||
}
|
||||
|
||||
.step-node {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.step-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
border-radius: 50%;
|
||||
border: 2px solid #cbd5e1;
|
||||
background-color: #fff;
|
||||
color: #64748b;
|
||||
font-size: 13px;
|
||||
font-weight: 650;
|
||||
transition: all 0.3s;
|
||||
}
|
||||
|
||||
.step-title {
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
color: #64748b;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
/* State: Active */
|
||||
.step-item.is-active {
|
||||
.step-icon {
|
||||
border-color: #5146e5;
|
||||
background-color: #eef2ff;
|
||||
color: #5146e5;
|
||||
}
|
||||
.step-title {
|
||||
color: #1e293b;
|
||||
}
|
||||
}
|
||||
|
||||
/* State: Completed */
|
||||
.step-item.is-completed {
|
||||
.step-icon {
|
||||
background-color: #5146e5;
|
||||
border-color: #5146e5;
|
||||
color: #fff;
|
||||
}
|
||||
.step-title {
|
||||
color: #1e293b;
|
||||
}
|
||||
}
|
||||
|
||||
.wizard-content {
|
||||
min-height: 400px;
|
||||
}
|
||||
|
||||
.wizard-footer {
|
||||
flex-shrink: 0;
|
||||
height: 64px;
|
||||
background: #ffffff;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
display: grid;
|
||||
grid-template-columns: 1fr auto 1fr;
|
||||
align-items: center;
|
||||
padding: 0 32px;
|
||||
box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.02);
|
||||
|
||||
.footer-left {
|
||||
justify-self: start;
|
||||
}
|
||||
|
||||
.footer-center {
|
||||
justify-self: center;
|
||||
}
|
||||
|
||||
.footer-right {
|
||||
justify-self: end;
|
||||
}
|
||||
}
|
||||
|
||||
.wizard-primary-action {
|
||||
min-width: 160px;
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.wizard-main {
|
||||
padding: 24px 20px;
|
||||
}
|
||||
|
||||
.custom-wizard-steps {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.step-connector {
|
||||
margin: 0 8px;
|
||||
}
|
||||
|
||||
.step-node {
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.step-title {
|
||||
max-width: 72px;
|
||||
font-size: 12px;
|
||||
line-height: 1.35;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.wizard-footer {
|
||||
padding: 0 20px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.step-title {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.step-connector {
|
||||
margin: 0 6px;
|
||||
}
|
||||
|
||||
.wizard-primary-action {
|
||||
min-width: 0;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
<style scoped lang="scss" src="./create/data-process-create.scss"></style>
|
||||
|
||||
185
frontend/src/views/data-process/create/DatasetSplitEditor.vue
Normal file
185
frontend/src/views/data-process/create/DatasetSplitEditor.vue
Normal file
@@ -0,0 +1,185 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
import type { DatasetSplitOptions } from './types'
|
||||
|
||||
const props = withDefaults(defineProps<{
|
||||
modelValue: DatasetSplitOptions
|
||||
ariaLabelPrefix?: string
|
||||
}>(), {
|
||||
ariaLabelPrefix: '',
|
||||
})
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:modelValue': [value: DatasetSplitOptions]
|
||||
}>()
|
||||
|
||||
const splitTotal = computed(() => {
|
||||
const { train, validation, test } = props.modelValue
|
||||
return train + validation + test
|
||||
})
|
||||
|
||||
function updateField(field: keyof DatasetSplitOptions, value: number | undefined) {
|
||||
emit('update:modelValue', {
|
||||
...props.modelValue,
|
||||
[field]: Math.min(100, Math.max(0, Number(value) || 0)),
|
||||
})
|
||||
}
|
||||
|
||||
function ariaLabel(label: string) {
|
||||
return `${props.ariaLabelPrefix}${label}比例`
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="generation-option-row dataset-split-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>数据集划分</strong>
|
||||
<small>按比例将生成后的问答对随机划分为训练集、验证集和测试集</small>
|
||||
</div>
|
||||
<div class="dataset-split-config">
|
||||
<div class="dataset-split-grid">
|
||||
<label class="dataset-split-field">
|
||||
<span>训练集</span>
|
||||
<span class="dataset-split-input">
|
||||
<el-input-number
|
||||
:model-value="modelValue.train"
|
||||
:min="0"
|
||||
:max="100"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
:aria-label="ariaLabel('训练集')"
|
||||
@update:model-value="updateField('train', $event)"
|
||||
/>
|
||||
<span>%</span>
|
||||
</span>
|
||||
</label>
|
||||
<label class="dataset-split-field">
|
||||
<span>验证集</span>
|
||||
<span class="dataset-split-input">
|
||||
<el-input-number
|
||||
:model-value="modelValue.validation"
|
||||
:min="0"
|
||||
:max="100"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
:aria-label="ariaLabel('验证集')"
|
||||
@update:model-value="updateField('validation', $event)"
|
||||
/>
|
||||
<span>%</span>
|
||||
</span>
|
||||
</label>
|
||||
<label class="dataset-split-field">
|
||||
<span>测试集</span>
|
||||
<span class="dataset-split-input">
|
||||
<el-input-number
|
||||
:model-value="modelValue.test"
|
||||
:min="0"
|
||||
:max="100"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
:aria-label="ariaLabel('测试集')"
|
||||
@update:model-value="updateField('test', $event)"
|
||||
/>
|
||||
<span>%</span>
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
<div class="dataset-split-summary" :class="{ 'is-invalid': splitTotal !== 100 }">
|
||||
<span>总计 {{ splitTotal }}%</span>
|
||||
<span v-if="splitTotal !== 100" role="alert">
|
||||
训练集、验证集和测试集比例总和必须为 100%
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.generation-option-row {
|
||||
display: flex;
|
||||
min-height: 64px;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 24px;
|
||||
padding: 12px 14px;
|
||||
box-sizing: border-box;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.generation-option-copy {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.dataset-split-row {
|
||||
align-items: stretch;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.dataset-split-config {
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.dataset-split-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.dataset-split-field {
|
||||
display: grid;
|
||||
gap: 7px;
|
||||
color: #5f6878;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.dataset-split-input {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
|
||||
:deep(.el-input-number) {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.dataset-split-summary {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
color: #2ca66a;
|
||||
font-size: 12px;
|
||||
|
||||
&.is-invalid {
|
||||
color: #dc2626;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.dataset-split-grid {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
|
||||
.dataset-split-summary {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
@@ -95,7 +95,7 @@ function sectionValidationMessage() {
|
||||
@update:model-value="updateField('generationPrompt', $event)"
|
||||
/>
|
||||
<div class="prompt-variables-hint">
|
||||
提示：可在文本中通过 <code>{{ content }}</code> 引用当前正在处理的数据内容。
|
||||
提示：可在文本中通过 <code v-text="'{{ content }}'" /> 引用当前正在处理的数据内容。
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import type { UploadFile } from 'element-plus'
|
||||
import type { ExternalDataSource, ProcessType } from './types'
|
||||
|
||||
interface UploadedSourceFile {
|
||||
uid: string | number
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
}
|
||||
import type { ExternalDataSource, ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
processType: ProcessType
|
||||
uploadedFiles: UploadedSourceFile[]
|
||||
uploadedFiles: UploadedDataFile[]
|
||||
externalSource: ExternalDataSource
|
||||
externalPulling: boolean
|
||||
externalConnected: boolean
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
<script setup lang="ts">
|
||||
import type { ModelItem } from '@/types'
|
||||
import type {
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
StructuredProcessOptions,
|
||||
} from './types'
|
||||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
options: StructuredProcessOptions
|
||||
generationModels: ModelItem[]
|
||||
validationAttempted: boolean
|
||||
generationValidationMessage: string
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:options': [value: StructuredProcessOptions]
|
||||
}>()
|
||||
|
||||
const PREPROCESS_OPTIONS: Array<{
|
||||
value: PreprocessOption
|
||||
label: string
|
||||
description: string
|
||||
}> = [
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '处理空行、空列和残缺行' },
|
||||
{ value: 'detect_structure', label: '识别表格结构', description: '识别表头、多级表头和合并单元格' },
|
||||
{ value: 'deduplicate', label: '重复数据去重', description: '删除完全重复或关键字段重复的数据' },
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '统一日期、数字、单位和枚举值格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '过滤乱码、无效内容和异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '处理姓名、手机号、邮箱等敏感信息' },
|
||||
]
|
||||
|
||||
function updateField<K extends keyof StructuredProcessOptions>(
|
||||
field: K,
|
||||
value: StructuredProcessOptions[K],
|
||||
) {
|
||||
emit('update:options', { ...props.options, [field]: value })
|
||||
}
|
||||
|
||||
function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
emit('update:options', { ...props.options, ...value })
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = value.filter(
|
||||
(option): option is PreprocessOption => typeof option === 'string' && allowedValues.has(option as PreprocessOption),
|
||||
)
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="form-section structured-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>选择在生成问答对之前需要执行的数据处理方式</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-checkbox-group
|
||||
:model-value="options.preprocessOptions"
|
||||
class="preprocess-option-grid"
|
||||
@update:model-value="updatePreprocessOptions"
|
||||
>
|
||||
<el-checkbox
|
||||
v-for="option in PREPROCESS_OPTIONS"
|
||||
:key="option.value"
|
||||
:value="option.value"
|
||||
class="preprocess-option"
|
||||
>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>{{ option.label }}</strong>
|
||||
<small>{{ option.description }}</small>
|
||||
</span>
|
||||
</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</div>
|
||||
|
||||
<div class="form-section generation-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>生成选项</h3>
|
||||
<p>配置每条结构化记录生成问答对的方式和数量</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="options"
|
||||
:models="generationModels"
|
||||
section="quality"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateGenerationOptions"
|
||||
/>
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>语义丰富表达</strong>
|
||||
<small>使用大模型将问答表述得更自然、柔和</small>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="options.semanticEnrichment"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
@update:model-value="updateField('semanticEnrichment', Boolean($event))"
|
||||
/>
|
||||
</div>
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>每行生成数量</strong>
|
||||
<small>每行结构化数据生成的问答对数量</small>
|
||||
</div>
|
||||
<el-input-number
|
||||
:model-value="options.qaPairsPerRow"
|
||||
:min="1"
|
||||
:max="5"
|
||||
:step="1"
|
||||
controls-position="right"
|
||||
@update:model-value="updateField('qaPairsPerRow', Number($event) || 1)"
|
||||
/>
|
||||
</div>
|
||||
<DatasetSplitEditor
|
||||
:model-value="options.datasetSplit"
|
||||
@update:model-value="updateField('datasetSplit', $event)"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section model-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>大模型</h3>
|
||||
<p>选择数据生成模型，并设置模型输出内容的要求</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="options"
|
||||
:models="generationModels"
|
||||
section="model"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateGenerationOptions"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.form-section {
|
||||
padding: 0 0 26px;
|
||||
margin-bottom: 26px;
|
||||
border-bottom: 1px solid #edf0f5;
|
||||
|
||||
&:last-child {
|
||||
margin-bottom: 0;
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin: 0 0 16px;
|
||||
color: #2f3747;
|
||||
font-size: 15px;
|
||||
font-weight: 650;
|
||||
}
|
||||
}
|
||||
|
||||
.section-title-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
|
||||
h3 {
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.preprocess-option-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.preprocess-option {
|
||||
width: 100%;
|
||||
min-height: 64px;
|
||||
margin-right: 0;
|
||||
padding: 12px 14px;
|
||||
box-sizing: border-box;
|
||||
align-items: flex-start;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
transition: border-color 0.18s ease, background-color 0.18s ease;
|
||||
|
||||
&:hover {
|
||||
border-color: #b7b2f7;
|
||||
}
|
||||
|
||||
&.is-checked {
|
||||
background: #fafaff;
|
||||
border-color: #8b82f4;
|
||||
}
|
||||
|
||||
:deep(.el-checkbox__input) {
|
||||
margin-top: 3px;
|
||||
}
|
||||
|
||||
:deep(.el-checkbox__label) {
|
||||
min-width: 0;
|
||||
padding-left: 10px;
|
||||
white-space: normal;
|
||||
}
|
||||
}
|
||||
|
||||
.preprocess-option-copy,
|
||||
.generation-option-copy {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.generation-option-list {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.generation-option-row {
|
||||
display: flex;
|
||||
min-height: 64px;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 24px;
|
||||
padding: 12px 14px;
|
||||
box-sizing: border-box;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.preprocess-option-grid {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
|
||||
.generation-option-row {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,604 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import type { ModelItem } from '@/types'
|
||||
import type {
|
||||
ChunkMethod,
|
||||
GenerationControlOptions,
|
||||
UnstructuredPreprocessOption,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
import DatasetSplitEditor from './DatasetSplitEditor.vue'
|
||||
import GenerationOptionsPanel from './GenerationOptionsPanel.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
options: UnstructuredProcessOptions
|
||||
generationModels: ModelItem[]
|
||||
validationAttempted: boolean
|
||||
generationValidationMessage: string
|
||||
chunkValidationMessage: string
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'update:options': [value: UnstructuredProcessOptions]
|
||||
}>()
|
||||
|
||||
const SMART_PREPROCESS_OPTIONS: UnstructuredPreprocessOption[] = [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
]
|
||||
|
||||
const CHUNK_METHODS: Array<{ value: ChunkMethod; label: string }> = [
|
||||
{ value: 'semantic', label: '自动语义切分' },
|
||||
{ value: 'heading', label: '按标题和段落' },
|
||||
{ value: 'fixed', label: '按固定长度' },
|
||||
{ value: 'custom', label: '自定义分隔符' },
|
||||
]
|
||||
|
||||
const UNSTRUCTURED_NUMBER_LIMITS = {
|
||||
chunkSize: { min: 200, max: 2000 },
|
||||
chunkOverlap: { min: 0, max: 500 },
|
||||
minChunkSize: { min: 20, max: 500 },
|
||||
qaPairsPerChunk: { min: 1, max: 3 },
|
||||
} as const
|
||||
|
||||
type UnstructuredNumberField = keyof typeof UNSTRUCTURED_NUMBER_LIMITS
|
||||
|
||||
const advancedChunkSettingsOpen = ref(props.options.chunkMethod === 'custom')
|
||||
|
||||
const smartPreprocessEnabled = computed(() => SMART_PREPROCESS_OPTIONS.every(
|
||||
(option) => props.options.preprocessOptions.includes(option),
|
||||
))
|
||||
|
||||
const desensitizeEnabled = computed(() => props.options.preprocessOptions.includes('desensitize'))
|
||||
|
||||
const preserveSpecialContentEnabled = computed(() => (
|
||||
props.options.preserveTables
|
||||
&& props.options.preserveCodeBlocks
|
||||
&& props.options.preserveLists
|
||||
))
|
||||
|
||||
watch(() => props.options.chunkMethod, (method) => {
|
||||
if (method === 'custom') advancedChunkSettingsOpen.value = true
|
||||
})
|
||||
|
||||
watch(() => props.chunkValidationMessage, (message) => {
|
||||
if (message) advancedChunkSettingsOpen.value = true
|
||||
})
|
||||
|
||||
function updateField<K extends keyof UnstructuredProcessOptions>(
|
||||
field: K,
|
||||
value: UnstructuredProcessOptions[K],
|
||||
) {
|
||||
emit('update:options', { ...props.options, [field]: value })
|
||||
}
|
||||
|
||||
function updateGenerationOptions(value: GenerationControlOptions) {
|
||||
emit('update:options', { ...props.options, ...value })
|
||||
}
|
||||
|
||||
function updateSmartPreprocess(value: string | number | boolean) {
|
||||
const enabled = Boolean(value)
|
||||
const remainingOptions = props.options.preprocessOptions.filter(
|
||||
(option) => !SMART_PREPROCESS_OPTIONS.includes(option),
|
||||
)
|
||||
updateField(
|
||||
'preprocessOptions',
|
||||
enabled ? [...SMART_PREPROCESS_OPTIONS, ...remainingOptions] : remainingOptions,
|
||||
)
|
||||
}
|
||||
|
||||
function updateDesensitize(value: string | number | boolean) {
|
||||
const preprocessOptions: UnstructuredPreprocessOption[] = props.options.preprocessOptions.filter(
|
||||
(option) => option !== 'desensitize',
|
||||
)
|
||||
if (Boolean(value)) preprocessOptions.push('desensitize')
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
}
|
||||
|
||||
function updateSpecialContentProtection(value: string | number | boolean) {
|
||||
const enabled = Boolean(value)
|
||||
emit('update:options', {
|
||||
...props.options,
|
||||
preserveTables: enabled,
|
||||
preserveCodeBlocks: enabled,
|
||||
preserveLists: enabled,
|
||||
})
|
||||
}
|
||||
|
||||
function updateUnstructuredNumber(field: UnstructuredNumberField, value: number | undefined) {
|
||||
const limits = UNSTRUCTURED_NUMBER_LIMITS[field]
|
||||
const parsedValue = Number(value)
|
||||
const nextValue = Number.isFinite(parsedValue) ? parsedValue : limits.min
|
||||
updateField(field, Math.min(limits.max, Math.max(limits.min, nextValue)))
|
||||
}
|
||||
|
||||
function revealValidation() {
|
||||
advancedChunkSettingsOpen.value = true
|
||||
}
|
||||
|
||||
defineExpose({ revealValidation })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="form-section unstructured-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>默认使用推荐策略，只需决定是否需要脱敏</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list compact-option-list">
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>智能预处理</strong>
|
||||
<small>自动完成内容清理、结构解析、短段合并、质量过滤、去重及上下文保留</small>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="smartPreprocessEnabled"
|
||||
aria-label="智能预处理"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
@update:model-value="updateSmartPreprocess"
|
||||
/>
|
||||
</div>
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>敏感信息脱敏</strong>
|
||||
<small>处理姓名、手机号、邮箱和证件号等信息</small>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="desensitizeEnabled"
|
||||
aria-label="敏感信息脱敏"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
@update:model-value="updateDesensitize"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section chunk-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>切分选项</h3>
|
||||
<p>以语义完整为优先，将长文档拆成可独立生成问答的内容块</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chunk-settings-grid is-basic">
|
||||
<label class="config-field">
|
||||
<span class="config-field-label">切分方式</span>
|
||||
<el-select
|
||||
:model-value="options.chunkMethod"
|
||||
aria-label="切分方式"
|
||||
@update:model-value="updateField('chunkMethod', $event)"
|
||||
>
|
||||
<el-option
|
||||
v-for="method in CHUNK_METHODS"
|
||||
:key="method.value"
|
||||
:label="method.label"
|
||||
:value="method.value"
|
||||
/>
|
||||
</el-select>
|
||||
<small>推荐使用自动语义切分，在长度限制内优先保留完整句段</small>
|
||||
</label>
|
||||
|
||||
<label class="config-field">
|
||||
<span class="config-field-label">切片长度</span>
|
||||
<span class="unit-input">
|
||||
<el-input-number
|
||||
:model-value="options.chunkSize"
|
||||
:min="200"
|
||||
:max="2000"
|
||||
:step="50"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
aria-label="切片长度"
|
||||
@update:model-value="updateUnstructuredNumber('chunkSize', $event)"
|
||||
/>
|
||||
<span>Token</span>
|
||||
</span>
|
||||
<small>单个切片的目标上限，默认 800 Token</small>
|
||||
</label>
|
||||
|
||||
<label class="config-field">
|
||||
<span class="config-field-label">重叠长度</span>
|
||||
<span class="unit-input">
|
||||
<el-input-number
|
||||
:model-value="options.chunkOverlap"
|
||||
:min="0"
|
||||
:max="500"
|
||||
:step="10"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
aria-label="重叠长度"
|
||||
@update:model-value="updateUnstructuredNumber('chunkOverlap', $event)"
|
||||
/>
|
||||
<span>Token</span>
|
||||
</span>
|
||||
<small>在相邻切片中最多重复保留的上下文，默认 100 Token</small>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<p class="chunk-estimation-note">
|
||||
Token 数为轻量估算值，实际长度以训练使用的模型分词器为准。
|
||||
</p>
|
||||
|
||||
<p v-if="chunkValidationMessage" class="option-validation-message" role="alert">
|
||||
{{ chunkValidationMessage }}
|
||||
</p>
|
||||
|
||||
<button
|
||||
type="button"
|
||||
class="advanced-settings-toggle"
|
||||
:aria-expanded="advancedChunkSettingsOpen"
|
||||
aria-controls="unstructured-advanced-settings"
|
||||
@click="advancedChunkSettingsOpen = !advancedChunkSettingsOpen"
|
||||
>
|
||||
<span>
|
||||
<strong>高级设置</strong>
|
||||
<small>最小切片长度、自定义分隔符和特殊内容保护</small>
|
||||
</span>
|
||||
<i class="fa" :class="advancedChunkSettingsOpen ? 'fa-chevron-up' : 'fa-chevron-down'" />
|
||||
</button>
|
||||
|
||||
<div
|
||||
v-if="advancedChunkSettingsOpen"
|
||||
id="unstructured-advanced-settings"
|
||||
class="advanced-settings-panel"
|
||||
>
|
||||
<div class="advanced-settings-grid">
|
||||
<label class="config-field">
|
||||
<span class="config-field-label">最小切片长度</span>
|
||||
<span class="unit-input">
|
||||
<el-input-number
|
||||
:model-value="options.minChunkSize"
|
||||
:min="20"
|
||||
:max="500"
|
||||
:step="10"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
aria-label="最小切片长度"
|
||||
@update:model-value="updateUnstructuredNumber('minChunkSize', $event)"
|
||||
/>
|
||||
<span>Token</span>
|
||||
</span>
|
||||
<small>过短的尾部内容会尽量并入前一个切片</small>
|
||||
</label>
|
||||
|
||||
<label v-if="options.chunkMethod === 'custom'" class="config-field">
|
||||
<span class="config-field-label">自定义分隔符</span>
|
||||
<el-input
|
||||
:model-value="options.customDelimiter"
|
||||
maxlength="40"
|
||||
show-word-limit
|
||||
placeholder="例如：--- 或 ###"
|
||||
aria-label="自定义分隔符"
|
||||
@update:model-value="updateField('customDelimiter', $event)"
|
||||
/>
|
||||
<small>系统会优先在分隔符位置结束当前切片</small>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="generation-option-row advanced-protection-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>保护表格、代码和列表</strong>
|
||||
<small>避免切分点破坏特殊内容块的完整性</small>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="preserveSpecialContentEnabled"
|
||||
aria-label="保护表格、代码和列表"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
@update:model-value="updateSpecialContentProtection"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section generation-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>生成选项</h3>
|
||||
<p>只保留会直接影响输出的核心设置</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="options"
|
||||
:models="generationModels"
|
||||
section="quality"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateGenerationOptions"
|
||||
/>
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>语义丰富表达</strong>
|
||||
<small>使用大模型将问答表述得更自然、柔和</small>
|
||||
</div>
|
||||
<el-switch
|
||||
:model-value="options.semanticEnrichment"
|
||||
aria-label="非结构化语义丰富表达"
|
||||
inline-prompt
|
||||
active-text="开"
|
||||
inactive-text="关"
|
||||
@update:model-value="updateField('semanticEnrichment', Boolean($event))"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="generation-option-row">
|
||||
<div class="generation-option-copy">
|
||||
<strong>每个切片生成数量</strong>
|
||||
<small>每个内容切片最多生成 3 个不同角度的问答对</small>
|
||||
</div>
|
||||
<el-input-number
|
||||
:model-value="options.qaPairsPerChunk"
|
||||
:min="1"
|
||||
:max="3"
|
||||
:step="1"
|
||||
:precision="0"
|
||||
controls-position="right"
|
||||
aria-label="每个切片生成数量"
|
||||
@update:model-value="updateUnstructuredNumber('qaPairsPerChunk', $event)"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<DatasetSplitEditor
|
||||
:model-value="options.datasetSplit"
|
||||
aria-label-prefix="非结构化"
|
||||
@update:model-value="updateField('datasetSplit', $event)"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section model-options-section">
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>大模型</h3>
|
||||
<p>选择数据生成模型，并设置模型输出内容的要求</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="generation-option-list">
|
||||
<GenerationOptionsPanel
|
||||
:options="options"
|
||||
:models="generationModels"
|
||||
section="model"
|
||||
:validation-message="validationAttempted ? generationValidationMessage : ''"
|
||||
@update:options="updateGenerationOptions"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style scoped lang="scss">
|
||||
.form-section {
|
||||
padding: 0 0 26px;
|
||||
margin-bottom: 26px;
|
||||
border-bottom: 1px solid #edf0f5;
|
||||
|
||||
&:last-child {
|
||||
margin-bottom: 0;
|
||||
border-bottom: 0;
|
||||
}
|
||||
|
||||
h3 {
|
||||
margin: 0 0 16px;
|
||||
color: #2f3747;
|
||||
font-size: 15px;
|
||||
font-weight: 650;
|
||||
}
|
||||
}
|
||||
|
||||
.section-title-row {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
|
||||
h3 {
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
p {
|
||||
margin: 0;
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.generation-option-list {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.compact-option-list {
|
||||
gap: 8px;
|
||||
|
||||
.generation-option-row {
|
||||
min-height: 58px;
|
||||
}
|
||||
}
|
||||
|
||||
.generation-option-row {
|
||||
display: flex;
|
||||
min-height: 64px;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 24px;
|
||||
padding: 12px 14px;
|
||||
box-sizing: border-box;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.generation-option-copy {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
|
||||
strong {
|
||||
color: #344054;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
}
|
||||
|
||||
.chunk-settings-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.config-field {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
gap: 8px;
|
||||
min-width: 0;
|
||||
padding: 14px;
|
||||
color: #344054;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
|
||||
> small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
:deep(.el-select),
|
||||
:deep(.el-input) {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.config-field-label {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.unit-input {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
color: #7b8495;
|
||||
font-size: 12px;
|
||||
|
||||
:deep(.el-input-number) {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.option-validation-message {
|
||||
display: block;
|
||||
margin: 8px 0 0;
|
||||
color: #dc2626;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.chunk-estimation-note {
|
||||
margin: 9px 0 0;
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.advanced-settings-toggle {
|
||||
display: flex;
|
||||
width: 100%;
|
||||
min-height: 48px;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
margin-top: 12px;
|
||||
padding: 10px 14px;
|
||||
color: #344054;
|
||||
text-align: left;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
cursor: pointer;
|
||||
transition: border-color 0.18s ease, background-color 0.18s ease;
|
||||
|
||||
&:hover {
|
||||
background: #f8f7ff;
|
||||
border-color: #b7b2f7;
|
||||
}
|
||||
|
||||
&:focus-visible {
|
||||
outline: 2px solid #5b50f2;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
> span {
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
}
|
||||
|
||||
strong {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
small {
|
||||
color: #8a93a3;
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
i {
|
||||
flex: 0 0 auto;
|
||||
color: #7b8495;
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.advanced-settings-panel {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
margin-top: 8px;
|
||||
padding: 12px;
|
||||
background: #fff;
|
||||
border: 1px solid #e2e5ec;
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.advanced-settings-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
@media (max-width: 900px) {
|
||||
.chunk-settings-grid,
|
||||
.advanced-settings-grid {
|
||||
grid-template-columns: minmax(0, 1fr);
|
||||
}
|
||||
|
||||
.generation-option-row {
|
||||
align-items: flex-start;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.compact-option-list .generation-option-row,
|
||||
.advanced-protection-row {
|
||||
align-items: center;
|
||||
flex-direction: row;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
193
frontend/src/views/data-process/create/data-process-create.scss
Normal file
193
frontend/src/views/data-process/create/data-process-create.scss
Normal file
@@ -0,0 +1,193 @@
|
||||
.create-wizard-layout {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
border: 1px solid #eef0f5;
|
||||
border-radius: 8px;
|
||||
background: #fff;
|
||||
}
|
||||
|
||||
.wizard-main {
|
||||
flex: 1;
|
||||
overflow-y: auto;
|
||||
padding: 32px;
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
width: 6px;
|
||||
}
|
||||
|
||||
&::-webkit-scrollbar-thumb {
|
||||
background: #cbd5e1;
|
||||
border-radius: 3px;
|
||||
}
|
||||
}
|
||||
|
||||
.wizard-main-inner {
|
||||
min-height: 100%;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
.wizard-steps-container {
|
||||
margin-bottom: 32px;
|
||||
padding-bottom: 24px;
|
||||
border-bottom: 1px dashed #e2e8f0;
|
||||
}
|
||||
|
||||
.custom-wizard-steps {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
max-width: 860px;
|
||||
margin: 0 auto;
|
||||
padding: 0 20px;
|
||||
}
|
||||
|
||||
.step-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex: 1;
|
||||
|
||||
&:first-child {
|
||||
flex: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.step-connector {
|
||||
flex: 1;
|
||||
height: 2px;
|
||||
margin: 0 16px;
|
||||
background-color: #e2e8f0;
|
||||
transition: background-color 0.3s;
|
||||
}
|
||||
|
||||
.step-item.is-completed .step-connector,
|
||||
.step-item.is-active .step-connector {
|
||||
background-color: #5146e5;
|
||||
}
|
||||
|
||||
.step-node {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.step-icon {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
color: #64748b;
|
||||
font-size: 13px;
|
||||
font-weight: 650;
|
||||
background-color: #fff;
|
||||
border: 2px solid #cbd5e1;
|
||||
border-radius: 50%;
|
||||
transition: all 0.3s;
|
||||
}
|
||||
|
||||
.step-title {
|
||||
color: #64748b;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.step-item.is-active {
|
||||
.step-icon {
|
||||
color: #5146e5;
|
||||
background-color: #eef2ff;
|
||||
border-color: #5146e5;
|
||||
}
|
||||
|
||||
.step-title {
|
||||
color: #1e293b;
|
||||
}
|
||||
}
|
||||
|
||||
.step-item.is-completed {
|
||||
.step-icon {
|
||||
color: #fff;
|
||||
background-color: #5146e5;
|
||||
border-color: #5146e5;
|
||||
}
|
||||
|
||||
.step-title {
|
||||
color: #1e293b;
|
||||
}
|
||||
}
|
||||
|
||||
.wizard-content {
|
||||
min-height: 400px;
|
||||
}
|
||||
|
||||
.wizard-footer {
|
||||
display: grid;
|
||||
flex-shrink: 0;
|
||||
grid-template-columns: 1fr auto 1fr;
|
||||
align-items: center;
|
||||
height: 64px;
|
||||
padding: 0 32px;
|
||||
background: #fff;
|
||||
border-top: 1px solid #e2e8f0;
|
||||
box-shadow: 0 -4px 6px -1px rgba(0, 0, 0, 0.02);
|
||||
|
||||
.footer-left {
|
||||
justify-self: start;
|
||||
}
|
||||
|
||||
.footer-center {
|
||||
justify-self: center;
|
||||
}
|
||||
|
||||
.footer-right {
|
||||
justify-self: end;
|
||||
}
|
||||
}
|
||||
|
||||
.wizard-primary-action {
|
||||
min-width: 160px;
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.wizard-main {
|
||||
padding: 24px 20px;
|
||||
}
|
||||
|
||||
.custom-wizard-steps {
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.step-connector {
|
||||
margin: 0 8px;
|
||||
}
|
||||
|
||||
.step-node {
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.step-title {
|
||||
max-width: 72px;
|
||||
font-size: 12px;
|
||||
line-height: 1.35;
|
||||
white-space: normal;
|
||||
}
|
||||
|
||||
.wizard-footer {
|
||||
padding: 0 20px;
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.step-title {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.step-connector {
|
||||
margin: 0 6px;
|
||||
}
|
||||
|
||||
.wizard-primary-action {
|
||||
min-width: 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { StructuredProcessOptions, UnstructuredProcessOptions } from './types'
|
||||
|
||||
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
|
||||
|
||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
}
|
||||
}
|
||||
|
||||
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
chunkMethod: 'semantic',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
customDelimiter: '',
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
generationModelId: '',
|
||||
generationPrompt: DEFAULT_GENERATION_PROMPT,
|
||||
temperature: 0.7,
|
||||
maxTokens: 1024,
|
||||
jsonMode: false,
|
||||
qualityFilterEnabled: false,
|
||||
filterLowQuality: true,
|
||||
filterShortContent: true,
|
||||
minOutputLength: 20,
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,14 @@ export interface ExternalDataSource {
|
||||
limit: number
|
||||
}
|
||||
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
content: string
|
||||
}
|
||||
|
||||
export interface SourceLine {
|
||||
number: number
|
||||
content: string
|
||||
|
||||
147
frontend/src/views/data-process/create/useDataProcessDraft.ts
Normal file
147
frontend/src/views/data-process/create/useDataProcessDraft.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
import { nextTick, type Reactive, type Ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import type {
|
||||
ExternalDataSource,
|
||||
ProcessType,
|
||||
StepId,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
|
||||
export const DATA_PROCESS_DRAFT_STORAGE_KEY = 'yg-data-process-create-draft'
|
||||
export const DATA_PROCESS_DRAFT_SCHEMA_VERSION = 6
|
||||
|
||||
interface DraftSnapshot {
|
||||
schemaVersion?: number
|
||||
currentStepId?: StepId
|
||||
task?: { name?: string; description?: string }
|
||||
processType?: ProcessType
|
||||
structuredOptions?: Partial<StructuredProcessOptions>
|
||||
unstructuredOptions?: Partial<UnstructuredProcessOptions>
|
||||
externalSource?: Partial<ExternalDataSource>
|
||||
}
|
||||
|
||||
interface DraftBindings {
|
||||
currentStepId: Readonly<Ref<StepId>>
|
||||
task: Reactive<{ name: string; description: string }>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
externalSource: Reactive<ExternalDataSource>
|
||||
restoringDraft: Ref<boolean>
|
||||
dirty: Ref<boolean>
|
||||
goToStep: (stepId: StepId) => void
|
||||
}
|
||||
|
||||
function sanitizeExternalSource(source: Partial<ExternalDataSource>) {
|
||||
return {
|
||||
type: typeof source.type === 'string' ? source.type : 'mysql',
|
||||
url: typeof source.url === 'string' ? source.url : '',
|
||||
authMode: typeof source.authMode === 'string' ? source.authMode : 'none',
|
||||
username: typeof source.username === 'string' ? source.username : '',
|
||||
limit: Number.isFinite(source.limit) ? Number(source.limit) : 1000,
|
||||
}
|
||||
}
|
||||
|
||||
function isDraftSnapshot(value: unknown): value is DraftSnapshot {
|
||||
return Boolean(value && typeof value === 'object')
|
||||
}
|
||||
|
||||
/**
|
||||
* 只持久化可重建的配置。密码、令牌、文件正文、预览与结果都不进入 localStorage。
|
||||
*/
|
||||
export function useDataProcessDraft(bindings: DraftBindings) {
|
||||
function draftSnapshot(): DraftSnapshot {
|
||||
return {
|
||||
schemaVersion: DATA_PROCESS_DRAFT_SCHEMA_VERSION,
|
||||
currentStepId: bindings.currentStepId.value,
|
||||
task: { ...bindings.task },
|
||||
processType: bindings.processType.value,
|
||||
structuredOptions: {
|
||||
...bindings.structuredOptions.value,
|
||||
preprocessOptions: [...bindings.structuredOptions.value.preprocessOptions],
|
||||
datasetSplit: { ...bindings.structuredOptions.value.datasetSplit },
|
||||
},
|
||||
unstructuredOptions: {
|
||||
...bindings.unstructuredOptions.value,
|
||||
preprocessOptions: [...bindings.unstructuredOptions.value.preprocessOptions],
|
||||
datasetSplit: { ...bindings.unstructuredOptions.value.datasetSplit },
|
||||
},
|
||||
externalSource: sanitizeExternalSource(bindings.externalSource),
|
||||
}
|
||||
}
|
||||
|
||||
function writeDraft(showWarning: boolean) {
|
||||
try {
|
||||
localStorage.setItem(DATA_PROCESS_DRAFT_STORAGE_KEY, JSON.stringify(draftSnapshot()))
|
||||
} catch {
|
||||
if (showWarning) ElMessage.warning('草稿保存失败，请检查浏览器存储空间')
|
||||
}
|
||||
}
|
||||
|
||||
function persistDraft() {
|
||||
if (!bindings.restoringDraft.value) writeDraft(true)
|
||||
}
|
||||
|
||||
function restoreDraft() {
|
||||
try {
|
||||
const raw = localStorage.getItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
if (!raw) return
|
||||
const snapshot: unknown = JSON.parse(raw)
|
||||
if (!isDraftSnapshot(snapshot)) return
|
||||
|
||||
bindings.restoringDraft.value = true
|
||||
bindings.goToStep('create')
|
||||
bindings.task.name = snapshot.task?.name || ''
|
||||
bindings.task.description = snapshot.task?.description || ''
|
||||
bindings.processType.value = snapshot.processType === 'structured' || snapshot.processType === 'external'
|
||||
? snapshot.processType
|
||||
: 'unstructured'
|
||||
|
||||
if (snapshot.structuredOptions) {
|
||||
bindings.structuredOptions.value = {
|
||||
...bindings.structuredOptions.value,
|
||||
...snapshot.structuredOptions,
|
||||
preprocessOptions: Array.isArray(snapshot.structuredOptions.preprocessOptions)
|
||||
? snapshot.structuredOptions.preprocessOptions
|
||||
: bindings.structuredOptions.value.preprocessOptions,
|
||||
datasetSplit: {
|
||||
...bindings.structuredOptions.value.datasetSplit,
|
||||
...snapshot.structuredOptions.datasetSplit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshot.unstructuredOptions) {
|
||||
bindings.unstructuredOptions.value = {
|
||||
...bindings.unstructuredOptions.value,
|
||||
...snapshot.unstructuredOptions,
|
||||
preprocessOptions: Array.isArray(snapshot.unstructuredOptions.preprocessOptions)
|
||||
? snapshot.unstructuredOptions.preprocessOptions
|
||||
: bindings.unstructuredOptions.value.preprocessOptions,
|
||||
datasetSplit: {
|
||||
...bindings.unstructuredOptions.value.datasetSplit,
|
||||
...snapshot.unstructuredOptions.datasetSplit,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
Object.assign(bindings.externalSource, sanitizeExternalSource(snapshot.externalSource || {}), {
|
||||
password: '',
|
||||
token: '',
|
||||
})
|
||||
bindings.dirty.value = false
|
||||
|
||||
nextTick(() => {
|
||||
bindings.restoringDraft.value = false
|
||||
// 立即覆盖 v5 及更早草稿，清除其中可能存在的敏感值和大段正文。
|
||||
writeDraft(false)
|
||||
})
|
||||
ElMessage.info('已恢复上次的任务配置，请重新上传或拉取源数据')
|
||||
} catch {
|
||||
localStorage.removeItem(DATA_PROCESS_DRAFT_STORAGE_KEY)
|
||||
}
|
||||
}
|
||||
|
||||
return { draftSnapshot, persistDraft, restoreDraft }
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { reactive, ref, type Ref } from 'vue'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { createResults } from './previewModel'
|
||||
import type {
|
||||
GenerationState,
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
ResultItem,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
} from './types'
|
||||
|
||||
interface GenerationBindings {
|
||||
previewItems: Ref<PreviewItem[]>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
dirty: Ref<boolean>
|
||||
}
|
||||
|
||||
export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
const results = ref<ResultItem[]>([])
|
||||
const selectedResultId = ref<string | null>(null)
|
||||
const generation = reactive<GenerationState>({
|
||||
status: 'idle',
|
||||
progress: 0,
|
||||
message: '确认摘要后即可开始生成，过程中可查看实时进度。',
|
||||
})
|
||||
let generationTimer: ReturnType<typeof setInterval> | null = null
|
||||
|
||||
function stopGenerationTimer() {
|
||||
if (generationTimer) clearInterval(generationTimer)
|
||||
generationTimer = null
|
||||
}
|
||||
|
||||
function resetDownstream() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'idle'
|
||||
generation.progress = 0
|
||||
generation.message = '确认摘要后即可开始生成，过程中可查看实时进度。'
|
||||
results.value = []
|
||||
selectedResultId.value = null
|
||||
}
|
||||
|
||||
function startGeneration() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'running'
|
||||
generation.progress = 0
|
||||
generation.message = '正在应用预览修改并生成标准化结果，请稍候。'
|
||||
|
||||
generationTimer = setInterval(() => {
|
||||
generation.progress = Math.min(100, generation.progress + 8)
|
||||
if (generation.progress < 100) return
|
||||
|
||||
stopGenerationTimer()
|
||||
generation.status = 'success'
|
||||
results.value = createResults(
|
||||
bindings.previewItems.value,
|
||||
bindings.processType.value === 'structured'
|
||||
? bindings.structuredOptions.value
|
||||
: bindings.processType.value === 'unstructured'
|
||||
? bindings.unstructuredOptions.value
|
||||
: undefined,
|
||||
)
|
||||
generation.message = `已完成 ${results.value.length.toLocaleString()} 条数据处理，可进入结果页检查。`
|
||||
selectedResultId.value = results.value[0]?.id ?? null
|
||||
bindings.dirty.value = true
|
||||
ElMessage.success('数据处理完成')
|
||||
}, 180)
|
||||
}
|
||||
|
||||
function stopGeneration() {
|
||||
stopGenerationTimer()
|
||||
generation.status = 'failed'
|
||||
generation.message = '任务已停止，预览修改仍然保留，可以重新生成。'
|
||||
}
|
||||
|
||||
function updateResultField(id: string, field: 'instruction' | 'input' | 'output', value: string) {
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item[field] = value
|
||||
const valid = item.instruction.trim() && item.output.trim()
|
||||
item.error = valid ? undefined : 'Instruction 和 Output 不能为空'
|
||||
const changed = item.instruction !== item.originalInstruction
|
||||
|| item.input !== item.originalInput
|
||||
|| item.output !== item.originalOutput
|
||||
item.status = item.error ? 'invalid' : changed ? 'modified' : 'valid'
|
||||
bindings.dirty.value = true
|
||||
}
|
||||
|
||||
function restoreResult(id: string) {
|
||||
const item = results.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
item.instruction = item.originalInstruction
|
||||
item.input = item.originalInput
|
||||
item.output = item.originalOutput
|
||||
item.error = undefined
|
||||
item.status = 'valid'
|
||||
bindings.dirty.value = true
|
||||
}
|
||||
|
||||
function validateResults() {
|
||||
let firstInvalidId: string | null = null
|
||||
for (const item of results.value) {
|
||||
if (!item.instruction.trim() || !item.output.trim()) {
|
||||
item.error = 'Instruction 和 Output 不能为空'
|
||||
item.status = 'invalid'
|
||||
firstInvalidId ??= item.id
|
||||
}
|
||||
}
|
||||
if (firstInvalidId) selectedResultId.value = firstInvalidId
|
||||
return firstInvalidId == null
|
||||
}
|
||||
|
||||
return {
|
||||
generation,
|
||||
results,
|
||||
selectedResultId,
|
||||
resetDownstream,
|
||||
restoreResult,
|
||||
startGeneration,
|
||||
stopGeneration,
|
||||
stopGenerationTimer,
|
||||
updateResultField,
|
||||
validateResults,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user