feat(data-process): 接入重新生成配置流程
This commit is contained in:
caoxiaozhu
@@ -14,9 +14,12 @@ import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
|
||||
import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
generationAffectingOptionsFor,
|
||||
previewAffectingOptionsFor,
|
||||
} from './create/dataProcessCreateState'
|
||||
import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
||||
import {
|
||||
mapDataProcessSourceFile,
|
||||
useDataProcessSourceUpload,
|
||||
@@ -51,7 +54,7 @@ import type {
|
||||
|
||||
const router = useRouter()
|
||||
const modelsStore = useModelsStore()
|
||||
const { list: modelList } = storeToRefs(modelsStore)
|
||||
const { list: modelList, loaded: modelsLoaded } = storeToRefs(modelsStore)
|
||||
// 候选范围与模型管理保持一致，不在数据处理页面重复定义模型过滤规则。
|
||||
const generationModels = computed(() => modelList.value)
|
||||
const taskSetupRef = ref<InstanceType<typeof TaskSetupStep>>()
|
||||
@@ -98,6 +101,7 @@ const selectedPreviewFileId = ref<string | null>(null)
|
||||
const selectedPreviewId = ref<string | null>(null)
|
||||
const selectedPreviewIdsByFile = ref<Record<string, string>>({})
|
||||
const dirty = ref(false)
|
||||
const modelSubmitLoading = ref(false)
|
||||
let allowLeave = false
|
||||
|
||||
const {
|
||||
@@ -236,7 +240,7 @@ function taskPayload() {
|
||||
return {
|
||||
name: task.name.trim(),
|
||||
description: task.description.trim(),
|
||||
process_type: processType.value,
|
||||
process_type: originalProcessType.value || processType.value,
|
||||
config: toBackendConfig(),
|
||||
}
|
||||
}
|
||||
@@ -274,82 +278,11 @@ function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
}
|
||||
|
||||
function previewAffectingOptions() {
|
||||
if (processType.value === 'structured') {
|
||||
return {
|
||||
preprocessOptions: structuredOptions.value.preprocessOptions,
|
||||
}
|
||||
}
|
||||
if (processType.value === 'unstructured') {
|
||||
const {
|
||||
preprocessOptions,
|
||||
chunkMethod,
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
minChunkSize,
|
||||
semanticBreakpointPercentile,
|
||||
preserveTables,
|
||||
preserveCodeBlocks,
|
||||
preserveLists,
|
||||
} = unstructuredOptions.value
|
||||
return {
|
||||
preprocessOptions,
|
||||
chunkMethod,
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
minChunkSize,
|
||||
semanticBreakpointPercentile,
|
||||
preserveTables,
|
||||
preserveCodeBlocks,
|
||||
preserveLists,
|
||||
}
|
||||
}
|
||||
return null
|
||||
return previewAffectingOptionsFor(processType.value, structuredOptions.value, unstructuredOptions.value)
|
||||
}
|
||||
|
||||
function generationAffectingOptions() {
|
||||
if (processType.value === 'structured') {
|
||||
const {
|
||||
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
|
||||
temperature, maxTokens, jsonMode,
|
||||
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
|
||||
} = structuredOptions.value
|
||||
return {
|
||||
semanticEnrichment, qaPairsPerRow, datasetSplit, generationModelId, generationPrompt,
|
||||
temperature, maxTokens, jsonMode,
|
||||
qualityFilterEnabled, filterLowQuality, filterShortContent, minOutputLength,
|
||||
}
|
||||
}
|
||||
if (processType.value === 'unstructured') {
|
||||
const {
|
||||
semanticEnrichment,
|
||||
qaPairsPerChunk,
|
||||
datasetSplit,
|
||||
generationModelId,
|
||||
generationPrompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
jsonMode,
|
||||
qualityFilterEnabled,
|
||||
filterLowQuality,
|
||||
filterShortContent,
|
||||
minOutputLength,
|
||||
} = unstructuredOptions.value
|
||||
return {
|
||||
semanticEnrichment,
|
||||
qaPairsPerChunk,
|
||||
datasetSplit,
|
||||
generationModelId,
|
||||
generationPrompt,
|
||||
temperature,
|
||||
maxTokens,
|
||||
jsonMode,
|
||||
qualityFilterEnabled,
|
||||
filterLowQuality,
|
||||
filterShortContent,
|
||||
minOutputLength,
|
||||
}
|
||||
}
|
||||
return null
|
||||
return generationAffectingOptionsFor(processType.value, structuredOptions.value, unstructuredOptions.value)
|
||||
}
|
||||
|
||||
const generationOptionsSignature = computed(() => JSON.stringify(generationAffectingOptions()))
|
||||
@@ -365,19 +298,58 @@ function buildPreviewSignature() {
|
||||
return `${buildPreviewConfigSignature()}:${filesSignature}`
|
||||
}
|
||||
|
||||
const {
|
||||
isRegeneration,
|
||||
originalProcessType,
|
||||
regenerationPrepared,
|
||||
hydrating,
|
||||
initializationError,
|
||||
loadSource: loadRegenerationSource,
|
||||
confirmPreviewConfigChange,
|
||||
prepareRegeneration,
|
||||
} = useDataProcessRegeneration({
|
||||
task,
|
||||
processType,
|
||||
structuredOptions,
|
||||
unstructuredOptions,
|
||||
uploadedFiles,
|
||||
previewItems,
|
||||
selectedPreviewFileId,
|
||||
selectedPreviewId,
|
||||
selectedPreviewIdsByFile,
|
||||
previewSignature,
|
||||
dirty,
|
||||
buildPreviewConfigSignature,
|
||||
buildPreviewSignature,
|
||||
mapPreviewItem,
|
||||
resetDownstream,
|
||||
})
|
||||
const leaveWarningMessage = computed(() => regenerationPrepared.value
|
||||
? '仍有尚未完成的本地操作；已提交到服务端的修改不会因离开页面而撤销。'
|
||||
: '当前存在未保存修改，离开后这些修改将不会保留。')
|
||||
const leaveConfirmText = computed(() => regenerationPrepared.value ? '离开页面' : '放弃修改')
|
||||
|
||||
watch(
|
||||
[() => task.name, () => task.description, processType, structuredOptions, unstructuredOptions, externalSource],
|
||||
() => { dirty.value = true },
|
||||
() => {
|
||||
if (!hydrating.value) dirty.value = true
|
||||
},
|
||||
{ deep: true },
|
||||
)
|
||||
|
||||
watch(processType, (nextType, previousType) => {
|
||||
if (hydrating.value) return
|
||||
if (isRegeneration.value && originalProcessType.value && nextType !== originalProcessType.value) {
|
||||
processType.value = originalProcessType.value
|
||||
return
|
||||
}
|
||||
if (nextType === previousType || uploadedFiles.value.length === 0) return
|
||||
resetSourceDataForProcessTypeChange()
|
||||
ElMessage.info('处理类型已变更，请重新上传或拉取匹配的源数据')
|
||||
})
|
||||
|
||||
watch(generationOptionsSignature, (currentSignature, previousSignature) => {
|
||||
if (hydrating.value) return
|
||||
if (currentSignature === previousSignature) return
|
||||
resetDownstream()
|
||||
})
|
||||
@@ -531,21 +503,50 @@ function resetSourceDataForProcessTypeChange() {
|
||||
async function nextFromCreate() {
|
||||
const valid = await taskSetupRef.value?.validate()
|
||||
if (!valid) return
|
||||
const confirmed = await confirmPreviewConfigChange((options) => (
|
||||
confirmDialogRef.value?.open(options) ?? Promise.resolve(false)
|
||||
))
|
||||
if (!confirmed) return
|
||||
goToStep('model')
|
||||
}
|
||||
|
||||
async function nextFromModel() {
|
||||
const valid = await modelSelectionRef.value?.validate()
|
||||
if (!valid) return
|
||||
if (modelSubmitLoading.value) return
|
||||
modelSubmitLoading.value = true
|
||||
try {
|
||||
const saved = taskId.value
|
||||
? await updateDataProcessTask(taskId.value, taskPayload())
|
||||
: await createDataProcessTask(taskPayload())
|
||||
taskId.value = String(saved.id)
|
||||
await modelsStore.load(true)
|
||||
if (!modelsLoaded.value) {
|
||||
ElMessage.error('模型列表加载失败，未提交任何修改，请稍后重试')
|
||||
return
|
||||
}
|
||||
const modelId = modelSelectionOptions.value.generationModelId
|
||||
const modelExists = generationModels.value.some(model => String(model.id) === String(modelId))
|
||||
if (modelId !== '' && !modelExists) {
|
||||
ElMessage.error(isRegeneration.value ? '原任务使用的模型已删除，请重新选择模型' : '所选模型已删除，请重新选择')
|
||||
return
|
||||
}
|
||||
const valid = await modelSelectionRef.value?.validate()
|
||||
if (!valid) return
|
||||
if (isRegeneration.value) {
|
||||
const regenerated = await prepareRegeneration(taskPayload())
|
||||
taskId.value = String(regenerated.task.id)
|
||||
if (regenerated.preview_invalidated) {
|
||||
ElMessage.info('切分配置已变化，现有源文件将在下一步按新配置重新切分')
|
||||
} else if (regenerated.published_outputs_preserved) {
|
||||
ElMessage.info('已保留原切片；已发布的三个数据集将在重新发布前保持不变')
|
||||
}
|
||||
} else {
|
||||
const saved = taskId.value
|
||||
? await updateDataProcessTask(taskId.value, taskPayload())
|
||||
: await createDataProcessTask(taskPayload())
|
||||
taskId.value = String(saved.id)
|
||||
}
|
||||
dirty.value = true
|
||||
goToStep('upload')
|
||||
} catch {
|
||||
// 请求层已展示名称冲突或配置非法等具体原因。
|
||||
} finally {
|
||||
modelSubmitLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -812,8 +813,8 @@ async function handleCancel() {
|
||||
}
|
||||
const confirmed = await confirmDialogRef.value?.open({
|
||||
title: '确认离开当前页面？',
|
||||
message: '当前存在未保存修改，离开后这些修改将不会保留。',
|
||||
confirmText: '放弃修改',
|
||||
message: leaveWarningMessage.value,
|
||||
confirmText: leaveConfirmText.value,
|
||||
cancelText: '继续编辑',
|
||||
tone: 'danger',
|
||||
})
|
||||
@@ -830,8 +831,8 @@ onBeforeRouteLeave(async () => {
|
||||
if (allowLeave || !dirty.value) return true
|
||||
const confirmed = await confirmDialogRef.value?.open({
|
||||
title: '确认离开当前页面？',
|
||||
message: '当前存在未保存修改，离开后这些修改将不会保留。',
|
||||
confirmText: '放弃修改',
|
||||
message: leaveWarningMessage.value,
|
||||
confirmText: leaveConfirmText.value,
|
||||
cancelText: '继续编辑',
|
||||
tone: 'danger',
|
||||
})
|
||||
@@ -845,6 +846,7 @@ onBeforeUnmount(() => {
|
||||
onMounted(() => {
|
||||
localStorage.removeItem('yg-data-process-create-draft')
|
||||
void modelsStore.load(true)
|
||||
if (isRegeneration.value) void loadRegenerationSource()
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -882,15 +884,20 @@ onMounted(() => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="wizard-content">
|
||||
<div class="wizard-content" v-loading="hydrating">
|
||||
<div v-if="initializationError" class="initialization-error" role="alert">
|
||||
<el-alert type="error" :closable="false" :title="initializationError" show-icon />
|
||||
<el-button type="primary" :loading="hydrating" @click="loadRegenerationSource">重试加载原任务</el-button>
|
||||
</div>
|
||||
<TaskSetupStep
|
||||
v-if="currentStepId === 'create'"
|
||||
v-else-if="currentStepId === 'create'"
|
||||
ref="taskSetupRef"
|
||||
v-model:name="task.name"
|
||||
v-model:description="task.description"
|
||||
v-model:process-type="processType"
|
||||
v-model:structured-options="structuredOptions"
|
||||
v-model:unstructured-options="unstructuredOptions"
|
||||
:process-type-locked="isRegeneration"
|
||||
/>
|
||||
|
||||
<ModelSelectionStep
|
||||
@@ -974,8 +981,8 @@ onMounted(() => {
|
||||
<el-button
|
||||
class="wizard-primary-action"
|
||||
type="primary"
|
||||
:loading="generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="(currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
:loading="modelSubmitLoading || generation.status === 'running' || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
@click="handlePrimaryAction"
|
||||
>
|
||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, onBeforeUnmount, onMounted, reactive, ref, watch } from 'vue'
|
||||
import { useRoute, useRouter } from 'vue-router'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import PageCard from '@/components/PageCard.vue'
|
||||
import ModelStatusTag from '@/components/ModelStatusTag.vue'
|
||||
import { usePolling } from '@/composables/usePolling'
|
||||
@@ -167,11 +167,14 @@ const outputDatasets = computed(() => {
|
||||
const outputDatasetName = computed(() => (
|
||||
outputDatasets.value.map((item) => item.name).join('、')
|
||||
))
|
||||
const outputDatasetBaseName = computed(() => {
|
||||
const name = String(outputDatasets.value[0]?.name || '').trim()
|
||||
return name.replace(/-(?:训练集|验证集|测试集)$/, '')
|
||||
})
|
||||
const outputDatasetId = computed(() => detail.value?.output_dataset_id || null)
|
||||
const canRegenerate = computed(() => {
|
||||
const status = detail.value?.status
|
||||
return status === 'pending'
|
||||
|| status === 'failed'
|
||||
|| status === 'stopped'
|
||||
|| (status === 'completed' && Boolean(outputDatasetId.value))
|
||||
})
|
||||
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
|
||||
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
|
||||
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
|
||||
@@ -428,10 +431,6 @@ function configuredSplit(): DataProcessDatasetSplit {
|
||||
|
||||
function openPublishDialog() {
|
||||
if (!detail.value) return
|
||||
if (outputDatasetId.value) {
|
||||
void router.push({ path: '/dataset', query: { tab: 'task', task_id: taskId.value } })
|
||||
return
|
||||
}
|
||||
publishForm.dataset_name = `${detail.value.name}-数据集`
|
||||
publishForm.split = configuredSplit()
|
||||
publishDialogVisible.value = true
|
||||
@@ -461,31 +460,8 @@ async function publishDataset() {
|
||||
}
|
||||
}
|
||||
|
||||
async function resyncPublishedDataset() {
|
||||
if (!detail.value || !outputDatasetId.value || publishing.value) return
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
'将按当前 8:1:1 配置同步为训练集、验证集、测试集三个独立数据集。',
|
||||
'重新同步三个数据集？',
|
||||
{ type: 'warning', confirmButtonText: '重新同步', cancelButtonText: '取消' },
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
publishing.value = true
|
||||
try {
|
||||
const published = await publishDataProcess(taskId.value, {
|
||||
...publishForm,
|
||||
dataset_name: outputDatasetBaseName.value || `${detail.value.name}-数据集`,
|
||||
split: configuredSplit(),
|
||||
})
|
||||
const datasets = published.datasets || published.output_datasets || []
|
||||
ElMessage.success(`训练集、验证集和测试集已同步，共 ${datasets.length || 3} 个数据集`)
|
||||
await router.push({ path: '/dataset', query: { tab: 'task', task_id: taskId.value } })
|
||||
} finally {
|
||||
publishing.value = false
|
||||
}
|
||||
function startRegeneration() {
|
||||
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
|
||||
}
|
||||
|
||||
watch([currentPage, pageSize], () => void loadResults())
|
||||
@@ -504,20 +480,20 @@ onBeforeUnmount(() => {
|
||||
<h1>{{ detail.name }}</h1>
|
||||
<ModelStatusTag :status="detail.status" />
|
||||
<el-button
|
||||
v-if="detail.status === 'completed'"
|
||||
v-if="detail.status === 'completed' && !outputDatasetId"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa" :class="outputDatasetId ? 'fa-external-link' : 'fa-database'" />
|
||||
{{ outputDatasetId ? '查看三个数据集' : '发布为三个数据集' }}
|
||||
<i class="fa fa-database" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && outputDatasetId"
|
||||
:loading="publishing"
|
||||
@click="resyncPublishedDataset"
|
||||
v-if="canRegenerate"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" />重新同步数据集
|
||||
<i class="fa fa-refresh" />重新生成
|
||||
</el-button>
|
||||
</div>
|
||||
<p>{{ detail.description || '暂无任务描述' }}</p>
|
||||
|
||||
@@ -16,6 +16,7 @@ const props = defineProps<{
|
||||
processType: ProcessType
|
||||
structuredOptions: StructuredProcessOptions
|
||||
unstructuredOptions: UnstructuredProcessOptions
|
||||
processTypeLocked?: boolean
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
@@ -153,7 +154,8 @@ defineExpose({ validate })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>处理类型</h3>
|
||||
<p>类型会影响下一步支持的数据源格式和后续预览方式</p>
|
||||
<p v-if="processTypeLocked">重新生成沿用原任务处理类型，不可修改</p>
|
||||
<p v-else>类型会影响下一步支持的数据源格式和后续预览方式</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-form-item prop="processType" class="type-form-item">
|
||||
@@ -162,6 +164,7 @@ defineExpose({ validate })
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'structured' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'structured')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-table" /></span>
|
||||
@@ -175,6 +178,7 @@ defineExpose({ validate })
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'unstructured' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'unstructured')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-file-text-o" /></span>
|
||||
@@ -188,6 +192,7 @@ defineExpose({ validate })
|
||||
type="button"
|
||||
class="type-option"
|
||||
:class="{ 'is-active': processType === 'external' }"
|
||||
:disabled="processTypeLocked"
|
||||
@click="emit('update:processType', 'external')"
|
||||
>
|
||||
<span class="type-icon"><i class="fa fa-cloud-download" /></span>
|
||||
@@ -293,12 +298,25 @@ defineExpose({ validate })
|
||||
border-color: #a8a3ff;
|
||||
}
|
||||
|
||||
&:disabled {
|
||||
cursor: not-allowed;
|
||||
opacity: 0.58;
|
||||
}
|
||||
|
||||
&:disabled:hover {
|
||||
border-color: #dfe3ea;
|
||||
}
|
||||
|
||||
&.is-active {
|
||||
background: #fafaff;
|
||||
border-color: #5b50f2;
|
||||
box-shadow: 0 0 0 1px rgba(91, 80, 242, 0.08);
|
||||
}
|
||||
|
||||
&.is-active:disabled {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
strong,
|
||||
small {
|
||||
display: block;
|
||||
|
||||
@@ -117,6 +117,12 @@
|
||||
min-height: 400px;
|
||||
}
|
||||
|
||||
.initialization-error {
|
||||
display: grid;
|
||||
justify-items: start;
|
||||
gap: 16px;
|
||||
}
|
||||
|
||||
.wizard-footer {
|
||||
display: grid;
|
||||
flex-shrink: 0;
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
import type { StructuredProcessOptions, UnstructuredProcessOptions } from './types'
|
||||
import type { DataProcessConfig, DataProcessDatasetSplit } from '@/types/dataProcess'
|
||||
import type {
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredPreprocessOption,
|
||||
UnstructuredProcessOptions,
|
||||
ProcessType,
|
||||
} from './types'
|
||||
|
||||
export const DEFAULT_GENERATION_PROMPT = '你是一名专业的数据生成助手。请根据输入内容生成准确、完整、可直接用于模型训练的问答数据。仅输出符合目标格式的内容，答案应事实清晰、语言自然，不要添加分析过程、说明或无关内容。'
|
||||
|
||||
@@ -52,3 +60,166 @@ export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
minOutputLength: 20,
|
||||
}
|
||||
}
|
||||
|
||||
function configValue<T>(config: DataProcessConfig, key: string, fallback: T): T {
|
||||
return Object.prototype.hasOwnProperty.call(config, key) ? config[key] as T : fallback
|
||||
}
|
||||
|
||||
function numberValue(config: DataProcessConfig, key: string, fallback: number): number {
|
||||
const value = Number(configValue(config, key, fallback))
|
||||
return Number.isFinite(value) ? value : fallback
|
||||
}
|
||||
|
||||
function datasetSplitValue(config: DataProcessConfig, fallback: DataProcessDatasetSplit) {
|
||||
const value = config.dataset_split
|
||||
if (!value || typeof value !== 'object') return { ...fallback }
|
||||
const split = value as unknown as Record<string, unknown>
|
||||
const splitNumber = (key: keyof DataProcessDatasetSplit) => {
|
||||
const parsed = Number(split[key])
|
||||
return Number.isFinite(parsed) ? parsed : fallback[key]
|
||||
}
|
||||
return {
|
||||
train: splitNumber('train'),
|
||||
validation: splitNumber('validation'),
|
||||
test: splitNumber('test'),
|
||||
}
|
||||
}
|
||||
|
||||
function generationOptionsFromConfig(
|
||||
config: DataProcessConfig,
|
||||
defaults: GenerationControlOptions,
|
||||
): GenerationControlOptions {
|
||||
return {
|
||||
generationModelId: configValue(config, 'generation_model_id', defaults.generationModelId),
|
||||
generationPrompt: String(configValue(config, 'generation_prompt', defaults.generationPrompt)),
|
||||
temperature: numberValue(config, 'temperature', defaults.temperature),
|
||||
maxTokens: numberValue(config, 'max_tokens', defaults.maxTokens),
|
||||
jsonMode: Boolean(configValue(config, 'json_mode', defaults.jsonMode)),
|
||||
qualityFilterEnabled: Boolean(configValue(
|
||||
config,
|
||||
'quality_filter_enabled',
|
||||
defaults.qualityFilterEnabled,
|
||||
)),
|
||||
filterLowQuality: Boolean(configValue(
|
||||
config,
|
||||
'filter_low_quality',
|
||||
defaults.filterLowQuality,
|
||||
)),
|
||||
filterShortContent: Boolean(configValue(
|
||||
config,
|
||||
'filter_short_content',
|
||||
defaults.filterShortContent,
|
||||
)),
|
||||
minOutputLength: numberValue(config, 'min_output_length', defaults.minOutputLength),
|
||||
}
|
||||
}
|
||||
|
||||
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
|
||||
const defaults = createDefaultStructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as PreprocessOption[]
|
||||
: defaults.preprocessOptions,
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
'semantic_enrichment',
|
||||
defaults.semanticEnrichment,
|
||||
)),
|
||||
qaPairsPerRow: numberValue(config, 'qa_pairs_per_row', defaults.qaPairsPerRow),
|
||||
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
|
||||
}
|
||||
}
|
||||
|
||||
export function createUnstructuredOptionsFromConfig(config: DataProcessConfig): UnstructuredProcessOptions {
|
||||
const defaults = createDefaultUnstructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as UnstructuredPreprocessOption[]
|
||||
: defaults.preprocessOptions,
|
||||
chunkMethod: configValue(config, 'chunk_method', defaults.chunkMethod),
|
||||
chunkSize: numberValue(config, 'chunk_size', defaults.chunkSize),
|
||||
chunkOverlap: numberValue(config, 'chunk_overlap', defaults.chunkOverlap),
|
||||
minChunkSize: numberValue(config, 'min_chunk_size', defaults.minChunkSize),
|
||||
semanticBreakpointPercentile: numberValue(
|
||||
config,
|
||||
'semantic_breakpoint_percentile',
|
||||
defaults.semanticBreakpointPercentile,
|
||||
),
|
||||
preserveTables: Boolean(configValue(config, 'preserve_tables', defaults.preserveTables)),
|
||||
preserveCodeBlocks: Boolean(configValue(
|
||||
config,
|
||||
'preserve_code_blocks',
|
||||
defaults.preserveCodeBlocks,
|
||||
)),
|
||||
preserveLists: Boolean(configValue(config, 'preserve_lists', defaults.preserveLists)),
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
'semantic_enrichment',
|
||||
defaults.semanticEnrichment,
|
||||
)),
|
||||
qaPairsPerChunk: numberValue(config, 'qa_pairs_per_chunk', defaults.qaPairsPerChunk),
|
||||
datasetSplit: datasetSplitValue(config, defaults.datasetSplit),
|
||||
}
|
||||
}
|
||||
|
||||
export function previewAffectingOptionsFor(
|
||||
processType: ProcessType,
|
||||
structured: StructuredProcessOptions,
|
||||
unstructured: UnstructuredProcessOptions,
|
||||
) {
|
||||
if (processType === 'structured') return { preprocessOptions: structured.preprocessOptions }
|
||||
if (processType !== 'unstructured') return null
|
||||
const {
|
||||
preprocessOptions,
|
||||
chunkMethod,
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
minChunkSize,
|
||||
semanticBreakpointPercentile,
|
||||
preserveTables,
|
||||
preserveCodeBlocks,
|
||||
preserveLists,
|
||||
} = unstructured
|
||||
return {
|
||||
preprocessOptions,
|
||||
chunkMethod,
|
||||
chunkSize,
|
||||
chunkOverlap,
|
||||
minChunkSize,
|
||||
semanticBreakpointPercentile,
|
||||
preserveTables,
|
||||
preserveCodeBlocks,
|
||||
preserveLists,
|
||||
}
|
||||
}
|
||||
|
||||
export function generationAffectingOptionsFor(
|
||||
processType: ProcessType,
|
||||
structured: StructuredProcessOptions,
|
||||
unstructured: UnstructuredProcessOptions,
|
||||
) {
|
||||
const options = processType === 'unstructured' ? unstructured : structured
|
||||
if (processType === 'external') return null
|
||||
const common = {
|
||||
semanticEnrichment: options.semanticEnrichment,
|
||||
datasetSplit: options.datasetSplit,
|
||||
generationModelId: options.generationModelId,
|
||||
generationPrompt: options.generationPrompt,
|
||||
temperature: options.temperature,
|
||||
maxTokens: options.maxTokens,
|
||||
jsonMode: options.jsonMode,
|
||||
qualityFilterEnabled: options.qualityFilterEnabled,
|
||||
filterLowQuality: options.filterLowQuality,
|
||||
filterShortContent: options.filterShortContent,
|
||||
minOutputLength: options.minOutputLength,
|
||||
}
|
||||
return processType === 'unstructured'
|
||||
? { ...common, qaPairsPerChunk: unstructured.qaPairsPerChunk }
|
||||
: { ...common, qaPairsPerRow: structured.qaPairsPerRow }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import {
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
regenerateDataProcessTask,
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type {
|
||||
DataProcessPreviewItem,
|
||||
DataProcessRegeneratePayload,
|
||||
DataProcessTask,
|
||||
} from '@/types/dataProcess'
|
||||
import {
|
||||
createStructuredOptionsFromConfig,
|
||||
createUnstructuredOptionsFromConfig,
|
||||
} from './dataProcessCreateState'
|
||||
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
StructuredProcessOptions,
|
||||
UnstructuredProcessOptions,
|
||||
UploadedDataFile,
|
||||
} from './types'
|
||||
|
||||
interface ConfirmOptions {
|
||||
title: string
|
||||
message: string
|
||||
confirmText: string
|
||||
cancelText: string
|
||||
tone: 'warning'
|
||||
}
|
||||
|
||||
interface RegenerationBindings {
|
||||
task: Reactive<{ name: string; description: string }>
|
||||
processType: Ref<ProcessType>
|
||||
structuredOptions: Ref<StructuredProcessOptions>
|
||||
unstructuredOptions: Ref<UnstructuredProcessOptions>
|
||||
uploadedFiles: Ref<UploadedDataFile[]>
|
||||
previewItems: Ref<PreviewItem[]>
|
||||
selectedPreviewFileId: Ref<string | null>
|
||||
selectedPreviewId: Ref<string | null>
|
||||
selectedPreviewIdsByFile: Ref<Record<string, string>>
|
||||
previewSignature: Ref<string>
|
||||
dirty: Ref<boolean>
|
||||
buildPreviewConfigSignature: () => string
|
||||
buildPreviewSignature: () => string
|
||||
mapPreviewItem: (item: DataProcessPreviewItem) => PreviewItem
|
||||
resetDownstream: () => void
|
||||
}
|
||||
|
||||
async function loadSourceContent(taskId: string, fileId: string | number) {
|
||||
const chunks: string[] = []
|
||||
let startLine = 1
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
start_line: startLine,
|
||||
line_count: 10_000,
|
||||
})
|
||||
chunks.push(source.content || '')
|
||||
if (!source.has_more) break
|
||||
const nextLine = Number(source.end_line || startLine) + 1
|
||||
if (nextLine <= startLine) break
|
||||
startLine = nextLine
|
||||
}
|
||||
// source_content_lines 已保留原始换行；分页之间直接拼接，避免凭空增加空行并破坏偏移。
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
|
||||
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
const pages = Math.ceil(first.total / first.page_size)
|
||||
for (let page = 2; page <= pages; page += 1) {
|
||||
const next = await getDataProcessPreview(taskId, { page, page_size: 500 })
|
||||
items.push(...next.items)
|
||||
}
|
||||
return items.map(mapPreviewItem)
|
||||
}
|
||||
|
||||
export function useDataProcessRegeneration(bindings: RegenerationBindings) {
|
||||
const route = useRoute()
|
||||
const sourceTaskId = computed(() => (
|
||||
route.name === 'data-process-regenerate' ? String(route.params.id || '') : ''
|
||||
))
|
||||
const isRegeneration = computed(() => Boolean(sourceTaskId.value))
|
||||
const originalProcessType = ref<ProcessType | null>(null)
|
||||
const originalTaskUpdatedAt = ref('')
|
||||
const regenerationPrepared = ref(false)
|
||||
const originalPreviewConfigSignature = ref('')
|
||||
const confirmedPreviewConfigSignature = ref('')
|
||||
const hydrating = ref(false)
|
||||
const initializationError = ref('')
|
||||
|
||||
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
|
||||
const taskId = String(task.id)
|
||||
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
|
||||
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
|
||||
)))
|
||||
bindings.previewItems.value = preservePreviews
|
||||
? await loadAllPreviews(taskId, bindings.mapPreviewItem)
|
||||
: []
|
||||
|
||||
const configSignature = bindings.buildPreviewConfigSignature()
|
||||
const previewCounts = new Map<string, number>()
|
||||
for (const item of bindings.previewItems.value) {
|
||||
previewCounts.set(item.sourceFileId, (previewCounts.get(item.sourceFileId) || 0) + 1)
|
||||
}
|
||||
for (const file of bindings.uploadedFiles.value) {
|
||||
const count = previewCounts.get(String(file.sourceFileId)) || 0
|
||||
file.previewCount = count
|
||||
file.previewStatus = preservePreviews && count > 0 ? 'success' : 'waiting'
|
||||
file.previewProgress = preservePreviews && count > 0 ? 100 : 0
|
||||
file.previewConfigSignature = preservePreviews && count > 0 ? configSignature : undefined
|
||||
}
|
||||
|
||||
bindings.selectedPreviewFileId.value = String(bindings.uploadedFiles.value[0]?.uid ?? '') || null
|
||||
bindings.selectedPreviewId.value = bindings.selectedPreviewFileId.value
|
||||
? bindings.previewItems.value.find((item) => (
|
||||
item.sourceFileId === bindings.selectedPreviewFileId.value
|
||||
))?.id ?? null
|
||||
: null
|
||||
bindings.selectedPreviewIdsByFile.value = (
|
||||
bindings.selectedPreviewFileId.value && bindings.selectedPreviewId.value
|
||||
) ? { [bindings.selectedPreviewFileId.value]: bindings.selectedPreviewId.value } : {}
|
||||
bindings.previewSignature.value = preservePreviews && bindings.previewItems.value.length
|
||||
? bindings.buildPreviewSignature()
|
||||
: ''
|
||||
bindings.resetDownstream()
|
||||
}
|
||||
|
||||
async function loadSource() {
|
||||
if (!sourceTaskId.value) return
|
||||
hydrating.value = true
|
||||
initializationError.value = ''
|
||||
try {
|
||||
const sourceTask = await getDataProcessTask(sourceTaskId.value)
|
||||
const sourceType = sourceTask.process_type as ProcessType
|
||||
originalProcessType.value = sourceType
|
||||
originalTaskUpdatedAt.value = sourceTask.updated_at
|
||||
bindings.task.name = sourceTask.name
|
||||
bindings.task.description = sourceTask.description || ''
|
||||
bindings.processType.value = sourceType
|
||||
const config = sourceTask.config || {}
|
||||
bindings.structuredOptions.value = createStructuredOptionsFromConfig(config)
|
||||
bindings.unstructuredOptions.value = createUnstructuredOptionsFromConfig(config)
|
||||
originalPreviewConfigSignature.value = bindings.buildPreviewConfigSignature()
|
||||
confirmedPreviewConfigSignature.value = ''
|
||||
await hydrateWorkspace(sourceTask, true)
|
||||
await nextTick()
|
||||
bindings.dirty.value = false
|
||||
} catch (error) {
|
||||
initializationError.value = error instanceof Error
|
||||
? error.message
|
||||
: '原数据处理任务加载失败，请返回详情页后重试'
|
||||
} finally {
|
||||
hydrating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmPreviewConfigChange(
|
||||
openConfirm: (options: ConfirmOptions) => Promise<boolean | undefined>,
|
||||
) {
|
||||
if (!isRegeneration.value) return true
|
||||
if (!bindings.previewItems.value.length) return true
|
||||
const currentSignature = bindings.buildPreviewConfigSignature()
|
||||
const changed = currentSignature !== originalPreviewConfigSignature.value
|
||||
if (!changed || currentSignature === confirmedPreviewConfigSignature.value) return true
|
||||
const confirmed = await openConfirm({
|
||||
title: '确认修改切分配置？',
|
||||
message: '修改预处理或切分配置将删除现有切片和生成结果；已发布三数据集暂时保留，直到重新发布后才会更新。',
|
||||
confirmText: '确认并继续',
|
||||
cancelText: '返回检查',
|
||||
tone: 'warning',
|
||||
})
|
||||
if (confirmed) confirmedPreviewConfigSignature.value = currentSignature
|
||||
return Boolean(confirmed)
|
||||
}
|
||||
|
||||
async function prepareRegeneration(
|
||||
payload: Omit<DataProcessRegeneratePayload, 'expected_updated_at'>,
|
||||
) {
|
||||
// 第一次 prepare 后，上传、切分、编辑预览和生成都会推进任务版本。
|
||||
// 再次提交前读取同一任务的最新版本，仍由后端事务处理读取后的并发竞争。
|
||||
if (regenerationPrepared.value) {
|
||||
const latestTask = await getDataProcessTask(sourceTaskId.value)
|
||||
originalTaskUpdatedAt.value = latestTask.updated_at || originalTaskUpdatedAt.value
|
||||
}
|
||||
const regenerated = await regenerateDataProcessTask(sourceTaskId.value, {
|
||||
...payload,
|
||||
expected_updated_at: originalTaskUpdatedAt.value,
|
||||
})
|
||||
originalTaskUpdatedAt.value = regenerated.task.updated_at || originalTaskUpdatedAt.value
|
||||
originalPreviewConfigSignature.value = bindings.buildPreviewConfigSignature()
|
||||
confirmedPreviewConfigSignature.value = ''
|
||||
regenerationPrepared.value = true
|
||||
bindings.dirty.value = true
|
||||
try {
|
||||
const regeneratedTask = regenerated.task.source_files
|
||||
? regenerated.task
|
||||
: await getDataProcessTask(regenerated.task.id)
|
||||
originalTaskUpdatedAt.value = regeneratedTask.updated_at || originalTaskUpdatedAt.value
|
||||
hydrating.value = true
|
||||
try {
|
||||
await hydrateWorkspace(regeneratedTask, !regenerated.preview_invalidated)
|
||||
} finally {
|
||||
hydrating.value = false
|
||||
}
|
||||
} catch (error) {
|
||||
initializationError.value = error instanceof Error
|
||||
? `任务已进入重新生成状态，但工作区恢复失败：${error.message}`
|
||||
: '任务已进入重新生成状态，但工作区恢复失败，请重试加载原任务'
|
||||
throw error
|
||||
}
|
||||
return regenerated
|
||||
}
|
||||
|
||||
return {
|
||||
sourceTaskId,
|
||||
isRegeneration,
|
||||
originalProcessType,
|
||||
originalTaskUpdatedAt,
|
||||
regenerationPrepared,
|
||||
hydrating,
|
||||
initializationError,
|
||||
hydrateWorkspace,
|
||||
loadSource,
|
||||
confirmPreviewConfigChange,
|
||||
prepareRegeneration,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user