fix: 完善数据预处理与 JSON 上传链路
This commit is contained in:
@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
|
||||
import PreviewCompareStep from './create/PreviewCompareStep.vue'
|
||||
import GenerationStep from './create/GenerationStep.vue'
|
||||
import ResultEditorStep from './create/ResultEditorStep.vue'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
|
||||
import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel'
|
||||
import {
|
||||
createDefaultStructuredOptions,
|
||||
createDefaultUnstructuredOptions,
|
||||
@@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration'
|
||||
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
|
||||
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
useDataProcessSourceUpload,
|
||||
validateSourceFileSelection,
|
||||
@@ -33,7 +34,6 @@ import {
|
||||
deleteDataProcessPreview,
|
||||
deleteDataProcessSourceFile,
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
pullDataProcessExternalSource,
|
||||
testDataProcessExternalSource,
|
||||
updateDataProcessPreview,
|
||||
@@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false)
|
||||
let allowLeave = false
|
||||
const {
|
||||
bulkRegeneration,
|
||||
canReturnFromGeneration,
|
||||
generation,
|
||||
generationStarting,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
@@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => {
|
||||
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
|
||||
return 'fa-arrow-right'
|
||||
})
|
||||
|
||||
const previousStepLabel = computed(() => currentStep.value > 0
|
||||
? WIZARD_STEPS[currentStep.value - 1].title
|
||||
: '')
|
||||
@@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload {
|
||||
}
|
||||
|
||||
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
|
||||
const sourceLocator = item.quality_score?.source_locator
|
||||
return {
|
||||
id: String(item.id),
|
||||
sourceFileId: String(item.source_file_id),
|
||||
originalContent: item.original_content,
|
||||
editedContent: item.edited_content,
|
||||
savedEditedContent: item.edited_content,
|
||||
sourceStart: item.source_start,
|
||||
sourceEnd: item.source_end,
|
||||
sourceStartLine: item.source_start_line,
|
||||
sourceEndLine: item.source_end_line,
|
||||
sourceStart: item.source_start ?? sourceLocator?.source_start ?? null,
|
||||
sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null,
|
||||
sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null,
|
||||
sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null,
|
||||
tokenCount: item.token_count,
|
||||
status: item.status,
|
||||
sourcePages: Array.isArray(item.quality_score?.source_pages)
|
||||
? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
|
||||
: [],
|
||||
sourceLocator,
|
||||
headingPath: Array.isArray(item.quality_score?.heading_path)
|
||||
? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string')
|
||||
: [],
|
||||
updatedAt: item.updated_at,
|
||||
}
|
||||
}
|
||||
@@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) {
|
||||
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
|
||||
uploadedFiles.value.push({
|
||||
uid: localUid,
|
||||
rawFile: raw,
|
||||
name: raw.name,
|
||||
size: raw.size,
|
||||
count: 0,
|
||||
@@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) {
|
||||
previewProgress: 0,
|
||||
})
|
||||
dirty.value = true
|
||||
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
|
||||
enqueueSourceUpload({ uid: localUid, file: raw })
|
||||
}
|
||||
|
||||
async function useSampleFile() {
|
||||
@@ -488,11 +493,8 @@ async function handlePullData() {
|
||||
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
|
||||
const newFiles: UploadedDataFile[] = []
|
||||
for (const file of response.files) {
|
||||
const source = await getDataProcessSourceContent(taskId.value, file.id, {
|
||||
start_line: 1,
|
||||
line_count: 5000,
|
||||
})
|
||||
newFiles.push(mapDataProcessSourceFile(file, source.content))
|
||||
const content = await loadCanonicalSourceContent(taskId.value, file.id)
|
||||
newFiles.push(mapDataProcessSourceFile(file, content))
|
||||
}
|
||||
uploadedFiles.value.push(...newFiles)
|
||||
externalConnected.value = true
|
||||
@@ -734,9 +736,14 @@ function selectPreviewItem(id: string) {
|
||||
function updatePreviewContent(id: string, value: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item) return
|
||||
const isManual = isManualPreviewItem(item)
|
||||
item.editedContent = value
|
||||
item.tokenCount = estimateTokenCount(value)
|
||||
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
|
||||
item.status = !value.trim()
|
||||
? 'invalid'
|
||||
: value === item.originalContent
|
||||
? 'original'
|
||||
: isManual ? 'manual' : 'modified'
|
||||
resetDownstream()
|
||||
dirty.value = true
|
||||
}
|
||||
@@ -758,7 +765,7 @@ async function syncPreviewChanges() {
|
||||
|
||||
function restorePreviewItem(id: string) {
|
||||
const item = previewItems.value.find((entry) => entry.id === id)
|
||||
if (!item || item.sourceStart == null) return
|
||||
if (!item || isManualPreviewItem(item)) return
|
||||
item.editedContent = item.originalContent
|
||||
item.tokenCount = estimateTokenCount(item.originalContent)
|
||||
item.status = 'original'
|
||||
@@ -897,7 +904,7 @@ async function handleBack() {
|
||||
ElMessage.warning('请等待当前文件切分完成')
|
||||
return
|
||||
}
|
||||
if (currentStepId.value === 'generate') return
|
||||
if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return
|
||||
if (currentStep.value > 0) {
|
||||
const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id
|
||||
if (!targetStep) return
|
||||
@@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() {
|
||||
if (sourceTask.status === 'running') resumeStep = 'generate'
|
||||
if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload'
|
||||
if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate'
|
||||
if (resumeStep === 'generate' || resumeStep === 'results') {
|
||||
const resume = resumeGeneration()
|
||||
goToStep(resumeStep)
|
||||
await resume
|
||||
return
|
||||
}
|
||||
goToStep(resumeStep)
|
||||
if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration()
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
@@ -1154,7 +1166,7 @@ onMounted(() => {
|
||||
<div class="footer-left">
|
||||
<el-button
|
||||
v-if="currentStep > 0"
|
||||
:disabled="currentStepId === 'generate' || previewBuilding || sourceUploading"
|
||||
:disabled="(currentStepId === 'generate' && !canReturnFromGeneration) || previewBuilding || sourceUploading"
|
||||
@click="handleBack"
|
||||
>
|
||||
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回:{{ previousStepLabel }}
|
||||
@@ -1167,8 +1179,8 @@ onMounted(() => {
|
||||
<el-button
|
||||
class="wizard-primary-action"
|
||||
type="primary"
|
||||
:loading="modelSubmitLoading || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
|
||||
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
|
||||
@click="handlePrimaryAction"
|
||||
>
|
||||
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
getDataProcessResults,
|
||||
getDataProcessTask,
|
||||
publishDataProcess,
|
||||
repeatDataProcessTask,
|
||||
restoreDataProcessResult,
|
||||
updateDataProcessResult,
|
||||
} from '@/api/modules/dataProcess'
|
||||
@@ -42,6 +43,8 @@ const savingResult = ref(false)
|
||||
const restoringResultId = ref<string | number | null>(null)
|
||||
const publishDialogVisible = ref(false)
|
||||
const publishing = ref(false)
|
||||
const repeatGenerating = ref(false)
|
||||
const repeatRequestId = ref('')
|
||||
const configExpanded = ref(false)
|
||||
const resultCellTooltipOptions = {
|
||||
popperClass: 'data-process-result-tooltip',
|
||||
@@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record<string, string> = {
|
||||
preserve_context: '保留上下文',
|
||||
}
|
||||
|
||||
const structuredPreprocessOptionKeys = new Set([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
|
||||
function formatStructuredPreprocessOptions(value: unknown[]) {
|
||||
const options = [...new Set(value.map((item) => String(item)))]
|
||||
const selected = new Set(options)
|
||||
const consumed = new Set<string>()
|
||||
const labels: string[] = []
|
||||
|
||||
function appendGroup(values: string[], groupLabel: string) {
|
||||
const selectedValues = values.filter((item) => selected.has(item))
|
||||
selectedValues.forEach((item) => consumed.add(item))
|
||||
if (selectedValues.length === values.length) {
|
||||
labels.push(groupLabel)
|
||||
return
|
||||
}
|
||||
selectedValues.forEach((item) => {
|
||||
labels.push(`${preprocessOptionLabelMap[item] || item}(历史部分配置)`)
|
||||
})
|
||||
}
|
||||
|
||||
appendGroup(['clean_invalid', 'deduplicate'], '数据清洗')
|
||||
appendGroup(['detect_structure', 'normalize_format'], '结构标准化')
|
||||
|
||||
if (selected.has('desensitize')) {
|
||||
consumed.add('desensitize')
|
||||
labels.push('敏感信息脱敏')
|
||||
}
|
||||
if (selected.has('filter_anomaly')) {
|
||||
consumed.add('filter_anomaly')
|
||||
labels.push('异常数据过滤(历史规则)')
|
||||
}
|
||||
|
||||
options.forEach((item) => {
|
||||
if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item)
|
||||
})
|
||||
return labels.length ? labels.join('、') : '-'
|
||||
}
|
||||
|
||||
function numeric(value: unknown) {
|
||||
const parsed = typeof value === 'number' ? value : Number(value)
|
||||
return Number.isFinite(parsed) ? parsed : 0
|
||||
@@ -199,6 +247,11 @@ const canRegenerate = computed(() => {
|
||||
|| status === 'stopped'
|
||||
|| (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value))
|
||||
})
|
||||
const canRepeatGeneration = computed(() => (
|
||||
detail.value?.status === 'completed'
|
||||
&& detail.value.results_confirmed !== false
|
||||
&& previewCount.value > 0
|
||||
))
|
||||
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
|
||||
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
|
||||
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
|
||||
@@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) {
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
if (key === 'preprocess_options') {
|
||||
return value.length
|
||||
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
|
||||
: '-'
|
||||
const containsStructuredOption = value.some((item) => (
|
||||
structuredPreprocessOptionKeys.has(String(item))
|
||||
))
|
||||
return containsStructuredOption
|
||||
? formatStructuredPreprocessOptions(value)
|
||||
: value.length
|
||||
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
|
||||
: '-'
|
||||
}
|
||||
return value.length ? value.join('、') : '-'
|
||||
}
|
||||
@@ -503,6 +561,48 @@ function startRegeneration() {
|
||||
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
|
||||
}
|
||||
|
||||
function createRepeatRequestId() {
|
||||
if (typeof globalThis.crypto?.randomUUID === 'function') {
|
||||
return globalThis.crypto.randomUUID()
|
||||
}
|
||||
return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
|
||||
}
|
||||
|
||||
async function repeatGeneration() {
|
||||
if (!detail.value?.updated_at || repeatGenerating.value) return
|
||||
try {
|
||||
await ElMessageBox.confirm(
|
||||
'系统会复制当前配置、源文件和切分结果,创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。',
|
||||
'按原配置再生成一批?',
|
||||
{
|
||||
confirmButtonText: '创建并开始生成',
|
||||
cancelButtonText: '取消',
|
||||
type: 'info',
|
||||
},
|
||||
)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
repeatGenerating.value = true
|
||||
repeatRequestId.value ||= createRepeatRequestId()
|
||||
try {
|
||||
const repeated = await repeatDataProcessTask(taskId.value, {
|
||||
expected_updated_at: detail.value.updated_at,
|
||||
request_id: repeatRequestId.value,
|
||||
})
|
||||
ElMessage.success(repeated.created ? '已创建新任务,正在后台生成' : '已恢复此前创建的新任务')
|
||||
await router.push({
|
||||
name: 'data-process-workflow',
|
||||
params: { id: repeated.task.id },
|
||||
})
|
||||
} catch {
|
||||
// 保留幂等请求 ID;网络超时后再次点击不会重复创建任务。
|
||||
} finally {
|
||||
repeatGenerating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
watch([currentPage, pageSize], () => void loadResults())
|
||||
|
||||
onMounted(loadPage)
|
||||
@@ -520,22 +620,32 @@ onBeforeUnmount(() => {
|
||||
<el-tag :type="displayStatus.type" size="small" effect="light">
|
||||
{{ displayStatus.label }}
|
||||
</el-tag>
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRegenerate"
|
||||
class="publish-button"
|
||||
type="primary"
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" style="margin-right: 4px;" />重新生成
|
||||
</el-button>
|
||||
<div class="heading-actions">
|
||||
<el-button
|
||||
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
|
||||
type="primary"
|
||||
@click="openPublishDialog"
|
||||
>
|
||||
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRepeatGeneration"
|
||||
type="primary"
|
||||
:loading="repeatGenerating"
|
||||
:disabled="repeatGenerating"
|
||||
@click="repeatGeneration"
|
||||
>
|
||||
<i class="fa fa-clone" style="margin-right: 4px;" />按原配置再生成一批
|
||||
</el-button>
|
||||
<el-button
|
||||
v-if="canRegenerate"
|
||||
type="warning"
|
||||
plain
|
||||
@click="startRegeneration"
|
||||
>
|
||||
<i class="fa fa-refresh" style="margin-right: 4px;" />覆盖当前任务重新生成
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
<p>{{ detail.description || '暂无任务描述' }}</p>
|
||||
<dl class="heading-meta">
|
||||
@@ -804,7 +914,15 @@ onBeforeUnmount(() => {
|
||||
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
|
||||
}
|
||||
|
||||
.publish-button { margin-left: auto; }
|
||||
.heading-actions {
|
||||
margin-left: auto;
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: flex-end;
|
||||
gap: 8px;
|
||||
|
||||
:deep(.el-button + .el-button) { margin-left: 0; }
|
||||
}
|
||||
.load-state-actions { display: flex; gap: 10px; }
|
||||
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
|
||||
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
|
||||
@@ -968,7 +1086,8 @@ onBeforeUnmount(() => {
|
||||
@media (max-width: 720px) {
|
||||
.metric-grid, .config-grid { grid-template-columns: 1fr; }
|
||||
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
|
||||
.publish-button { width: 100%; margin-left: 0; }
|
||||
.heading-actions { width: 100%; margin-left: 0; }
|
||||
.heading-actions :deep(.el-button) { width: 100%; }
|
||||
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
|
||||
.result-toolbar { align-items: stretch; flex-direction: column; }
|
||||
.result-filters { padding: 0 16px 16px; flex-direction: column; }
|
||||
|
||||
@@ -46,6 +46,13 @@ const sourceUrl = computed(() => (
|
||||
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
|
||||
: ''
|
||||
))
|
||||
const selectedXlsxLocator = computed(() => {
|
||||
const locator = props.selectedItem?.sourceLocator
|
||||
if (!locator) return null
|
||||
const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name)
|
||||
const hasRow = locator.row_number != null || locator.sheet_record_index != null
|
||||
return hasSheet && hasRow ? locator : null
|
||||
})
|
||||
const visibleRowRange = computed(() => {
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
|
||||
@@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => {
|
||||
})
|
||||
|
||||
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
|
||||
const locator = selectedXlsxLocator.value
|
||||
const sheet = xlsxPreview.value?.active_sheet
|
||||
if (locator && sheet) {
|
||||
const sheetMatches = locator.sheet_index != null
|
||||
? sheet.index === locator.sheet_index
|
||||
: sheet.name === locator.sheet_name
|
||||
if (!sheetMatches) return false
|
||||
if (locator.row_number != null) return row.row_number === locator.row_number
|
||||
return row.record_index === locator.sheet_record_index
|
||||
}
|
||||
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
|
||||
}
|
||||
|
||||
@@ -119,8 +136,11 @@ async function locateSelectedItem() {
|
||||
async function loadPreview(options: { reset?: boolean } = {}) {
|
||||
const sequence = ++loadSequence
|
||||
if (options.reset) {
|
||||
activeSheetIndex.value = 0
|
||||
pageOffset.value = 0
|
||||
const locator = selectedXlsxLocator.value
|
||||
activeSheetIndex.value = locator?.sheet_index ?? 0
|
||||
pageOffset.value = locator?.sheet_record_index == null
|
||||
? 0
|
||||
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
|
||||
preview.value = null
|
||||
}
|
||||
errorMessage.value = ''
|
||||
@@ -178,8 +198,34 @@ watch(
|
||||
)
|
||||
|
||||
watch(
|
||||
() => props.selectedItem?.id,
|
||||
() => void locateSelectedItem(),
|
||||
() => [
|
||||
props.selectedItem?.id,
|
||||
props.selectedItem?.sourceLocator?.sheet_index,
|
||||
props.selectedItem?.sourceLocator?.sheet_record_index,
|
||||
props.selectedItem?.sourceLocator?.row_number,
|
||||
],
|
||||
() => {
|
||||
const locator = selectedXlsxLocator.value
|
||||
if (!locator || isDocx.value) {
|
||||
void locateSelectedItem()
|
||||
return
|
||||
}
|
||||
const targetSheet = locator.sheet_index ?? activeSheetIndex.value
|
||||
const targetOffset = locator.sheet_record_index == null
|
||||
? pageOffset.value
|
||||
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
|
||||
const activeSheet = xlsxPreview.value?.active_sheet
|
||||
if (
|
||||
activeSheet?.index === targetSheet
|
||||
&& activeSheet.offset === targetOffset
|
||||
) {
|
||||
void locateSelectedItem()
|
||||
return
|
||||
}
|
||||
activeSheetIndex.value = targetSheet
|
||||
pageOffset.value = targetOffset
|
||||
void loadPreview()
|
||||
},
|
||||
)
|
||||
</script>
|
||||
|
||||
@@ -301,6 +347,8 @@ watch(
|
||||
:key="row.row_number"
|
||||
class="xlsx-row"
|
||||
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
|
||||
:data-row-number="row.row_number"
|
||||
:data-record-index="row.record_index"
|
||||
>
|
||||
<th class="row-number-cell">{{ row.row_number }}</th>
|
||||
<td
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
import { computed, nextTick, ref, watch } from 'vue'
|
||||
import OfficeSourceViewer from './OfficeSourceViewer.vue'
|
||||
import PdfSourceViewer from './PdfSourceViewer.vue'
|
||||
import { sourceLines } from './previewModel'
|
||||
import {
|
||||
isManualPreviewItem,
|
||||
sourceLineNumberAtOffset,
|
||||
sourceLineWindow,
|
||||
} from './previewModel'
|
||||
import type { PreviewItem, ProcessType } from './types'
|
||||
|
||||
const props = defineProps<{
|
||||
@@ -31,9 +35,11 @@ const sourceViewerRef = ref<HTMLElement | null>(null)
|
||||
const search = ref('')
|
||||
const currentPage = ref(1)
|
||||
const PREVIEW_PAGE_SIZE = 10
|
||||
const SOURCE_LINE_RENDER_LIMIT = 240
|
||||
const SOURCE_LINE_CHARACTER_LIMIT = 4_000
|
||||
const sourceWindowStartLine = ref(1)
|
||||
const editingItemId = ref<string | null>(null)
|
||||
const editorDraft = ref('')
|
||||
const lines = computed(() => sourceLines(props.sourceText))
|
||||
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
|
||||
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
|
||||
const normalizedFileFormat = computed(() => (
|
||||
@@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => (
|
||||
))
|
||||
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
|
||||
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
|
||||
const selectedSourceOffset = computed(() => {
|
||||
const item = selectedItem.value
|
||||
return item ? sourceOffsetRange(item)?.start ?? null : null
|
||||
})
|
||||
const selectedSourceLine = computed(() => {
|
||||
const item = selectedItem.value
|
||||
if (!item) return null
|
||||
return sourceLineRange(item)?.start
|
||||
?? (selectedSourceOffset.value == null
|
||||
? null
|
||||
: sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value))
|
||||
})
|
||||
const visibleSourceWindow = computed(() => sourceLineWindow(
|
||||
props.sourceText,
|
||||
sourceWindowStartLine.value,
|
||||
SOURCE_LINE_RENDER_LIMIT,
|
||||
SOURCE_LINE_CHARACTER_LIMIT,
|
||||
selectedSourceLine.value,
|
||||
selectedSourceOffset.value,
|
||||
))
|
||||
const lines = computed(() => visibleSourceWindow.value.lines)
|
||||
|
||||
const filteredItems = computed(() => props.items.filter((item, index) => {
|
||||
const matchesSearch = !search.value.trim()
|
||||
@@ -58,10 +85,27 @@ const pagedItems = computed(() => {
|
||||
|
||||
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
|
||||
|
||||
function isLineHighlighted(lineStart: number, lineEnd: number) {
|
||||
function sourceLineRange(item: PreviewItem) {
|
||||
const start = item.sourceLocator?.start_line ?? item.sourceStartLine
|
||||
const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start
|
||||
return start == null ? null : { start, end: end ?? start }
|
||||
}
|
||||
|
||||
function sourceOffsetRange(item: PreviewItem) {
|
||||
const start = item.sourceLocator?.source_start ?? item.sourceStart
|
||||
const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start
|
||||
return start == null ? null : { start, end: Math.max(start, end ?? start) }
|
||||
}
|
||||
|
||||
function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) {
|
||||
const item = selectedItem.value
|
||||
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
|
||||
return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd
|
||||
if (!item) return false
|
||||
const lineRange = sourceLineRange(item)
|
||||
if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end
|
||||
const offsetRange = sourceOffsetRange(item)
|
||||
if (!offsetRange) return false
|
||||
const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end)
|
||||
return lineEnd >= offsetRange.start && lineStart < effectiveEnd
|
||||
}
|
||||
|
||||
function selectItem(id: string) {
|
||||
@@ -107,34 +151,88 @@ watch(search, () => {
|
||||
|
||||
watch(() => props.selectedFileId, closeEditor)
|
||||
|
||||
watch(selectedItem, async (item) => {
|
||||
watch([selectedItem, () => props.sourceText], async ([item]) => {
|
||||
if (!item) return
|
||||
const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id)
|
||||
if (visibleIndex >= 0) {
|
||||
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
|
||||
}
|
||||
|
||||
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
|
||||
if (isPdfSource.value || isOfficeSource.value) return
|
||||
const itemLineRange = sourceLineRange(item)
|
||||
const itemOffsetRange = sourceOffsetRange(item)
|
||||
if (!itemLineRange && !itemOffsetRange) {
|
||||
sourceWindowStartLine.value = 1
|
||||
return
|
||||
}
|
||||
const targetLine = selectedSourceLine.value
|
||||
?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0)
|
||||
sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3))
|
||||
await nextTick()
|
||||
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
|
||||
const exactTarget = sourceViewerRef.value
|
||||
?.querySelector<HTMLElement>(`[data-line-number="${targetLine}"]`)
|
||||
const target = exactTarget
|
||||
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
|
||||
target?.scrollIntoView({ block: 'center', behavior: 'smooth' })
|
||||
}, { immediate: true })
|
||||
|
||||
async function showPreviousSourceWindow() {
|
||||
sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT)
|
||||
await nextTick()
|
||||
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
|
||||
}
|
||||
|
||||
async function showNextSourceWindow() {
|
||||
if (!visibleSourceWindow.value.hasMore) return
|
||||
sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1
|
||||
await nextTick()
|
||||
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
|
||||
}
|
||||
|
||||
function itemNumber(item: PreviewItem) {
|
||||
return props.items.findIndex((entry) => entry.id === item.id) + 1
|
||||
}
|
||||
|
||||
function lineRange(item: PreviewItem) {
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
return first === last ? `来源:第 ${first} 页` : `来源:第 ${first}–${last} 页`
|
||||
if (isManualPreviewItem(item)) return '手动新增,无源文件定位'
|
||||
|
||||
const locator = item.sourceLocator
|
||||
const locatedLines = sourceLineRange(item)
|
||||
if (props.processType === 'unstructured') {
|
||||
const parts: string[] = []
|
||||
if (item.sourcePages?.length) {
|
||||
const first = item.sourcePages[0]
|
||||
const last = item.sourcePages[item.sourcePages.length - 1]
|
||||
parts.push(first === last ? `第 ${first} 页` : `第 ${first}–${last} 页`)
|
||||
}
|
||||
if (locatedLines) {
|
||||
parts.push(
|
||||
locatedLines.start === locatedLines.end
|
||||
? `第 ${locatedLines.start} 行`
|
||||
: `第 ${locatedLines.start}–${locatedLines.end} 行`,
|
||||
)
|
||||
}
|
||||
if (item.headingPath?.length) parts.push(`章节:${item.headingPath.join(' / ')}`)
|
||||
return parts.length ? `来源:${parts.join(' · ')}` : '来源:源文件内容(无精确定位)'
|
||||
}
|
||||
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增,无源文件定位'
|
||||
return item.sourceStartLine === item.sourceEndLine
|
||||
? `来源:第 ${item.sourceStartLine} 行`
|
||||
: `来源:第 ${item.sourceStartLine}–${item.sourceEndLine} 行`
|
||||
|
||||
if (locator?.kind === 'xlsx') {
|
||||
const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}`
|
||||
return locator.row_number != null
|
||||
? `来源:${sheet} · 第 ${locator.row_number} 行`
|
||||
: `来源:${sheet}`
|
||||
}
|
||||
if (locator?.kind === 'json') {
|
||||
return locator.json_pointer
|
||||
? `来源:JSON 路径 ${locator.json_pointer}`
|
||||
: '来源:JSON 根对象'
|
||||
}
|
||||
if (locatedLines) {
|
||||
return locatedLines.start === locatedLines.end
|
||||
? `来源:第 ${locatedLines.start} 行`
|
||||
: `来源:第 ${locatedLines.start}–${locatedLines.end} 行`
|
||||
}
|
||||
return '来源:源文件记录'
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
|
||||
<div>
|
||||
<strong>源文件 · {{ fileName }}</strong>
|
||||
</div>
|
||||
<div
|
||||
v-if="!isPdfSource && !isOfficeSource && lines.length"
|
||||
class="source-window-controls"
|
||||
aria-label="源文件行窗口"
|
||||
>
|
||||
<span>第 {{ visibleSourceWindow.startLine }}–{{ visibleSourceWindow.endLine }} 行</span>
|
||||
<el-button
|
||||
link
|
||||
size="small"
|
||||
aria-label="查看上一段源文件"
|
||||
:disabled="!visibleSourceWindow.hasPrevious"
|
||||
@click="showPreviousSourceWindow"
|
||||
>
|
||||
上一段
|
||||
</el-button>
|
||||
<el-button
|
||||
link
|
||||
size="small"
|
||||
aria-label="查看下一段源文件"
|
||||
:disabled="!visibleSourceWindow.hasMore"
|
||||
@click="showNextSourceWindow"
|
||||
>
|
||||
下一段
|
||||
</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<PdfSourceViewer
|
||||
@@ -197,8 +320,9 @@ function lineRange(item: PreviewItem) {
|
||||
v-for="line in lines"
|
||||
:key="line.number"
|
||||
class="source-line"
|
||||
:class="{ 'is-highlighted': isLineHighlighted(line.start, line.end) }"
|
||||
:class="{ 'is-highlighted': isLineHighlighted(line.number, line.start, line.end) }"
|
||||
:data-source-start="line.start"
|
||||
:data-line-number="line.number"
|
||||
>
|
||||
<span class="line-number">{{ line.number }}</span>
|
||||
<span class="line-content">{{ line.content || ' ' }}</span>
|
||||
@@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) {
|
||||
/>
|
||||
<div class="editor-actions">
|
||||
<el-button
|
||||
v-if="editingItem.sourceStart != null"
|
||||
v-if="!isManualPreviewItem(editingItem)"
|
||||
link
|
||||
@click="restoreItem"
|
||||
>
|
||||
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) {
|
||||
}
|
||||
}
|
||||
|
||||
.source-window-controls {
|
||||
flex: none;
|
||||
gap: 2px !important;
|
||||
|
||||
> span {
|
||||
margin-right: 4px;
|
||||
color: #8a93a3;
|
||||
font-size: 11px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
:deep(.el-button) {
|
||||
margin-left: 0;
|
||||
}
|
||||
}
|
||||
|
||||
.source-viewer {
|
||||
flex: 1;
|
||||
height: 538px;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { computed } from 'vue'
|
||||
import type {
|
||||
GenerationControlOptions,
|
||||
PreprocessOption,
|
||||
@@ -21,27 +22,32 @@ const emit = defineEmits<{
|
||||
'update:options': [value: StructuredProcessOptions]
|
||||
}>()
|
||||
|
||||
const PREPROCESS_OPTIONS: Array<{
|
||||
value: PreprocessOption
|
||||
const PREPROCESS_GROUPS: Array<{
|
||||
values: PreprocessOption[]
|
||||
label: string
|
||||
description: string
|
||||
}> = [
|
||||
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
|
||||
{
|
||||
value: 'detect_structure',
|
||||
label: '嵌套结构展平',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段;Excel 表头与合并单元格在上传时自动解析',
|
||||
values: ['clean_invalid', 'deduplicate'],
|
||||
label: '数据清洗',
|
||||
description: '清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
|
||||
},
|
||||
{
|
||||
value: 'deduplicate',
|
||||
label: '重复记录去重',
|
||||
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
|
||||
values: ['detect_structure', 'normalize_format'],
|
||||
label: '结构标准化',
|
||||
description: '展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
|
||||
},
|
||||
{
|
||||
values: ['desensitize'],
|
||||
label: '敏感信息脱敏',
|
||||
description: '识别并脱敏姓名、手机号、邮箱和身份证号',
|
||||
},
|
||||
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
|
||||
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
|
||||
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
|
||||
]
|
||||
|
||||
const legacyAnomalyFilterEnabled = computed(() => (
|
||||
props.options.preprocessOptions.includes('filter_anomaly')
|
||||
))
|
||||
|
||||
function updateField<K extends keyof StructuredProcessOptions>(
|
||||
field: K,
|
||||
value: StructuredProcessOptions[K],
|
||||
@@ -57,14 +63,26 @@ function updateQaPairsPerRow(value: number | undefined) {
|
||||
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
|
||||
}
|
||||
|
||||
function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
|
||||
const preprocessOptions = Array.from(new Set(value.filter(
|
||||
(option): option is PreprocessOption => (
|
||||
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
|
||||
),
|
||||
)))
|
||||
updateField('preprocessOptions', preprocessOptions)
|
||||
function selectedCount(values: PreprocessOption[]) {
|
||||
return values.filter((value) => props.options.preprocessOptions.includes(value)).length
|
||||
}
|
||||
|
||||
function groupSelected(values: PreprocessOption[]) {
|
||||
return selectedCount(values) === values.length
|
||||
}
|
||||
|
||||
function groupIndeterminate(values: PreprocessOption[]) {
|
||||
const count = selectedCount(values)
|
||||
return count > 0 && count < values.length
|
||||
}
|
||||
|
||||
function updatePreprocessGroup(values: PreprocessOption[], checked: string | number | boolean) {
|
||||
const next = new Set(props.options.preprocessOptions)
|
||||
values.forEach((value) => {
|
||||
if (Boolean(checked)) next.add(value)
|
||||
else next.delete(value)
|
||||
})
|
||||
updateField('preprocessOptions', [...next])
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>选择在生成问答对之前需要执行的数据处理方式</p>
|
||||
<p>默认不执行预处理,请按数据情况自行选择</p>
|
||||
</div>
|
||||
</div>
|
||||
<el-checkbox-group
|
||||
:model-value="options.preprocessOptions"
|
||||
class="preprocess-option-grid"
|
||||
@update:model-value="updatePreprocessOptions"
|
||||
>
|
||||
<el-checkbox
|
||||
v-for="option in PREPROCESS_OPTIONS"
|
||||
:key="option.value"
|
||||
:value="option.value"
|
||||
<div class="preprocess-option-grid">
|
||||
<label
|
||||
v-for="group in PREPROCESS_GROUPS"
|
||||
:key="group.label"
|
||||
class="preprocess-option"
|
||||
:class="{ 'is-checked': groupSelected(group.values) }"
|
||||
>
|
||||
<el-checkbox
|
||||
:model-value="groupSelected(group.values)"
|
||||
:indeterminate="groupIndeterminate(group.values)"
|
||||
@update:model-value="updatePreprocessGroup(group.values, $event)"
|
||||
/>
|
||||
<span class="preprocess-option-copy">
|
||||
<strong>{{ option.label }}</strong>
|
||||
<small>{{ option.description }}</small>
|
||||
<strong>{{ group.label }}</strong>
|
||||
<small>{{ group.description }}</small>
|
||||
</span>
|
||||
</el-checkbox>
|
||||
</el-checkbox-group>
|
||||
</label>
|
||||
</div>
|
||||
<el-alert
|
||||
v-if="legacyAnomalyFilterEnabled"
|
||||
class="legacy-preprocess-alert"
|
||||
type="warning"
|
||||
:closable="false"
|
||||
title="该历史任务仍启用了已停用的“异常数据过滤”;为保证结果可复现,本次继续保留"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div class="form-section generation-options-section">
|
||||
|
||||
@@ -119,7 +119,7 @@ defineExpose({ revealValidation })
|
||||
<div class="section-title-row">
|
||||
<div>
|
||||
<h3>预处理选项</h3>
|
||||
<p>默认启用结构感知的推荐策略,只需决定是否需要脱敏</p>
|
||||
<p>默认不执行预处理,请按文档情况自行选择</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="preprocess-option-grid">
|
||||
|
||||
@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
|
||||
|
||||
export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
|
||||
preprocessOptions: [],
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerRow: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
|
||||
|
||||
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
|
||||
return {
|
||||
preprocessOptions: [
|
||||
'clean_invalid_content',
|
||||
'detect_document_structure',
|
||||
'merge_short_content',
|
||||
'filter_low_quality',
|
||||
'deduplicate_content',
|
||||
'preserve_context',
|
||||
],
|
||||
preprocessOptions: [],
|
||||
chunkMethod: 'layout_hybrid',
|
||||
chunkSize: 800,
|
||||
chunkOverlap: 100,
|
||||
minChunkSize: 100,
|
||||
semanticBreakpointPercentile: 95,
|
||||
preserveTables: true,
|
||||
preserveCodeBlocks: true,
|
||||
preserveLists: true,
|
||||
preserveTables: false,
|
||||
preserveCodeBlocks: false,
|
||||
preserveLists: false,
|
||||
semanticEnrichment: false,
|
||||
qaPairsPerChunk: 1,
|
||||
datasetSplit: { train: 80, validation: 10, test: 10 },
|
||||
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
|
||||
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
|
||||
const defaults = createDefaultStructuredOptions()
|
||||
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
|
||||
const supportedPreprocessOptions = new Set<PreprocessOption>([
|
||||
'clean_invalid',
|
||||
'deduplicate',
|
||||
'detect_structure',
|
||||
'normalize_format',
|
||||
'desensitize',
|
||||
'filter_anomaly',
|
||||
])
|
||||
return {
|
||||
...defaults,
|
||||
...generationOptionsFromConfig(config, defaults),
|
||||
preprocessOptions: Array.isArray(preprocessOptions)
|
||||
? preprocessOptions.map(String) as PreprocessOption[]
|
||||
? Array.from(new Set(preprocessOptions.map(String).filter(
|
||||
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
|
||||
)))
|
||||
: defaults.preprocessOptions,
|
||||
semanticEnrichment: Boolean(configValue(
|
||||
config,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { SourceLine } from './types'
|
||||
import type { PreviewItem, SourceLine } from './types'
|
||||
|
||||
/** 仅用于“使用示例”上传;正式预览和切片全部由后端生成。 */
|
||||
export const DEFAULT_SOURCE_TEXT = [
|
||||
@@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [
|
||||
'答:复利是将上一期利息加入本金,再计算下一期利息。',
|
||||
].join('\n')
|
||||
|
||||
/**
|
||||
* 把后端返回的字符偏移映射为源文件行,仅负责界面高亮,不参与切片。
|
||||
*/
|
||||
export function sourceLines(sourceText: string): SourceLine[] {
|
||||
const rawLines = sourceText.split('\n')
|
||||
let cursor = 0
|
||||
export interface SourceLineWindow {
|
||||
lines: SourceLine[]
|
||||
startLine: number
|
||||
endLine: number
|
||||
hasPrevious: boolean
|
||||
hasMore: boolean
|
||||
}
|
||||
|
||||
return rawLines.map((content, index) => {
|
||||
const start = cursor
|
||||
const end = start + content.length
|
||||
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
|
||||
return { number: index + 1, content, start, end }
|
||||
})
|
||||
function unicodeCodePointLength(value: string, start = 0, end = value.length) {
|
||||
let length = 0
|
||||
let index = start
|
||||
while (index < end) {
|
||||
const codePoint = value.codePointAt(index)
|
||||
index += codePoint != null && codePoint > 0xffff ? 2 : 1
|
||||
length += 1
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
function advanceCodePoints(value: string, start: number, end: number, count: number) {
|
||||
let index = start
|
||||
let remaining = Math.max(0, count)
|
||||
while (index < end && remaining > 0) {
|
||||
const codePoint = value.codePointAt(index)
|
||||
index += codePoint != null && codePoint > 0xffff ? 2 : 1
|
||||
remaining -= 1
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
/**
|
||||
* 只扫描并返回当前可见行窗口,不对全文 split,避免大文件生成巨量字符串数组。
|
||||
* 字符定位场景可开启 code point 偏移,以与后端 Python 的字符计数保持一致。
|
||||
*/
|
||||
export function sourceLineWindow(
|
||||
sourceText: string,
|
||||
requestedStartLine: number,
|
||||
maxLines: number,
|
||||
maxCharactersPerLine: number,
|
||||
focusLine: number | null = null,
|
||||
focusOffset: number | null = null,
|
||||
): SourceLineWindow {
|
||||
const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1)
|
||||
const limit = Math.max(1, Math.trunc(maxLines) || 1)
|
||||
const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1)
|
||||
const trackUnicodeOffsets = focusOffset != null
|
||||
const lines: SourceLine[] = []
|
||||
let lineNumber = 1
|
||||
let jsCursor = 0
|
||||
let sourceCursor = 0
|
||||
|
||||
while (jsCursor <= sourceText.length && lineNumber < startLine) {
|
||||
const newlineIndex = sourceText.indexOf('\n', jsCursor)
|
||||
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
|
||||
sourceCursor = trackUnicodeOffsets
|
||||
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0)
|
||||
: (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1)
|
||||
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
|
||||
lineNumber += 1
|
||||
}
|
||||
|
||||
while (jsCursor <= sourceText.length && lines.length < limit) {
|
||||
const newlineIndex = sourceText.indexOf('\n', jsCursor)
|
||||
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
|
||||
const fullSourceEnd = trackUnicodeOffsets
|
||||
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd)
|
||||
: jsEnd
|
||||
const focusedStart = focusLine === lineNumber && focusOffset != null
|
||||
? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3))
|
||||
: sourceCursor
|
||||
const segmentSourceStart = Math.min(
|
||||
focusedStart,
|
||||
Math.max(sourceCursor, fullSourceEnd - characterLimit),
|
||||
)
|
||||
const relativeSegmentStart = trackUnicodeOffsets
|
||||
? segmentSourceStart - sourceCursor
|
||||
: Math.max(0, segmentSourceStart - jsCursor)
|
||||
const segmentJsStart = advanceCodePoints(
|
||||
sourceText,
|
||||
jsCursor,
|
||||
jsEnd,
|
||||
relativeSegmentStart,
|
||||
)
|
||||
const segmentJsEnd = advanceCodePoints(
|
||||
sourceText,
|
||||
segmentJsStart,
|
||||
jsEnd,
|
||||
characterLimit,
|
||||
)
|
||||
const segmentLength = trackUnicodeOffsets
|
||||
? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd)
|
||||
: segmentJsEnd - segmentJsStart
|
||||
const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart
|
||||
const end = start + segmentLength
|
||||
const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}`
|
||||
lines.push({ number: lineNumber, content, start, end })
|
||||
sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0)
|
||||
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
|
||||
lineNumber += 1
|
||||
}
|
||||
|
||||
return {
|
||||
lines,
|
||||
startLine: lines[0]?.number ?? startLine,
|
||||
endLine: lines[lines.length - 1]?.number ?? startLine,
|
||||
hasPrevious: startLine > 1,
|
||||
hasMore: jsCursor <= sourceText.length,
|
||||
}
|
||||
}
|
||||
|
||||
/** 根据后端 code point 偏移查找物理行号,不构建全文行数组。 */
|
||||
export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) {
|
||||
const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0)
|
||||
let offset = 0
|
||||
let lineNumber = 1
|
||||
for (const character of sourceText) {
|
||||
if (offset >= normalizedOffset) break
|
||||
if (character === '\n') lineNumber += 1
|
||||
offset += 1
|
||||
}
|
||||
return lineNumber
|
||||
}
|
||||
|
||||
/**
|
||||
* 手动新增项可能先以空内容保存为 invalid,编辑后又由后端标记为 modified,
|
||||
* 因此不能只依赖可变的 status;空原文且完全没有来源定位才是稳定兜底。
|
||||
*/
|
||||
export function isManualPreviewItem(item: PreviewItem): boolean {
|
||||
const hasSourceLocation = item.sourceStart != null
|
||||
|| item.sourceEnd != null
|
||||
|| item.sourceStartLine != null
|
||||
|| item.sourceEndLine != null
|
||||
|| Boolean(item.sourcePages?.length)
|
||||
|| Boolean(item.sourceLocator)
|
||||
return item.status === 'manual' || (!item.originalContent && !hasSourceLocation)
|
||||
}
|
||||
|
||||
/** 与后端预览 token 估算规则一致,仅用于编辑中的即时计数。 */
|
||||
|
||||
@@ -24,6 +24,7 @@ export type PreprocessOption =
|
||||
| 'detect_structure'
|
||||
| 'deduplicate'
|
||||
| 'normalize_format'
|
||||
/** 仅用于恢复历史任务,新任务界面不再提供。 */
|
||||
| 'filter_anomaly'
|
||||
| 'desensitize'
|
||||
|
||||
@@ -94,7 +95,6 @@ export interface ExternalDataSource {
|
||||
export interface UploadedDataFile {
|
||||
uid: string | number
|
||||
sourceFileId?: string
|
||||
rawFile?: File
|
||||
name: string
|
||||
size: number
|
||||
count: number
|
||||
@@ -118,6 +118,22 @@ export interface SourceLine {
|
||||
end: number
|
||||
}
|
||||
|
||||
export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
|
||||
|
||||
export interface PreviewSourceLocator {
|
||||
kind: PreviewSourceLocatorKind
|
||||
record_index?: number | null
|
||||
start_line?: number | null
|
||||
end_line?: number | null
|
||||
source_start?: number | null
|
||||
source_end?: number | null
|
||||
json_pointer?: string | null
|
||||
sheet_index?: number | null
|
||||
sheet_name?: string | null
|
||||
row_number?: number | null
|
||||
sheet_record_index?: number | null
|
||||
}
|
||||
|
||||
export interface PreviewItem {
|
||||
id: string
|
||||
sourceFileId: string
|
||||
@@ -129,6 +145,8 @@ export interface PreviewItem {
|
||||
sourceStartLine: number | null
|
||||
sourceEndLine: number | null
|
||||
sourcePages?: number[]
|
||||
sourceLocator?: PreviewSourceLocator
|
||||
headingPath?: string[]
|
||||
tokenCount: number
|
||||
status: 'original' | 'modified' | 'manual' | 'invalid'
|
||||
qualityScore?: number
|
||||
|
||||
@@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
let generationTimer: ReturnType<typeof setTimeout> | null = null
|
||||
let generationRun = 0
|
||||
let pollFailureCount = 0
|
||||
let generationStarting = false
|
||||
const generationStarting = ref(false)
|
||||
const generationRestoring = ref(false)
|
||||
const canReturnFromGeneration = computed(() => (
|
||||
generation.status === 'idle' && !generationStarting.value && !generationRestoring.value
|
||||
))
|
||||
|
||||
function stopGenerationTimer() {
|
||||
generationRun += 1
|
||||
@@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
}
|
||||
|
||||
async function startGeneration() {
|
||||
if (generationStarting || generation.status === 'running') return false
|
||||
if (generationStarting.value || generation.status === 'running') return false
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) {
|
||||
ElMessage.error('任务尚未创建,请返回上一步重试')
|
||||
return false
|
||||
}
|
||||
|
||||
generationStarting = true
|
||||
generationStarting.value = true
|
||||
let runId: number | null = null
|
||||
try {
|
||||
const canStart = await bindings.beforeGenerate?.()
|
||||
@@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
generation.message = error instanceof Error ? error.message : '启动数据处理失败,请重试。'
|
||||
return false
|
||||
} finally {
|
||||
generationStarting = false
|
||||
generationStarting.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function resumeGeneration() {
|
||||
const taskId = bindings.taskId.value
|
||||
if (!taskId) return
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
generationRestoring.value = true
|
||||
try {
|
||||
stopGenerationTimer()
|
||||
const activeRunId = generationRun
|
||||
pollFailureCount = 0
|
||||
const progress = await getDataProcessProgress(taskId)
|
||||
if (activeRunId !== generationRun) return
|
||||
if (progress.status === 'running') {
|
||||
@@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
} catch (error) {
|
||||
generation.status = 'failed'
|
||||
generation.message = error instanceof Error ? error.message : '查询任务进度失败,请重试。'
|
||||
} finally {
|
||||
generationRestoring.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
|
||||
|
||||
return {
|
||||
bulkRegeneration,
|
||||
canReturnFromGeneration,
|
||||
generation,
|
||||
generationStarting,
|
||||
regeneratingResultId,
|
||||
resultRegenerationBusy,
|
||||
results,
|
||||
|
||||
@@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import {
|
||||
getDataProcessPreview,
|
||||
getDataProcessSourceContent,
|
||||
getDataProcessTask,
|
||||
regenerateDataProcessTask,
|
||||
} from '@/api/modules/dataProcess'
|
||||
@@ -15,7 +14,10 @@ import {
|
||||
createStructuredOptionsFromConfig,
|
||||
createUnstructuredOptionsFromConfig,
|
||||
} from './dataProcessCreateState'
|
||||
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
|
||||
import {
|
||||
loadCanonicalSourceContent,
|
||||
mapDataProcessSourceFile,
|
||||
} from './useDataProcessSourceUpload'
|
||||
import type {
|
||||
PreviewItem,
|
||||
ProcessType,
|
||||
@@ -50,24 +52,6 @@ interface RegenerationBindings {
|
||||
resetDownstream: () => void
|
||||
}
|
||||
|
||||
async function loadSourceContent(taskId: string, fileId: string | number) {
|
||||
const chunks: string[] = []
|
||||
let startLine = 1
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
start_line: startLine,
|
||||
line_count: 10_000,
|
||||
})
|
||||
chunks.push(source.content || '')
|
||||
if (!source.has_more) break
|
||||
const nextLine = Number(source.end_line || startLine) + 1
|
||||
if (nextLine <= startLine) break
|
||||
startLine = nextLine
|
||||
}
|
||||
// source_content_lines 已保留原始换行;分页之间直接拼接,避免凭空增加空行并破坏偏移。
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
|
||||
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
|
||||
const items = [...first.items]
|
||||
@@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) {
|
||||
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
|
||||
const taskId = String(task.id)
|
||||
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
|
||||
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
|
||||
mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id))
|
||||
)))
|
||||
bindings.previewItems.value = preservePreviews
|
||||
? await loadAllPreviews(taskId, bindings.mapPreviewItem)
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
} from '@/api/modules/dataProcess'
|
||||
import type { ProcessType, UploadedDataFile } from './types'
|
||||
|
||||
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
|
||||
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
|
||||
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
|
||||
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
|
||||
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
|
||||
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
|
||||
const MAX_SOURCE_FILE_COUNT = 20
|
||||
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
|
||||
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
|
||||
|
||||
interface SourceUploadJob {
|
||||
uid: string
|
||||
file: File
|
||||
extension: string
|
||||
}
|
||||
|
||||
interface SourceUploadOptions {
|
||||
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
|
||||
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
|
||||
}
|
||||
}
|
||||
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
|
||||
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
|
||||
}
|
||||
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
|
||||
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
|
||||
}
|
||||
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
|
||||
return { valid: true, extension }
|
||||
}
|
||||
|
||||
function unicodeCodePointLength(value: string) {
|
||||
let length = 0
|
||||
for (const _character of value) length += 1
|
||||
return length
|
||||
}
|
||||
|
||||
/** 分页读取服务端保存的规范化正文,避免重新使用浏览器本地解码结果。 */
|
||||
export async function loadCanonicalSourceContent(
|
||||
taskId: string | number,
|
||||
fileId: string | number,
|
||||
) {
|
||||
const chunks: string[] = []
|
||||
let offset = 0
|
||||
while (true) {
|
||||
const source = await getDataProcessSourceContent(taskId, fileId, {
|
||||
offset,
|
||||
limit: SOURCE_CONTENT_PAGE_CHARS,
|
||||
})
|
||||
const content = source.content || ''
|
||||
chunks.push(content)
|
||||
if (!source.has_more) break
|
||||
const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
|
||||
if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常,请删除文件后重试')
|
||||
offset = nextOffset
|
||||
}
|
||||
return chunks.join('')
|
||||
}
|
||||
|
||||
export function mapDataProcessSourceFile(
|
||||
file: DataProcessSourceFile,
|
||||
content = '',
|
||||
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
pending.uploadError = undefined
|
||||
|
||||
try {
|
||||
let content = ''
|
||||
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
|
||||
} catch {
|
||||
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
|
||||
}
|
||||
if (!content.trim()) throw new Error('不能上传空文件')
|
||||
}
|
||||
|
||||
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
|
||||
pending.uploadProgress = progress
|
||||
})
|
||||
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
|
||||
|
||||
// 先登记后端 ID,确保正文读取失败时仍可正确删除已落库的文件。
|
||||
Object.assign(pending, mapDataProcessSourceFile(source), {
|
||||
rawFile: job.file,
|
||||
status: 'uploading',
|
||||
uploadProgress: 99,
|
||||
})
|
||||
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
|
||||
try {
|
||||
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
|
||||
start_line: 1,
|
||||
line_count: 10_000,
|
||||
})
|
||||
pending.content = parsed.content
|
||||
} catch {
|
||||
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
|
||||
}
|
||||
} else {
|
||||
pending.content = content
|
||||
try {
|
||||
pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
|
||||
} catch {
|
||||
throw new Error('文件已上传,但服务端规范化内容读取失败,请删除文件后重试')
|
||||
}
|
||||
|
||||
pending.status = 'ready'
|
||||
|
||||
Reference in New Issue
Block a user