fix: 完善数据预处理与 JSON 上传链路

This commit is contained in:
caoxiaozhu
2026-07-30 16:53:54 +08:00
parent f917a025e1
commit b975de02da
25 changed files with 3277 additions and 419 deletions

View File

@@ -10,7 +10,7 @@ import SourceUploadStep from './create/SourceUploadStep.vue'
import PreviewCompareStep from './create/PreviewCompareStep.vue'
import GenerationStep from './create/GenerationStep.vue'
import ResultEditorStep from './create/ResultEditorStep.vue'
import { DEFAULT_SOURCE_TEXT, estimateTokenCount } from './create/previewModel'
import { DEFAULT_SOURCE_TEXT, estimateTokenCount, isManualPreviewItem } from './create/previewModel'
import {
createDefaultStructuredOptions,
createDefaultUnstructuredOptions,
@@ -21,6 +21,7 @@ import { useDataProcessGeneration } from './create/useDataProcessGeneration'
import { useDataProcessPreviewBuild } from './create/useDataProcessPreviewBuild'
import { useDataProcessRegeneration } from './create/useDataProcessRegeneration'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
useDataProcessSourceUpload,
validateSourceFileSelection,
@@ -33,7 +34,6 @@ import {
deleteDataProcessPreview,
deleteDataProcessSourceFile,
getDataProcessPreview,
getDataProcessSourceContent,
pullDataProcessExternalSource,
testDataProcessExternalSource,
updateDataProcessPreview,
@@ -116,7 +116,9 @@ const modelSubmitLoading = ref(false)
let allowLeave = false
const {
bulkRegeneration,
canReturnFromGeneration,
generation,
generationStarting,
regeneratingResultId,
resultRegenerationBusy,
results,
@@ -189,7 +191,6 @@ const primaryActionIcon = computed(() => {
if (currentStepId.value === 'generate' && generation.status !== 'success') return 'fa-play'
return 'fa-arrow-right'
})
const previousStepLabel = computed(() => currentStep.value > 0
? WIZARD_STEPS[currentStep.value - 1].title
: '')
@@ -290,21 +291,26 @@ function externalPayload(): DataProcessExternalSourcePayload {
}
function mapPreviewItem(item: DataProcessPreviewItem): PreviewItem {
const sourceLocator = item.quality_score?.source_locator
return {
id: String(item.id),
sourceFileId: String(item.source_file_id),
originalContent: item.original_content,
editedContent: item.edited_content,
savedEditedContent: item.edited_content,
sourceStart: item.source_start,
sourceEnd: item.source_end,
sourceStartLine: item.source_start_line,
sourceEndLine: item.source_end_line,
sourceStart: item.source_start ?? sourceLocator?.source_start ?? null,
sourceEnd: item.source_end ?? sourceLocator?.source_end ?? null,
sourceStartLine: item.source_start_line ?? sourceLocator?.start_line ?? null,
sourceEndLine: item.source_end_line ?? sourceLocator?.end_line ?? null,
tokenCount: item.token_count,
status: item.status,
sourcePages: Array.isArray(item.quality_score?.source_pages)
? item.quality_score.source_pages.filter((value): value is number => typeof value === 'number')
: [],
sourceLocator,
headingPath: Array.isArray(item.quality_score?.heading_path)
? item.quality_score.heading_path.filter((value): value is string => typeof value === 'string')
: [],
updatedAt: item.updated_at,
}
}
@@ -419,7 +425,6 @@ function handleFileChange(uploadFile: UploadFile) {
const localUid = `local-${uploadFile.uid}-${Date.now()}-${uploadedFiles.value.length}`
uploadedFiles.value.push({
uid: localUid,
rawFile: raw,
name: raw.name,
size: raw.size,
count: 0,
@@ -431,7 +436,7 @@ function handleFileChange(uploadFile: UploadFile) {
previewProgress: 0,
})
dirty.value = true
enqueueSourceUpload({ uid: localUid, file: raw, extension: validation.extension })
enqueueSourceUpload({ uid: localUid, file: raw })
}
async function useSampleFile() {
@@ -488,11 +493,8 @@ async function handlePullData() {
const response = await pullDataProcessExternalSource(taskId.value, externalPayload())
const newFiles: UploadedDataFile[] = []
for (const file of response.files) {
const source = await getDataProcessSourceContent(taskId.value, file.id, {
start_line: 1,
line_count: 5000,
})
newFiles.push(mapDataProcessSourceFile(file, source.content))
const content = await loadCanonicalSourceContent(taskId.value, file.id)
newFiles.push(mapDataProcessSourceFile(file, content))
}
uploadedFiles.value.push(...newFiles)
externalConnected.value = true
@@ -734,9 +736,14 @@ function selectPreviewItem(id: string) {
function updatePreviewContent(id: string, value: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item) return
const isManual = isManualPreviewItem(item)
item.editedContent = value
item.tokenCount = estimateTokenCount(value)
item.status = value === item.originalContent ? 'original' : item.sourceStart == null ? 'manual' : 'modified'
item.status = !value.trim()
? 'invalid'
: value === item.originalContent
? 'original'
: isManual ? 'manual' : 'modified'
resetDownstream()
dirty.value = true
}
@@ -758,7 +765,7 @@ async function syncPreviewChanges() {
function restorePreviewItem(id: string) {
const item = previewItems.value.find((entry) => entry.id === id)
if (!item || item.sourceStart == null) return
if (!item || isManualPreviewItem(item)) return
item.editedContent = item.originalContent
item.tokenCount = estimateTokenCount(item.originalContent)
item.status = 'original'
@@ -897,7 +904,7 @@ async function handleBack() {
ElMessage.warning('请等待当前文件切分完成')
return
}
if (currentStepId.value === 'generate') return
if (currentStepId.value === 'generate' && !canReturnFromGeneration.value) return
if (currentStep.value > 0) {
const targetStep = WIZARD_STEPS[currentStep.value - 1]?.id
if (!targetStep) return
@@ -1014,8 +1021,13 @@ async function initializeExistingWorkflow() {
if (sourceTask.status === 'running') resumeStep = 'generate'
if (resumeStep === 'preview' && !previewItems.value.length) resumeStep = 'upload'
if (resumeStep === 'results' && sourceTask.status !== 'completed') resumeStep = 'generate'
if (resumeStep === 'generate' || resumeStep === 'results') {
const resume = resumeGeneration()
goToStep(resumeStep)
await resume
return
}
goToStep(resumeStep)
if (resumeStep === 'generate' || resumeStep === 'results') await resumeGeneration()
}
onBeforeUnmount(() => {
@@ -1154,7 +1166,7 @@ onMounted(() => {
<div class="footer-left">
<el-button
v-if="currentStep > 0"
:disabled="currentStepId === 'generate' || previewBuilding || sourceUploading"
:disabled="(currentStepId === 'generate' && !canReturnFromGeneration) || previewBuilding || sourceUploading"
@click="handleBack"
>
<i class="fa fa-arrow-left" style="margin-right: 6px;" /> 返回{{ previousStepLabel }}
@@ -1167,8 +1179,8 @@ onMounted(() => {
<el-button
class="wizard-primary-action"
type="primary"
:loading="modelSubmitLoading || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
:loading="modelSubmitLoading || generationStarting || generation.status === 'running' || resultRegenerationBusy || (currentStepId === 'upload' && (sourceUploading || previewBuilding))"
:disabled="hydrating || modelSubmitLoading || generationStarting || Boolean(initializationError) || resultRegenerationBusy || (currentStepId === 'generate' && generation.status === 'running') || previewBuilding || sourceUploading || (currentStepId === 'upload' && hasUnfinishedUploads)"
@click="handlePrimaryAction"
>
{{ primaryActionLabel }} <i class="fa" :class="primaryActionIcon" style="margin-left: 6px;" />

View File

@@ -9,6 +9,7 @@ import {
getDataProcessResults,
getDataProcessTask,
publishDataProcess,
repeatDataProcessTask,
restoreDataProcessResult,
updateDataProcessResult,
} from '@/api/modules/dataProcess'
@@ -42,6 +43,8 @@ const savingResult = ref(false)
const restoringResultId = ref<string | number | null>(null)
const publishDialogVisible = ref(false)
const publishing = ref(false)
const repeatGenerating = ref(false)
const repeatRequestId = ref('')
const configExpanded = ref(false)
const resultCellTooltipOptions = {
popperClass: 'data-process-result-tooltip',
@@ -113,6 +116,51 @@ const preprocessOptionLabelMap: Record<string, string> = {
preserve_context: '保留上下文',
}
const structuredPreprocessOptionKeys = new Set([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
function formatStructuredPreprocessOptions(value: unknown[]) {
const options = [...new Set(value.map((item) => String(item)))]
const selected = new Set(options)
const consumed = new Set<string>()
const labels: string[] = []
function appendGroup(values: string[], groupLabel: string) {
const selectedValues = values.filter((item) => selected.has(item))
selectedValues.forEach((item) => consumed.add(item))
if (selectedValues.length === values.length) {
labels.push(groupLabel)
return
}
selectedValues.forEach((item) => {
labels.push(`${preprocessOptionLabelMap[item] || item}(历史部分配置)`)
})
}
appendGroup(['clean_invalid', 'deduplicate'], '数据清洗')
appendGroup(['detect_structure', 'normalize_format'], '结构标准化')
if (selected.has('desensitize')) {
consumed.add('desensitize')
labels.push('敏感信息脱敏')
}
if (selected.has('filter_anomaly')) {
consumed.add('filter_anomaly')
labels.push('异常数据过滤(历史规则)')
}
options.forEach((item) => {
if (!consumed.has(item)) labels.push(preprocessOptionLabelMap[item] || item)
})
return labels.length ? labels.join('、') : '-'
}
function numeric(value: unknown) {
const parsed = typeof value === 'number' ? value : Number(value)
return Number.isFinite(parsed) ? parsed : 0
@@ -199,6 +247,11 @@ const canRegenerate = computed(() => {
|| status === 'stopped'
|| (status === 'completed' && (Boolean(outputDatasetId.value) || hasPublishedOutputs.value))
})
const canRepeatGeneration = computed(() => (
detail.value?.status === 'completed'
&& detail.value.results_confirmed !== false
&& previewCount.value > 0
))
const creatorName = computed(() => detail.value?.creator_name || detail.value?.creator || '-')
const createTime = computed(() => detail.value?.create_time || detail.value?.created_at)
const startTime = computed(() => detail.value?.start_time || detail.value?.started_at)
@@ -263,9 +316,14 @@ function formatConfigValue(key: string, value: unknown) {
}
if (Array.isArray(value)) {
if (key === 'preprocess_options') {
return value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
const containsStructuredOption = value.some((item) => (
structuredPreprocessOptionKeys.has(String(item))
))
return containsStructuredOption
? formatStructuredPreprocessOptions(value)
: value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
}
return value.length ? value.join('、') : '-'
}
@@ -503,6 +561,48 @@ function startRegeneration() {
void router.push({ name: 'data-process-regenerate', params: { id: taskId.value } })
}
function createRepeatRequestId() {
if (typeof globalThis.crypto?.randomUUID === 'function') {
return globalThis.crypto.randomUUID()
}
return `${Date.now()}_${Math.random().toString(36).slice(2, 14)}`
}
async function repeatGeneration() {
if (!detail.value?.updated_at || repeatGenerating.value) return
try {
await ElMessageBox.confirm(
'系统会复制当前配置、源文件和切分结果,创建一个独立的新任务并在后台生成。原任务和原结果不会被修改。',
'按原配置再生成一批?',
{
confirmButtonText: '创建并开始生成',
cancelButtonText: '取消',
type: 'info',
},
)
} catch {
return
}
repeatGenerating.value = true
repeatRequestId.value ||= createRepeatRequestId()
try {
const repeated = await repeatDataProcessTask(taskId.value, {
expected_updated_at: detail.value.updated_at,
request_id: repeatRequestId.value,
})
ElMessage.success(repeated.created ? '已创建新任务,正在后台生成' : '已恢复此前创建的新任务')
await router.push({
name: 'data-process-workflow',
params: { id: repeated.task.id },
})
} catch {
// 保留幂等请求 ID网络超时后再次点击不会重复创建任务。
} finally {
repeatGenerating.value = false
}
}
watch([currentPage, pageSize], () => void loadResults())
onMounted(loadPage)
@@ -520,22 +620,32 @@ onBeforeUnmount(() => {
<el-tag :type="displayStatus.type" size="small" effect="light">
{{ displayStatus.label }}
</el-tag>
<el-button
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
class="publish-button"
type="primary"
@click="openPublishDialog"
>
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
</el-button>
<el-button
v-if="canRegenerate"
class="publish-button"
type="primary"
@click="startRegeneration"
>
<i class="fa fa-refresh" style="margin-right: 4px;" />重新生成
</el-button>
<div class="heading-actions">
<el-button
v-if="detail.status === 'completed' && !hasCurrentPublishedDataset"
type="primary"
@click="openPublishDialog"
>
<i class="fa fa-database" style="margin-right: 4px;" />发布为三个数据集
</el-button>
<el-button
v-if="canRepeatGeneration"
type="primary"
:loading="repeatGenerating"
:disabled="repeatGenerating"
@click="repeatGeneration"
>
<i class="fa fa-clone" style="margin-right: 4px;" />按原配置再生成一批
</el-button>
<el-button
v-if="canRegenerate"
type="warning"
plain
@click="startRegeneration"
>
<i class="fa fa-refresh" style="margin-right: 4px;" />覆盖当前任务重新生成
</el-button>
</div>
</div>
<p>{{ detail.description || '暂无任务描述' }}</p>
<dl class="heading-meta">
@@ -804,7 +914,15 @@ onBeforeUnmount(() => {
> p { margin: 8px 0 0; color: #64748b; font-size: 13px; }
}
.publish-button { margin-left: auto; }
.heading-actions {
margin-left: auto;
display: flex;
flex-wrap: wrap;
justify-content: flex-end;
gap: 8px;
:deep(.el-button + .el-button) { margin-left: 0; }
}
.load-state-actions { display: flex; gap: 10px; }
.compact-empty { padding: 28px 18px; color: #94a3b8; font-size: 13px; text-align: center; }
.publish-form-grid { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 12px; }
@@ -968,7 +1086,8 @@ onBeforeUnmount(() => {
@media (max-width: 720px) {
.metric-grid, .config-grid { grid-template-columns: 1fr; }
.detail-heading .heading-row { align-items: flex-start; flex-wrap: wrap; }
.publish-button { width: 100%; margin-left: 0; }
.heading-actions { width: 100%; margin-left: 0; }
.heading-actions :deep(.el-button) { width: 100%; }
.publish-form-grid { grid-template-columns: 1fr; gap: 0; }
.result-toolbar { align-items: stretch; flex-direction: column; }
.result-filters { padding: 0 16px 16px; flex-direction: column; }

View File

@@ -46,6 +46,13 @@ const sourceUrl = computed(() => (
? getDataProcessSourceRawUrl(props.taskId, props.sourceFileId)
: ''
))
const selectedXlsxLocator = computed(() => {
const locator = props.selectedItem?.sourceLocator
if (!locator) return null
const hasSheet = locator.sheet_index != null || Boolean(locator.sheet_name)
const hasRow = locator.row_number != null || locator.sheet_record_index != null
return hasSheet && hasRow ? locator : null
})
const visibleRowRange = computed(() => {
const sheet = xlsxPreview.value?.active_sheet
if (!sheet || !sheet.rows.length) return '当前工作表没有可预览记录'
@@ -95,6 +102,16 @@ const selectedRecordKey = computed(() => {
})
function xlsxRowHighlighted(row: DataProcessXlsxPreviewRow) {
const locator = selectedXlsxLocator.value
const sheet = xlsxPreview.value?.active_sheet
if (locator && sheet) {
const sheetMatches = locator.sheet_index != null
? sheet.index === locator.sheet_index
: sheet.name === locator.sheet_name
if (!sheetMatches) return false
if (locator.row_number != null) return row.row_number === locator.row_number
return row.record_index === locator.sheet_record_index
}
return Boolean(selectedRecordKey.value && recordKey(row.record) === selectedRecordKey.value)
}
@@ -119,8 +136,11 @@ async function locateSelectedItem() {
async function loadPreview(options: { reset?: boolean } = {}) {
const sequence = ++loadSequence
if (options.reset) {
activeSheetIndex.value = 0
pageOffset.value = 0
const locator = selectedXlsxLocator.value
activeSheetIndex.value = locator?.sheet_index ?? 0
pageOffset.value = locator?.sheet_record_index == null
? 0
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
preview.value = null
}
errorMessage.value = ''
@@ -178,8 +198,34 @@ watch(
)
watch(
() => props.selectedItem?.id,
() => void locateSelectedItem(),
() => [
props.selectedItem?.id,
props.selectedItem?.sourceLocator?.sheet_index,
props.selectedItem?.sourceLocator?.sheet_record_index,
props.selectedItem?.sourceLocator?.row_number,
],
() => {
const locator = selectedXlsxLocator.value
if (!locator || isDocx.value) {
void locateSelectedItem()
return
}
const targetSheet = locator.sheet_index ?? activeSheetIndex.value
const targetOffset = locator.sheet_record_index == null
? pageOffset.value
: Math.floor(locator.sheet_record_index / XLSX_PAGE_SIZE) * XLSX_PAGE_SIZE
const activeSheet = xlsxPreview.value?.active_sheet
if (
activeSheet?.index === targetSheet
&& activeSheet.offset === targetOffset
) {
void locateSelectedItem()
return
}
activeSheetIndex.value = targetSheet
pageOffset.value = targetOffset
void loadPreview()
},
)
</script>
@@ -301,6 +347,8 @@ watch(
:key="row.row_number"
class="xlsx-row"
:class="{ 'is-highlighted': xlsxRowHighlighted(row) }"
:data-row-number="row.row_number"
:data-record-index="row.record_index"
>
<th class="row-number-cell">{{ row.row_number }}</th>
<td

View File

@@ -2,7 +2,11 @@
import { computed, nextTick, ref, watch } from 'vue'
import OfficeSourceViewer from './OfficeSourceViewer.vue'
import PdfSourceViewer from './PdfSourceViewer.vue'
import { sourceLines } from './previewModel'
import {
isManualPreviewItem,
sourceLineNumberAtOffset,
sourceLineWindow,
} from './previewModel'
import type { PreviewItem, ProcessType } from './types'
const props = defineProps<{
@@ -31,9 +35,11 @@ const sourceViewerRef = ref<HTMLElement | null>(null)
const search = ref('')
const currentPage = ref(1)
const PREVIEW_PAGE_SIZE = 10
const SOURCE_LINE_RENDER_LIMIT = 240
const SOURCE_LINE_CHARACTER_LIMIT = 4_000
const sourceWindowStartLine = ref(1)
const editingItemId = ref<string | null>(null)
const editorDraft = ref('')
const lines = computed(() => sourceLines(props.sourceText))
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
const editingItem = computed(() => props.items.find((item) => item.id === editingItemId.value))
const normalizedFileFormat = computed(() => (
@@ -43,6 +49,27 @@ const normalizedFileFormat = computed(() => (
))
const isPdfSource = computed(() => normalizedFileFormat.value === 'pdf')
const isOfficeSource = computed(() => ['docx', 'xlsx'].includes(normalizedFileFormat.value))
const selectedSourceOffset = computed(() => {
const item = selectedItem.value
return item ? sourceOffsetRange(item)?.start ?? null : null
})
const selectedSourceLine = computed(() => {
const item = selectedItem.value
if (!item) return null
return sourceLineRange(item)?.start
?? (selectedSourceOffset.value == null
? null
: sourceLineNumberAtOffset(props.sourceText, selectedSourceOffset.value))
})
const visibleSourceWindow = computed(() => sourceLineWindow(
props.sourceText,
sourceWindowStartLine.value,
SOURCE_LINE_RENDER_LIMIT,
SOURCE_LINE_CHARACTER_LIMIT,
selectedSourceLine.value,
selectedSourceOffset.value,
))
const lines = computed(() => visibleSourceWindow.value.lines)
const filteredItems = computed(() => props.items.filter((item, index) => {
const matchesSearch = !search.value.trim()
@@ -58,10 +85,27 @@ const pagedItems = computed(() => {
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
function isLineHighlighted(lineStart: number, lineEnd: number) {
function sourceLineRange(item: PreviewItem) {
const start = item.sourceLocator?.start_line ?? item.sourceStartLine
const end = item.sourceLocator?.end_line ?? item.sourceEndLine ?? start
return start == null ? null : { start, end: end ?? start }
}
function sourceOffsetRange(item: PreviewItem) {
const start = item.sourceLocator?.source_start ?? item.sourceStart
const end = item.sourceLocator?.source_end ?? item.sourceEnd ?? start
return start == null ? null : { start, end: Math.max(start, end ?? start) }
}
function isLineHighlighted(lineNumber: number, lineStart: number, lineEnd: number) {
const item = selectedItem.value
if (!item || item.sourceStart == null || item.sourceEnd == null) return false
return lineEnd >= item.sourceStart && lineStart <= item.sourceEnd
if (!item) return false
const lineRange = sourceLineRange(item)
if (lineRange) return lineNumber >= lineRange.start && lineNumber <= lineRange.end
const offsetRange = sourceOffsetRange(item)
if (!offsetRange) return false
const effectiveEnd = Math.max(offsetRange.start + 1, offsetRange.end)
return lineEnd >= offsetRange.start && lineStart < effectiveEnd
}
function selectItem(id: string) {
@@ -107,34 +151,88 @@ watch(search, () => {
watch(() => props.selectedFileId, closeEditor)
watch(selectedItem, async (item) => {
watch([selectedItem, () => props.sourceText], async ([item]) => {
if (!item) return
const visibleIndex = filteredItems.value.findIndex((entry) => entry.id === item.id)
if (visibleIndex >= 0) {
currentPage.value = Math.floor(visibleIndex / PREVIEW_PAGE_SIZE) + 1
}
if (isPdfSource.value || isOfficeSource.value || item.sourceStart == null) return
if (isPdfSource.value || isOfficeSource.value) return
const itemLineRange = sourceLineRange(item)
const itemOffsetRange = sourceOffsetRange(item)
if (!itemLineRange && !itemOffsetRange) {
sourceWindowStartLine.value = 1
return
}
const targetLine = selectedSourceLine.value
?? sourceLineNumberAtOffset(props.sourceText, itemOffsetRange?.start ?? 0)
sourceWindowStartLine.value = Math.max(1, targetLine - Math.floor(SOURCE_LINE_RENDER_LIMIT / 3))
await nextTick()
const target = sourceViewerRef.value?.querySelector<HTMLElement>(`[data-source-start="${item.sourceStart}"]`)
const exactTarget = sourceViewerRef.value
?.querySelector<HTMLElement>(`[data-line-number="${targetLine}"]`)
const target = exactTarget
?? sourceViewerRef.value?.querySelector<HTMLElement>('.source-line.is-highlighted')
target?.scrollIntoView({ block: 'center', behavior: 'smooth' })
}, { immediate: true })
async function showPreviousSourceWindow() {
sourceWindowStartLine.value = Math.max(1, sourceWindowStartLine.value - SOURCE_LINE_RENDER_LIMIT)
await nextTick()
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
}
async function showNextSourceWindow() {
if (!visibleSourceWindow.value.hasMore) return
sourceWindowStartLine.value = visibleSourceWindow.value.endLine + 1
await nextTick()
if (sourceViewerRef.value) sourceViewerRef.value.scrollTop = 0
}
function itemNumber(item: PreviewItem) {
return props.items.findIndex((entry) => entry.id === item.id) + 1
}
function lineRange(item: PreviewItem) {
if (item.sourcePages?.length) {
const first = item.sourcePages[0]
const last = item.sourcePages[item.sourcePages.length - 1]
return first === last ? `来源:第 ${first}` : `来源:第 ${first}${last}`
if (isManualPreviewItem(item)) return '手动新增,无源文件定位'
const locator = item.sourceLocator
const locatedLines = sourceLineRange(item)
if (props.processType === 'unstructured') {
const parts: string[] = []
if (item.sourcePages?.length) {
const first = item.sourcePages[0]
const last = item.sourcePages[item.sourcePages.length - 1]
parts.push(first === last ? `${first}` : `${first}${last}`)
}
if (locatedLines) {
parts.push(
locatedLines.start === locatedLines.end
? `${locatedLines.start}`
: `${locatedLines.start}${locatedLines.end}`,
)
}
if (item.headingPath?.length) parts.push(`章节:${item.headingPath.join(' / ')}`)
return parts.length ? `来源:${parts.join(' · ')}` : '来源:源文件内容(无精确定位)'
}
if (item.sourceStartLine == null || item.sourceEndLine == null) return '手动新增,无源文件定位'
return item.sourceStartLine === item.sourceEndLine
? `来源:第 ${item.sourceStartLine}`
: `来源:第 ${item.sourceStartLine}${item.sourceEndLine}`
if (locator?.kind === 'xlsx') {
const sheet = locator.sheet_name || `工作表 ${Number(locator.sheet_index ?? 0) + 1}`
return locator.row_number != null
? `来源:${sheet} · 第 ${locator.row_number}`
: `来源:${sheet}`
}
if (locator?.kind === 'json') {
return locator.json_pointer
? `来源JSON 路径 ${locator.json_pointer}`
: '来源JSON 根对象'
}
if (locatedLines) {
return locatedLines.start === locatedLines.end
? `来源:第 ${locatedLines.start}`
: `来源:第 ${locatedLines.start}${locatedLines.end}`
}
return '来源:源文件记录'
}
</script>
@@ -175,6 +273,31 @@ function lineRange(item: PreviewItem) {
<div>
<strong>源文件 · {{ fileName }}</strong>
</div>
<div
v-if="!isPdfSource && !isOfficeSource && lines.length"
class="source-window-controls"
aria-label="源文件行窗口"
>
<span> {{ visibleSourceWindow.startLine }}{{ visibleSourceWindow.endLine }} </span>
<el-button
link
size="small"
aria-label="查看上一段源文件"
:disabled="!visibleSourceWindow.hasPrevious"
@click="showPreviousSourceWindow"
>
上一段
</el-button>
<el-button
link
size="small"
aria-label="查看下一段源文件"
:disabled="!visibleSourceWindow.hasMore"
@click="showNextSourceWindow"
>
下一段
</el-button>
</div>
</div>
<PdfSourceViewer
@@ -197,8 +320,9 @@ function lineRange(item: PreviewItem) {
v-for="line in lines"
:key="line.number"
class="source-line"
:class="{ 'is-highlighted': isLineHighlighted(line.start, line.end) }"
:class="{ 'is-highlighted': isLineHighlighted(line.number, line.start, line.end) }"
:data-source-start="line.start"
:data-line-number="line.number"
>
<span class="line-number">{{ line.number }}</span>
<span class="line-content">{{ line.content || ' ' }}</span>
@@ -282,7 +406,7 @@ function lineRange(item: PreviewItem) {
/>
<div class="editor-actions">
<el-button
v-if="editingItem.sourceStart != null"
v-if="!isManualPreviewItem(editingItem)"
link
@click="restoreItem"
>
@@ -431,6 +555,22 @@ function lineRange(item: PreviewItem) {
}
}
.source-window-controls {
flex: none;
gap: 2px !important;
> span {
margin-right: 4px;
color: #8a93a3;
font-size: 11px;
white-space: nowrap;
}
:deep(.el-button) {
margin-left: 0;
}
}
.source-viewer {
flex: 1;
height: 538px;

View File

@@ -1,4 +1,5 @@
<script setup lang="ts">
import { computed } from 'vue'
import type {
GenerationControlOptions,
PreprocessOption,
@@ -21,27 +22,32 @@ const emit = defineEmits<{
'update:options': [value: StructuredProcessOptions]
}>()
const PREPROCESS_OPTIONS: Array<{
value: PreprocessOption
const PREPROCESS_GROUPS: Array<{
values: PreprocessOption[]
label: string
description: string
}> = [
{ value: 'clean_invalid', label: '清理无效数据', description: '清理全空列,并剔除关键字段残缺的数据行' },
{
value: 'detect_structure',
label: '嵌套结构展平',
description: '展平嵌套对象和可解析的 JSON 字段Excel 表头与合并单元格在上传时自动解析',
values: ['clean_invalid', 'deduplicate'],
label: '数据清洗',
description: '清理全空列和空记录,并删除内容完全相同的记录;不会猜测可空字段是否必填',
},
{
value: 'deduplicate',
label: '重复记录去重',
description: '按整行内容或 id、uuid、key、code、*_id 等身份字段去重,暂不支持自定义组合字段',
values: ['detect_structure', 'normalize_format'],
label: '结构标准化',
description: '展平嵌套对象和可解析的 JSON 字段,并统一编码、空白、字段名和 JSON 序列化格式',
},
{
values: ['desensitize'],
label: '敏感信息脱敏',
description: '识别并脱敏姓名、手机号、邮箱和身份证号',
},
{ value: 'normalize_format', label: '数据格式标准化', description: '按所选规则统一编码、空白、字段名及 JSON 序列化格式' },
{ value: 'filter_anomaly', label: '异常数据过滤', description: '使用 IQR 识别数值离群值,并过滤乱码等异常记录' },
{ value: 'desensitize', label: '敏感信息脱敏', description: '识别并脱敏姓名、手机号、邮箱和身份证号' },
]
const legacyAnomalyFilterEnabled = computed(() => (
props.options.preprocessOptions.includes('filter_anomaly')
))
function updateField<K extends keyof StructuredProcessOptions>(
field: K,
value: StructuredProcessOptions[K],
@@ -57,14 +63,26 @@ function updateQaPairsPerRow(value: number | undefined) {
updateField('qaPairsPerRow', normalizeQaPairsGenerationCount(value))
}
function updatePreprocessOptions(value: Array<string | number | boolean>) {
const allowedValues = new Set(PREPROCESS_OPTIONS.map((option) => option.value))
const preprocessOptions = Array.from(new Set(value.filter(
(option): option is PreprocessOption => (
typeof option === 'string' && allowedValues.has(option as PreprocessOption)
),
)))
updateField('preprocessOptions', preprocessOptions)
function selectedCount(values: PreprocessOption[]) {
return values.filter((value) => props.options.preprocessOptions.includes(value)).length
}
function groupSelected(values: PreprocessOption[]) {
return selectedCount(values) === values.length
}
function groupIndeterminate(values: PreprocessOption[]) {
const count = selectedCount(values)
return count > 0 && count < values.length
}
function updatePreprocessGroup(values: PreprocessOption[], checked: string | number | boolean) {
const next = new Set(props.options.preprocessOptions)
values.forEach((value) => {
if (Boolean(checked)) next.add(value)
else next.delete(value)
})
updateField('preprocessOptions', [...next])
}
</script>
@@ -73,26 +91,34 @@ function updatePreprocessOptions(value: Array<string | number | boolean>) {
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>选择在生成问答对之前需要执行的数据处理方式</p>
<p>默认不执行预处理请按数据情况自行选择</p>
</div>
</div>
<el-checkbox-group
:model-value="options.preprocessOptions"
class="preprocess-option-grid"
@update:model-value="updatePreprocessOptions"
>
<el-checkbox
v-for="option in PREPROCESS_OPTIONS"
:key="option.value"
:value="option.value"
<div class="preprocess-option-grid">
<label
v-for="group in PREPROCESS_GROUPS"
:key="group.label"
class="preprocess-option"
:class="{ 'is-checked': groupSelected(group.values) }"
>
<el-checkbox
:model-value="groupSelected(group.values)"
:indeterminate="groupIndeterminate(group.values)"
@update:model-value="updatePreprocessGroup(group.values, $event)"
/>
<span class="preprocess-option-copy">
<strong>{{ option.label }}</strong>
<small>{{ option.description }}</small>
<strong>{{ group.label }}</strong>
<small>{{ group.description }}</small>
</span>
</el-checkbox>
</el-checkbox-group>
</label>
</div>
<el-alert
v-if="legacyAnomalyFilterEnabled"
class="legacy-preprocess-alert"
type="warning"
:closable="false"
title="该历史任务仍启用了已停用的“异常数据过滤”;为保证结果可复现,本次继续保留"
/>
</div>
<div class="form-section generation-options-section">

View File

@@ -119,7 +119,7 @@ defineExpose({ revealValidation })
<div class="section-title-row">
<div>
<h3>预处理选项</h3>
<p>默认启用结构感知的推荐策略只需决定是否需要脱敏</p>
<p>默认不执行预处理请按文档情况自行选择</p>
</div>
</div>
<div class="preprocess-option-grid">

View File

@@ -97,7 +97,7 @@ export function isBuiltInGenerationPrompt(value: string) {
export function createDefaultStructuredOptions(): StructuredProcessOptions {
return {
preprocessOptions: ['clean_invalid', 'detect_structure', 'deduplicate', 'normalize_format'],
preprocessOptions: [],
semanticEnrichment: false,
qaPairsPerRow: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -117,22 +117,15 @@ export function createDefaultStructuredOptions(): StructuredProcessOptions {
export function createDefaultUnstructuredOptions(): UnstructuredProcessOptions {
return {
preprocessOptions: [
'clean_invalid_content',
'detect_document_structure',
'merge_short_content',
'filter_low_quality',
'deduplicate_content',
'preserve_context',
],
preprocessOptions: [],
chunkMethod: 'layout_hybrid',
chunkSize: 800,
chunkOverlap: 100,
minChunkSize: 100,
semanticBreakpointPercentile: 95,
preserveTables: true,
preserveCodeBlocks: true,
preserveLists: true,
preserveTables: false,
preserveCodeBlocks: false,
preserveLists: false,
semanticEnrichment: false,
qaPairsPerChunk: 1,
datasetSplit: { train: 80, validation: 10, test: 10 },
@@ -218,11 +211,21 @@ function generationOptionsFromConfig(
export function createStructuredOptionsFromConfig(config: DataProcessConfig): StructuredProcessOptions {
const defaults = createDefaultStructuredOptions()
const preprocessOptions = configValue<unknown>(config, 'preprocess_options', [])
const supportedPreprocessOptions = new Set<PreprocessOption>([
'clean_invalid',
'deduplicate',
'detect_structure',
'normalize_format',
'desensitize',
'filter_anomaly',
])
return {
...defaults,
...generationOptionsFromConfig(config, defaults),
preprocessOptions: Array.isArray(preprocessOptions)
? preprocessOptions.map(String) as PreprocessOption[]
? Array.from(new Set(preprocessOptions.map(String).filter(
(option): option is PreprocessOption => supportedPreprocessOptions.has(option as PreprocessOption),
)))
: defaults.preprocessOptions,
semanticEnrichment: Boolean(configValue(
config,

View File

@@ -1,4 +1,4 @@
import type { SourceLine } from './types'
import type { PreviewItem, SourceLine } from './types'
/** 仅用于“使用示例”上传;正式预览和切片全部由后端生成。 */
export const DEFAULT_SOURCE_TEXT = [
@@ -12,19 +12,141 @@ export const DEFAULT_SOURCE_TEXT = [
'答:复利是将上一期利息加入本金,再计算下一期利息。',
].join('\n')
/**
* 把后端返回的字符偏移映射为源文件行,仅负责界面高亮,不参与切片。
*/
export function sourceLines(sourceText: string): SourceLine[] {
const rawLines = sourceText.split('\n')
let cursor = 0
export interface SourceLineWindow {
lines: SourceLine[]
startLine: number
endLine: number
hasPrevious: boolean
hasMore: boolean
}
return rawLines.map((content, index) => {
const start = cursor
const end = start + content.length
cursor = end + (index < rawLines.length - 1 ? 1 : 0)
return { number: index + 1, content, start, end }
})
function unicodeCodePointLength(value: string, start = 0, end = value.length) {
let length = 0
let index = start
while (index < end) {
const codePoint = value.codePointAt(index)
index += codePoint != null && codePoint > 0xffff ? 2 : 1
length += 1
}
return length
}
function advanceCodePoints(value: string, start: number, end: number, count: number) {
let index = start
let remaining = Math.max(0, count)
while (index < end && remaining > 0) {
const codePoint = value.codePointAt(index)
index += codePoint != null && codePoint > 0xffff ? 2 : 1
remaining -= 1
}
return index
}
/**
* 只扫描并返回当前可见行窗口,不对全文 split避免大文件生成巨量字符串数组。
* 字符定位场景可开启 code point 偏移,以与后端 Python 的字符计数保持一致。
*/
export function sourceLineWindow(
sourceText: string,
requestedStartLine: number,
maxLines: number,
maxCharactersPerLine: number,
focusLine: number | null = null,
focusOffset: number | null = null,
): SourceLineWindow {
const startLine = Math.max(1, Math.trunc(requestedStartLine) || 1)
const limit = Math.max(1, Math.trunc(maxLines) || 1)
const characterLimit = Math.max(1, Math.trunc(maxCharactersPerLine) || 1)
const trackUnicodeOffsets = focusOffset != null
const lines: SourceLine[] = []
let lineNumber = 1
let jsCursor = 0
let sourceCursor = 0
while (jsCursor <= sourceText.length && lineNumber < startLine) {
const newlineIndex = sourceText.indexOf('\n', jsCursor)
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
sourceCursor = trackUnicodeOffsets
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd) + (newlineIndex >= 0 ? 1 : 0)
: (newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1)
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
lineNumber += 1
}
while (jsCursor <= sourceText.length && lines.length < limit) {
const newlineIndex = sourceText.indexOf('\n', jsCursor)
const jsEnd = newlineIndex >= 0 ? newlineIndex : sourceText.length
const fullSourceEnd = trackUnicodeOffsets
? sourceCursor + unicodeCodePointLength(sourceText, jsCursor, jsEnd)
: jsEnd
const focusedStart = focusLine === lineNumber && focusOffset != null
? Math.max(sourceCursor, focusOffset - Math.floor(characterLimit / 3))
: sourceCursor
const segmentSourceStart = Math.min(
focusedStart,
Math.max(sourceCursor, fullSourceEnd - characterLimit),
)
const relativeSegmentStart = trackUnicodeOffsets
? segmentSourceStart - sourceCursor
: Math.max(0, segmentSourceStart - jsCursor)
const segmentJsStart = advanceCodePoints(
sourceText,
jsCursor,
jsEnd,
relativeSegmentStart,
)
const segmentJsEnd = advanceCodePoints(
sourceText,
segmentJsStart,
jsEnd,
characterLimit,
)
const segmentLength = trackUnicodeOffsets
? unicodeCodePointLength(sourceText, segmentJsStart, segmentJsEnd)
: segmentJsEnd - segmentJsStart
const start = trackUnicodeOffsets ? segmentSourceStart : segmentJsStart
const end = start + segmentLength
const content = `${segmentJsStart > jsCursor ? '… ' : ''}${sourceText.slice(segmentJsStart, segmentJsEnd)}${segmentJsEnd < jsEnd ? ' …' : ''}`
lines.push({ number: lineNumber, content, start, end })
sourceCursor = fullSourceEnd + (newlineIndex >= 0 ? 1 : 0)
jsCursor = newlineIndex >= 0 ? newlineIndex + 1 : sourceText.length + 1
lineNumber += 1
}
return {
lines,
startLine: lines[0]?.number ?? startLine,
endLine: lines[lines.length - 1]?.number ?? startLine,
hasPrevious: startLine > 1,
hasMore: jsCursor <= sourceText.length,
}
}
/** 根据后端 code point 偏移查找物理行号,不构建全文行数组。 */
export function sourceLineNumberAtOffset(sourceText: string, targetOffset: number) {
const normalizedOffset = Math.max(0, Math.trunc(targetOffset) || 0)
let offset = 0
let lineNumber = 1
for (const character of sourceText) {
if (offset >= normalizedOffset) break
if (character === '\n') lineNumber += 1
offset += 1
}
return lineNumber
}
/**
* 手动新增项可能先以空内容保存为 invalid编辑后又由后端标记为 modified
* 因此不能只依赖可变的 status空原文且完全没有来源定位才是稳定兜底。
*/
export function isManualPreviewItem(item: PreviewItem): boolean {
const hasSourceLocation = item.sourceStart != null
|| item.sourceEnd != null
|| item.sourceStartLine != null
|| item.sourceEndLine != null
|| Boolean(item.sourcePages?.length)
|| Boolean(item.sourceLocator)
return item.status === 'manual' || (!item.originalContent && !hasSourceLocation)
}
/** 与后端预览 token 估算规则一致,仅用于编辑中的即时计数。 */

View File

@@ -24,6 +24,7 @@ export type PreprocessOption =
| 'detect_structure'
| 'deduplicate'
| 'normalize_format'
/** 仅用于恢复历史任务,新任务界面不再提供。 */
| 'filter_anomaly'
| 'desensitize'
@@ -94,7 +95,6 @@ export interface ExternalDataSource {
export interface UploadedDataFile {
uid: string | number
sourceFileId?: string
rawFile?: File
name: string
size: number
count: number
@@ -118,6 +118,22 @@ export interface SourceLine {
end: number
}
export type PreviewSourceLocatorKind = 'json' | 'jsonl' | 'csv' | 'xlsx'
export interface PreviewSourceLocator {
kind: PreviewSourceLocatorKind
record_index?: number | null
start_line?: number | null
end_line?: number | null
source_start?: number | null
source_end?: number | null
json_pointer?: string | null
sheet_index?: number | null
sheet_name?: string | null
row_number?: number | null
sheet_record_index?: number | null
}
export interface PreviewItem {
id: string
sourceFileId: string
@@ -129,6 +145,8 @@ export interface PreviewItem {
sourceStartLine: number | null
sourceEndLine: number | null
sourcePages?: number[]
sourceLocator?: PreviewSourceLocator
headingPath?: string[]
tokenCount: number
status: 'original' | 'modified' | 'manual' | 'invalid'
qualityScore?: number

View File

@@ -71,7 +71,11 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
let generationTimer: ReturnType<typeof setTimeout> | null = null
let generationRun = 0
let pollFailureCount = 0
let generationStarting = false
const generationStarting = ref(false)
const generationRestoring = ref(false)
const canReturnFromGeneration = computed(() => (
generation.status === 'idle' && !generationStarting.value && !generationRestoring.value
))
function stopGenerationTimer() {
generationRun += 1
@@ -170,14 +174,14 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
}
async function startGeneration() {
if (generationStarting || generation.status === 'running') return false
if (generationStarting.value || generation.status === 'running') return false
const taskId = bindings.taskId.value
if (!taskId) {
ElMessage.error('任务尚未创建,请返回上一步重试')
return false
}
generationStarting = true
generationStarting.value = true
let runId: number | null = null
try {
const canStart = await bindings.beforeGenerate?.()
@@ -204,17 +208,18 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
generation.message = error instanceof Error ? error.message : '启动数据处理失败,请重试。'
return false
} finally {
generationStarting = false
generationStarting.value = false
}
}
async function resumeGeneration() {
const taskId = bindings.taskId.value
if (!taskId) return
stopGenerationTimer()
const activeRunId = generationRun
pollFailureCount = 0
generationRestoring.value = true
try {
stopGenerationTimer()
const activeRunId = generationRun
pollFailureCount = 0
const progress = await getDataProcessProgress(taskId)
if (activeRunId !== generationRun) return
if (progress.status === 'running') {
@@ -236,6 +241,8 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
} catch (error) {
generation.status = 'failed'
generation.message = error instanceof Error ? error.message : '查询任务进度失败,请重试。'
} finally {
generationRestoring.value = false
}
}
@@ -430,7 +437,9 @@ export function useDataProcessGeneration(bindings: GenerationBindings) {
return {
bulkRegeneration,
canReturnFromGeneration,
generation,
generationStarting,
regeneratingResultId,
resultRegenerationBusy,
results,

View File

@@ -2,7 +2,6 @@ import { computed, nextTick, ref, type Reactive, type Ref } from 'vue'
import { useRoute } from 'vue-router'
import {
getDataProcessPreview,
getDataProcessSourceContent,
getDataProcessTask,
regenerateDataProcessTask,
} from '@/api/modules/dataProcess'
@@ -15,7 +14,10 @@ import {
createStructuredOptionsFromConfig,
createUnstructuredOptionsFromConfig,
} from './dataProcessCreateState'
import { mapDataProcessSourceFile } from './useDataProcessSourceUpload'
import {
loadCanonicalSourceContent,
mapDataProcessSourceFile,
} from './useDataProcessSourceUpload'
import type {
PreviewItem,
ProcessType,
@@ -50,24 +52,6 @@ interface RegenerationBindings {
resetDownstream: () => void
}
async function loadSourceContent(taskId: string, fileId: string | number) {
const chunks: string[] = []
let startLine = 1
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
start_line: startLine,
line_count: 10_000,
})
chunks.push(source.content || '')
if (!source.has_more) break
const nextLine = Number(source.end_line || startLine) + 1
if (nextLine <= startLine) break
startLine = nextLine
}
// source_content_lines 已保留原始换行;分页之间直接拼接,避免凭空增加空行并破坏偏移。
return chunks.join('')
}
async function loadAllPreviews(taskId: string, mapPreviewItem: RegenerationBindings['mapPreviewItem']) {
const first = await getDataProcessPreview(taskId, { page: 1, page_size: 500 })
const items = [...first.items]
@@ -97,7 +81,7 @@ export function useDataProcessRegeneration(bindings: RegenerationBindings) {
async function hydrateWorkspace(task: DataProcessTask, preservePreviews: boolean) {
const taskId = String(task.id)
bindings.uploadedFiles.value = await Promise.all((task.source_files || []).map(async (file) => (
mapDataProcessSourceFile(file, await loadSourceContent(taskId, file.id))
mapDataProcessSourceFile(file, await loadCanonicalSourceContent(taskId, file.id))
)))
bindings.previewItems.value = preservePreviews
? await loadAllPreviews(taskId, bindings.mapPreviewItem)

View File

@@ -6,7 +6,6 @@ import {
} from '@/api/modules/dataProcess'
import type { ProcessType, UploadedDataFile } from './types'
const BINARY_FILE_EXTENSIONS = new Set(['xlsx', 'pdf', 'docx', 'pptx'])
const STRUCTURED_FILE_EXTENSIONS = new Set(['json', 'jsonl', 'ndjson', 'csv', 'tsv', 'xlsx'])
const UNSTRUCTURED_FILE_EXTENSIONS = new Set([
'txt', 'md', 'markdown', 'pdf', 'docx', 'pptx', 'json', 'jsonl', 'ndjson',
@@ -15,11 +14,11 @@ const LEGACY_OFFICE_EXTENSIONS = new Set(['doc', 'xls', 'ppt'])
const MAX_SOURCE_FILE_BYTES = 200 * 1024 * 1024
const MAX_SOURCE_FILE_COUNT = 20
const MAX_SOURCE_BATCH_BYTES = 500 * 1024 * 1024
const SOURCE_CONTENT_PAGE_CHARS = 1_000_000
interface SourceUploadJob {
uid: string
file: File
extension: string
}
interface SourceUploadOptions {
@@ -60,9 +59,6 @@ export function validateSourceFileSelection(
: '结构化数据支持 JSON、JSONL、NDJSON、CSV、TSV、XLSX',
}
}
if (selectedFiles.some((file) => file.name === raw.name && file.size === raw.size)) {
return { valid: false, severity: 'warning', message: '同名且同大小的文件已经选择' }
}
if (selectedFiles.length >= MAX_SOURCE_FILE_COUNT) {
return { valid: false, severity: 'warning', message: `每个任务最多选择 ${MAX_SOURCE_FILE_COUNT} 个文件` }
}
@@ -73,6 +69,34 @@ export function validateSourceFileSelection(
return { valid: true, extension }
}
function unicodeCodePointLength(value: string) {
let length = 0
for (const _character of value) length += 1
return length
}
/** 分页读取服务端保存的规范化正文,避免重新使用浏览器本地解码结果。 */
export async function loadCanonicalSourceContent(
taskId: string | number,
fileId: string | number,
) {
const chunks: string[] = []
let offset = 0
while (true) {
const source = await getDataProcessSourceContent(taskId, fileId, {
offset,
limit: SOURCE_CONTENT_PAGE_CHARS,
})
const content = source.content || ''
chunks.push(content)
if (!source.has_more) break
const nextOffset = Number(source.offset ?? offset) + unicodeCodePointLength(content)
if (nextOffset <= offset) throw new Error('服务端规范化内容分页异常,请删除文件后重试')
offset = nextOffset
}
return chunks.join('')
}
export function mapDataProcessSourceFile(
file: DataProcessSourceFile,
content = '',
@@ -126,16 +150,6 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
pending.uploadError = undefined
try {
let content = ''
if (!BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(await job.file.arrayBuffer())
} catch {
throw new Error('文本文件不是有效的 UTF-8 编码,请转换编码后重试')
}
if (!content.trim()) throw new Error('不能上传空文件')
}
const uploaded = await uploadDataProcessSourceFiles(currentTaskId, [job.file], (progress) => {
pending.uploadProgress = progress
})
@@ -144,22 +158,13 @@ export function useDataProcessSourceUpload(options: SourceUploadOptions) {
// 先登记后端 ID确保正文读取失败时仍可正确删除已落库的文件。
Object.assign(pending, mapDataProcessSourceFile(source), {
rawFile: job.file,
status: 'uploading',
uploadProgress: 99,
})
if (BINARY_FILE_EXTENSIONS.has(job.extension)) {
try {
const parsed = await getDataProcessSourceContent(currentTaskId, source.id, {
start_line: 1,
line_count: 10_000,
})
pending.content = parsed.content
} catch {
// 原文件已经成功落库,正文稍后仍可由预览构建接口读取,不重复上传。
}
} else {
pending.content = content
try {
pending.content = await loadCanonicalSourceContent(currentTaskId, source.id)
} catch {
throw new Error('文件已上传,但服务端规范化内容读取失败,请删除文件后重试')
}
pending.status = 'ready'