fix(data-process): 补齐原文参照与详情统计

This commit is contained in:
caoxiaozhu
2026-07-24 16:30:29 +08:00
parent 994ec6644a
commit b2c570f607
8 changed files with 249 additions and 28 deletions

View File

@@ -37,6 +37,7 @@ export interface DataProcessTask {
output_dataset_name?: string | null
output_dataset?: string | null
source_file_count?: number
preview_count?: number
input_count?: number
output_count?: number
filtered_count?: number

View File

@@ -950,6 +950,7 @@ onMounted(() => {
v-else-if="currentStepId === 'results'"
v-model:selected-id="selectedResultId"
:items="results"
:preview-items="previewItems"
@update:field="updateResultField"
@restore:item="restoreResult"
/>

View File

@@ -42,6 +42,7 @@ const savingResult = ref(false)
const restoringResultId = ref<string | number | null>(null)
const publishDialogVisible = ref(false)
const publishing = ref(false)
const configExpanded = ref(false)
let resultFilterTimer: ReturnType<typeof setTimeout> | null = null
const editForm = reactive({ instruction: '', input: '', output: '' })
@@ -90,8 +91,24 @@ const chunkMethodLabelMap: Record<string, string> = {
custom: '自定义分隔符',
}
function numeric(value: number | undefined) {
return Number.isFinite(value) ? Number(value) : 0
const preprocessOptionLabelMap: Record<string, string> = {
clean_invalid: '清理无效数据',
detect_structure: '识别表格结构',
deduplicate: '重复数据去重',
normalize_format: '数据格式标准化',
filter_anomaly: '异常数据过滤',
desensitize: '敏感信息脱敏',
clean_invalid_content: '清理无效内容',
detect_document_structure: '识别文档结构',
merge_short_content: '合并过短内容',
filter_low_quality: '过滤低质量内容',
deduplicate_content: '重复内容去重',
preserve_context: '保留上下文',
}
function numeric(value: unknown) {
const parsed = typeof value === 'number' ? value : Number(value)
return Number.isFinite(parsed) ? parsed : 0
}
function formatDateTime(value?: string | null) {
@@ -101,13 +118,31 @@ function formatDateTime(value?: string | null) {
}
const retentionRate = computed(() => {
const inputCount = numeric(detail.value?.input_count)
return inputCount
? Number(((numeric(detail.value?.output_count) / inputCount) * 100).toFixed(1))
const outputCount = numeric(detail.value?.output_count)
const denominator = outputCount + numeric(detail.value?.filtered_count)
return denominator
? Number(((outputCount / denominator) * 100).toFixed(1))
: 0
})
const progressPercentage = computed(() => Math.min(100, Math.max(0, numeric(detail.value?.progress))))
const retentionDenominator = computed(() => (
numeric(detail.value?.output_count) + numeric(detail.value?.filtered_count)
))
const retentionPercentage = computed(() => Math.min(100, Math.max(0, retentionRate.value)))
const progressPercentage = computed(() => (
detail.value?.status === 'completed'
? 100
: Math.min(100, Math.max(0, numeric(detail.value?.progress)))
))
const isUnstructured = computed(() => detail.value?.process_type === 'unstructured')
const sourceFileCount = computed(() => (
numeric(detail.value?.source_file_count) || detail.value?.source_files?.length || 0
))
const previewCount = computed(() => numeric(detail.value?.preview_count))
const inputMetricLabel = computed(() => isUnstructured.value ? '输入切片' : '输入记录')
const inputMetricCount = computed(() => (
isUnstructured.value ? previewCount.value : numeric(detail.value?.input_count)
))
const sourceDatasetName = computed(() => (
detail.value?.source_dataset_name
|| detail.value?.source_dataset
@@ -125,22 +160,44 @@ const completeTime = computed(() => detail.value?.complete_time || detail.value?
const durationText = computed(() => {
if (detail.value?.duration) return detail.value.duration
const seconds = detail.value?.duration_seconds
if (!Number.isFinite(seconds)) return detail.value?.status === 'running' ? '处理中' : '-'
const safeSeconds = Math.max(0, Math.round(Number(seconds)))
const providedDuration = detail.value?.duration_seconds
let seconds = providedDuration == null ? null : numeric(providedDuration)
if (seconds == null && startTime.value) {
const started = new Date(startTime.value).getTime()
const finished = completeTime.value
? new Date(completeTime.value).getTime()
: detail.value?.status === 'running' ? Date.now() : Number.NaN
if (Number.isFinite(started) && Number.isFinite(finished) && finished >= started) {
seconds = (finished - started) / 1000
}
}
if (seconds == null) return detail.value?.status === 'running' ? '处理中' : '-'
const safeSeconds = Math.max(0, Math.round(seconds))
const hours = Math.floor(safeSeconds / 3600)
const minutes = Math.floor(safeSeconds / 60)
const restSeconds = safeSeconds % 60
if (hours) return `${hours} 小时 ${minutes % 60}${restSeconds}`
return minutes ? `${minutes}${restSeconds}` : `${restSeconds}`
})
const configRows = computed(() => Object.entries(detail.value?.config || {})
.filter(([key]) => !/(?:password|secret|token|api_key)/i.test(key))
.filter(([key]) => (
key !== 'generation_model_snapshot'
&& !/(?:password|secret|token|api_key)/i.test(key)
))
.map(([key, value]) => ({
label: configLabelMap[key] || key.split('_').join(' '),
value: formatConfigValue(key, value),
})))
function formatConfigValue(key: string, value: unknown) {
if (key === 'generation_model_id') {
const snapshot = detail.value?.config?.generation_model_snapshot
if (snapshot && typeof snapshot === 'object' && !Array.isArray(snapshot)) {
const model = snapshot as Record<string, unknown>
return String(model.name || model.display_name || model.model_name || value || '-')
}
}
if (key === 'chunk_method' && typeof value === 'string') {
return chunkMethodLabelMap[value] || value
}
@@ -148,7 +205,14 @@ function formatConfigValue(key: string, value: unknown) {
const split = value as Partial<DataProcessDatasetSplit>
return `训练集 ${split.train ?? 0}% / 验证集 ${split.validation ?? 0}% / 测试集 ${split.test ?? 0}%`
}
if (Array.isArray(value)) return value.length ? value.join('、') : '-'
if (Array.isArray(value)) {
if (key === 'preprocess_options') {
return value.length
? value.map((item) => preprocessOptionLabelMap[String(item)] || String(item)).join('、')
: '-'
}
return value.length ? value.join('、') : '-'
}
if (typeof value === 'boolean') return value ? '是' : '否'
if (value && typeof value === 'object') return JSON.stringify(value)
return value == null || value === '' ? '-' : String(value)
@@ -436,19 +500,19 @@ onBeforeUnmount(() => {
<small>{{ completeTime ? `完成于 ${formatDateTime(completeTime)}` : `当前进度 ${progressPercentage}%` }}</small>
</div>
<div class="metric-card">
<span>输入数据</span>
<strong>{{ numeric(detail.input_count).toLocaleString() }}</strong>
<small>来源{{ sourceDatasetName }}</small>
<span>{{ inputMetricLabel }}</span>
<strong>{{ inputMetricCount.toLocaleString() }}</strong>
<small>{{ sourceFileCount ? `源文件 ${sourceFileCount} 个:` : '来源:' }}{{ sourceDatasetName }}</small>
</div>
<div class="metric-card">
<span>输出结果</span>
<strong>{{ numeric(detail.output_count).toLocaleString() }}</strong>
<small>{{ outputDatasetName || '尚未生成输出数据集' }}</small>
<small>{{ outputDatasetName || '尚未发布为数据集' }}</small>
</div>
<div class="metric-card is-primary">
<span>数据保留率</span>
<strong>{{ numeric(detail.input_count) ? `${retentionRate}%` : '-' }}</strong>
<el-progress :percentage="progressPercentage" :show-text="false" :stroke-width="5" />
<span>结果保留率</span>
<strong>{{ retentionDenominator ? `${retentionRate}%` : '-' }}</strong>
<el-progress :percentage="retentionPercentage" :show-text="false" :stroke-width="5" />
</div>
</section>
@@ -477,7 +541,7 @@ onBeforeUnmount(() => {
link
@click="router.push(`/dataset/${outputDatasetId}/preview`)"
>{{ outputDatasetName }} <i class="fa fa-external-link" /></el-button>
<span v-else>{{ outputDatasetName || '尚未生成' }}</span>
<span v-else>{{ outputDatasetName || '尚未发布为数据集' }}</span>
</dd>
</div>
</dl>
@@ -488,7 +552,8 @@ onBeforeUnmount(() => {
<div><h2 id="statistics-title">处理统计</h2><p>查看数据清洗过滤和输出情况</p></div>
</div>
<div class="statistics-grid">
<div><span>原始数据</span><strong>{{ numeric(detail.input_count).toLocaleString() }}</strong></div>
<div><span>源文件</span><strong>{{ sourceFileCount.toLocaleString() }}</strong></div>
<div><span>{{ inputMetricLabel }}</span><strong>{{ inputMetricCount.toLocaleString() }}</strong></div>
<div><span>成功输出</span><strong>{{ numeric(detail.output_count).toLocaleString() }}</strong></div>
<div><span>过滤数据</span><strong>{{ numeric(detail.filtered_count).toLocaleString() }}</strong></div>
<div><span>重复数据</span><strong>{{ numeric(detail.duplicate_count).toLocaleString() }}</strong></div>
@@ -499,13 +564,29 @@ onBeforeUnmount(() => {
</div>
<section class="detail-section config-section" aria-labelledby="config-title">
<div class="section-heading">
<div class="section-heading config-heading">
<div><h2 id="config-title">处理配置</h2><p>任务执行时使用的规则与参数</p></div>
<div class="config-actions">
<span> {{ configRows.length }} </span>
<el-button
text
:aria-expanded="configExpanded"
aria-controls="config-content"
@click="configExpanded = !configExpanded"
>
{{ configExpanded ? '收起' : '展开' }}
<i class="fa" :class="configExpanded ? 'fa-chevron-up' : 'fa-chevron-down'" />
</el-button>
</div>
</div>
<dl v-if="configRows.length" class="config-grid">
<div v-for="item in configRows" :key="item.label"><dt>{{ item.label }}</dt><dd>{{ item.value }}</dd></div>
</dl>
<div v-else class="compact-empty">暂无处理配置</div>
<el-collapse-transition>
<div v-show="configExpanded" id="config-content" class="config-content">
<dl v-if="configRows.length" class="config-grid">
<div v-for="item in configRows" :key="item.label"><dt>{{ item.label }}</dt><dd>{{ item.value }}</dd></div>
</dl>
<div v-else class="compact-empty">暂无处理配置</div>
</div>
</el-collapse-transition>
</section>
<section class="detail-section result-section" aria-labelledby="result-title" v-loading="resultLoading">
@@ -698,6 +779,22 @@ onBeforeUnmount(() => {
p { margin: 4px 0 0; color: #94a3b8; font-size: 12px; }
}
.config-heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
}
.config-actions {
display: flex;
align-items: center;
gap: 8px;
> span { color: #94a3b8; font-size: 12px; }
:deep(.el-button) { gap: 6px; }
}
.info-list { margin: 0; padding: 4px 18px 12px; }
.info-list > div {
min-height: 44px; border-bottom: 1px solid #f1f5f9; display: grid; grid-template-columns: 100px minmax(0, 1fr); align-items: center;
@@ -718,7 +815,7 @@ onBeforeUnmount(() => {
.config-grid { margin: 0; padding: 8px 18px 16px; display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); column-gap: 40px; }
.config-grid > div { min-height: 48px; border-bottom: 1px solid #f1f5f9; display: flex; align-items: center; justify-content: space-between; gap: 20px; }
.config-grid dt { color: #64748b; font-size: 12px; }
.config-grid dd { margin: 0; color: #334155; font-size: 13px; text-align: right; }
.config-grid dd { max-width: 68%; margin: 0; color: #334155; font-size: 13px; overflow-wrap: anywhere; text-align: right; }
.result-toolbar { display: flex; align-items: center; justify-content: space-between; gap: 20px; border-bottom: 1px solid #eef0f3; }
.result-toolbar .section-heading { border-bottom: 0; }

View File

@@ -1,9 +1,10 @@
<script setup lang="ts">
import { computed, ref } from 'vue'
import type { ResultItem } from './types'
import type { PreviewItem, ResultItem } from './types'
const props = defineProps<{
items: ResultItem[]
previewItems: PreviewItem[]
selectedId: string | null
}>()
@@ -17,6 +18,35 @@ const search = ref('')
const invalidOnly = ref(false)
const selectedItem = computed(() => props.items.find((item) => item.id === props.selectedId) ?? props.items[0])
const selectedIndex = computed(() => props.items.findIndex((item) => item.id === selectedItem.value?.id))
const selectedSource = computed(() => {
const previewItemId = selectedItem.value?.previewItemId
if (!previewItemId) return null
return props.previewItems.find((item) => item.id === previewItemId) ?? null
})
const selectedSourceContent = computed(() => (
selectedSource.value?.editedContent.trim()
|| selectedSource.value?.originalContent.trim()
|| ''
))
const selectedSourceWasPreprocessed = computed(() => Boolean(
selectedSource.value
&& selectedSource.value.editedContent.trim()
&& selectedSource.value.editedContent !== selectedSource.value.originalContent
))
const selectedSourceMeta = computed(() => {
const source = selectedSource.value
if (!source) return ''
const parts: string[] = []
if (source.sourceStartLine != null) {
parts.push(
source.sourceEndLine != null && source.sourceEndLine !== source.sourceStartLine
? `${source.sourceStartLine}${source.sourceEndLine}`
: `${source.sourceStartLine}`,
)
}
if (source.tokenCount > 0) parts.push(`${source.tokenCount} Token`)
return parts.join(' · ')
})
const filteredItems = computed(() => props.items.filter((item, index) => {
const keyword = search.value.trim().toLowerCase()
@@ -79,6 +109,19 @@ function selectRelative(offset: number) {
<el-button link @click="emit('restore:item', selectedItem.id)"><i class="fa fa-undo" /> 恢复生成结果</el-button>
</div>
<article class="source-reference" aria-labelledby="source-reference-title">
<div class="source-reference-heading">
<div>
<strong id="source-reference-title">原文参照</strong>
<span v-if="selectedSourceMeta">{{ selectedSourceMeta }}</span>
</div>
<el-tag v-if="selectedSourceWasPreprocessed" size="small" effect="plain">已智能预处理</el-tag>
</div>
<pre v-if="selectedSourceContent">{{ selectedSourceContent }}</pre>
<p v-else>当前结果没有关联到可用的原文切片</p>
<small v-if="selectedSourceWasPreprocessed">这里展示的是实际送入模型的预处理后原文便于核对问题和答案是否有依据</small>
</article>
<div class="field-editor">
<label>Instruction <em>必填</em></label>
<el-input
@@ -255,6 +298,62 @@ function selectRelative(offset: number) {
min-width: 0;
}
.source-reference {
margin: 14px 18px 2px;
padding: 12px 14px;
border: 1px solid #dfe4ec;
border-radius: 7px;
background: #f8fafc;
pre {
max-height: 168px;
margin: 10px 0 0;
overflow: auto;
color: #344054;
font-family: inherit;
font-size: 12px;
line-height: 1.7;
white-space: pre-wrap;
word-break: break-word;
}
p {
margin: 10px 0 0;
color: #98a2b3;
font-size: 12px;
}
> small {
display: block;
margin-top: 8px;
color: #8a93a3;
font-size: 10px;
}
}
.source-reference-heading {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
> div {
display: flex;
align-items: baseline;
gap: 8px;
}
strong {
color: #344054;
font-size: 12px;
}
span {
color: #8a93a3;
font-size: 10px;
}
}
.field-editor {
padding: 13px 18px 0;

View File

@@ -126,6 +126,7 @@ export interface GenerationState {
export interface ResultItem {
id: string
previewItemId: string | null
instruction: string
input: string
output: string

View File

@@ -24,6 +24,7 @@ const POLL_INTERVAL_MS = 1500
function mapResult(item: DataProcessResult): ResultItem {
return {
id: String(item.id),
previewItemId: item.preview_item_id == null ? null : String(item.preview_item_id),
instruction: item.instruction,
input: item.input || '',
output: item.output,