304 lines
9.2 KiB
JavaScript
304 lines
9.2 KiB
JavaScript
'use client';
|
||
|
||
import { useState, useEffect, useRef } from 'react';
|
||
import {
|
||
Box,
|
||
Typography,
|
||
LinearProgress,
|
||
Alert,
|
||
Paper,
|
||
List,
|
||
ListItem,
|
||
ListItemIcon,
|
||
ListItemText,
|
||
Chip
|
||
} from '@mui/material';
|
||
import { CheckCircle as CheckIcon, Error as ErrorIcon, Info as InfoIcon } from '@mui/icons-material';
|
||
import { useTranslation } from 'react-i18next';
|
||
|
||
/**
|
||
* 导入进度步骤组件
|
||
*/
|
||
export default function ImportProgressStep({ projectId, rawData, fieldMapping, sourceInfo, onComplete, onError }) {
|
||
const { t } = useTranslation();
|
||
const [progress, setProgress] = useState(0);
|
||
const [currentStep, setCurrentStep] = useState('');
|
||
const [importStats, setImportStats] = useState({
|
||
total: 0,
|
||
processed: 0,
|
||
success: 0,
|
||
failed: 0,
|
||
skipped: 0,
|
||
errors: []
|
||
});
|
||
const [completed, setCompleted] = useState(false);
|
||
const startedRef = useRef(false); // 防止在开发模式下因严格模式导致重复执行
|
||
|
||
useEffect(() => {
|
||
if (!startedRef.current && rawData && fieldMapping && projectId) {
|
||
startedRef.current = true;
|
||
startImport();
|
||
}
|
||
}, [rawData, fieldMapping, projectId]);
|
||
|
||
const startImport = async () => {
|
||
try {
|
||
setCurrentStep(t('import.preparingData', '准备数据...'));
|
||
setImportStats(prev => ({ ...prev, total: rawData.length }));
|
||
|
||
// 转换数据格式
|
||
const convertedData = rawData.map(item => {
|
||
// 支持 question 映射多个字段，拼接为一个字符串
|
||
const qFields = fieldMapping.question;
|
||
const question = Array.isArray(qFields)
|
||
? qFields
|
||
.map(f => item[f] || '')
|
||
.filter(v => v && String(v).trim())
|
||
.join('\n')
|
||
: item[qFields] || '';
|
||
|
||
const converted = {
|
||
question,
|
||
answer: item[fieldMapping.answer] || '',
|
||
cot: fieldMapping.cot ? item[fieldMapping.cot] || '' : '',
|
||
questionLabel: '', // 默认标签，后续可以通过AI生成
|
||
chunkName: sourceInfo?.datasetName || sourceInfo?.fileName || 'Imported Data',
|
||
chunkContent: `Imported from ${sourceInfo?.type || 'file'}`,
|
||
model: 'imported',
|
||
confirmed: false,
|
||
score: 0,
|
||
tags: fieldMapping.tags ? JSON.stringify(parseTagsField(item[fieldMapping.tags])) : '[]',
|
||
note: '',
|
||
other: JSON.stringify(getOtherFields(item, fieldMapping))
|
||
};
|
||
|
||
// 不在前端抛错，由后端负责校验并统计 skipped
|
||
return converted;
|
||
});
|
||
|
||
setProgress(25);
|
||
setCurrentStep(t('import.uploadingData', '上传数据...'));
|
||
|
||
// 分批上传数据
|
||
const batchSize = 500;
|
||
let processed = 0;
|
||
let success = 0;
|
||
let failed = 0;
|
||
let skipped = 0;
|
||
const errors = [];
|
||
|
||
for (let i = 0; i < convertedData.length; i += batchSize) {
|
||
const batch = convertedData.slice(i, i + batchSize);
|
||
|
||
try {
|
||
const response = await fetch(`/api/projects/${projectId}/datasets/import`, {
|
||
method: 'POST',
|
||
headers: {
|
||
'Content-Type': 'application/json'
|
||
},
|
||
body: JSON.stringify({
|
||
datasets: batch,
|
||
sourceInfo
|
||
})
|
||
});
|
||
|
||
if (!response.ok) {
|
||
throw new Error(`批次上传失败: ${response.statusText}`);
|
||
}
|
||
|
||
const result = await response.json();
|
||
success += result.success || 0;
|
||
failed += typeof result.failed === 'number' ? result.failed : result.errors?.length || 0;
|
||
skipped += result.skipped || 0;
|
||
processed += batch.length;
|
||
|
||
if (result.errors && result.errors.length > 0) {
|
||
errors.push(...result.errors);
|
||
}
|
||
} catch (error) {
|
||
failed += batch.length;
|
||
processed += batch.length;
|
||
errors.push(`批次 ${Math.floor(i / batchSize) + 1}: ${error.message}`);
|
||
}
|
||
|
||
// 更新进度
|
||
const progressPercent = 25 + (processed / convertedData.length) * 70;
|
||
setProgress(progressPercent);
|
||
setImportStats({
|
||
total: convertedData.length,
|
||
processed,
|
||
success,
|
||
failed,
|
||
skipped,
|
||
errors
|
||
});
|
||
|
||
setCurrentStep(
|
||
t('import.processing', '处理中... {{processed}}/{{total}}', {
|
||
processed,
|
||
total: convertedData.length
|
||
})
|
||
);
|
||
}
|
||
|
||
setProgress(100);
|
||
setCurrentStep(t('import.completed', '导入完成'));
|
||
setCompleted(true);
|
||
|
||
// 延迟一下再调用完成回调，让用户看到完成状态
|
||
setTimeout(() => {
|
||
onComplete();
|
||
}, 2000);
|
||
} catch (error) {
|
||
onError(error.message);
|
||
setImportStats(prev => ({
|
||
...prev,
|
||
errors: [...prev.errors, error.message]
|
||
}));
|
||
}
|
||
};
|
||
|
||
// 解析标签字段
|
||
const parseTagsField = tagsValue => {
|
||
if (!tagsValue) return [];
|
||
|
||
if (Array.isArray(tagsValue)) {
|
||
return tagsValue;
|
||
}
|
||
|
||
if (typeof tagsValue === 'string') {
|
||
return tagsValue
|
||
.split(',')
|
||
.map(tag => tag.trim())
|
||
.filter(tag => tag);
|
||
}
|
||
|
||
return [];
|
||
};
|
||
|
||
// 获取其他字段（兼容数组映射）
|
||
const getOtherFields = (item, mapping) => {
|
||
const used = [];
|
||
Object.values(mapping).forEach(field => {
|
||
if (!field) return;
|
||
if (Array.isArray(field)) used.push(...field);
|
||
else used.push(field);
|
||
});
|
||
const mappedFields = new Set(used);
|
||
const otherFields = {};
|
||
|
||
Object.keys(item).forEach(key => {
|
||
if (!mappedFields.has(key)) {
|
||
otherFields[key] = item[key];
|
||
}
|
||
});
|
||
|
||
return otherFields;
|
||
};
|
||
|
||
return (
|
||
<Box>
|
||
<Typography variant="h6" gutterBottom>
|
||
{t('import.importing', '正在导入数据集')}
|
||
</Typography>
|
||
|
||
{/* 进度条 */}
|
||
<Paper sx={{ p: 3, mb: 3 }}>
|
||
<Typography variant="body1" gutterBottom>
|
||
{currentStep}
|
||
</Typography>
|
||
<LinearProgress variant="determinate" value={progress} sx={{ height: 8, borderRadius: 4, mb: 2 }} />
|
||
<Typography variant="body2" color="text.secondary">
|
||
{Math.round(progress)}% {t('import.complete', '完成')}
|
||
</Typography>
|
||
</Paper>
|
||
|
||
{/* 导入统计 */}
|
||
<Paper sx={{ p: 3, mb: 3 }}>
|
||
<Typography variant="subtitle1" gutterBottom>
|
||
{t('import.importStats', '导入统计')}
|
||
</Typography>
|
||
|
||
<Box sx={{ display: 'flex', gap: 2, flexWrap: 'wrap', mb: 2 }}>
|
||
<Chip
|
||
icon={<InfoIcon />}
|
||
label={t('import.total', '总计: {{count}}', { count: importStats.total })}
|
||
variant="outlined"
|
||
/>
|
||
<Chip
|
||
icon={<CheckIcon />}
|
||
label={t('import.success', '成功: {{count}}', { count: importStats.success })}
|
||
color="success"
|
||
variant="outlined"
|
||
/>
|
||
{importStats.skipped > 0 && (
|
||
<Chip
|
||
icon={<InfoIcon />}
|
||
label={t('import.skipped', '跳过: {{count}}', { count: importStats.skipped })}
|
||
color="warning"
|
||
variant="outlined"
|
||
/>
|
||
)}
|
||
{importStats.failed > 0 && (
|
||
<Chip
|
||
icon={<ErrorIcon />}
|
||
label={t('import.failed', '失败: {{count}}', { count: importStats.failed })}
|
||
color="error"
|
||
variant="outlined"
|
||
/>
|
||
)}
|
||
</Box>
|
||
|
||
{sourceInfo && (
|
||
<Box>
|
||
<Typography variant="body2" color="text.secondary">
|
||
{t('import.source', '数据源')}:{' '}
|
||
{sourceInfo.type === 'file' ? sourceInfo.fileName : sourceInfo.datasetName}
|
||
</Typography>
|
||
{sourceInfo.description && (
|
||
<Typography variant="body2" color="text.secondary">
|
||
{t('import.description', '描述')}: {sourceInfo.description}
|
||
</Typography>
|
||
)}
|
||
</Box>
|
||
)}
|
||
</Paper>
|
||
|
||
{/* 错误列表 */}
|
||
{importStats.errors.length > 0 && (
|
||
<Paper sx={{ p: 3 }}>
|
||
<Typography variant="subtitle1" gutterBottom color="error">
|
||
{t('import.errors', '错误信息')}
|
||
</Typography>
|
||
<List dense>
|
||
{importStats.errors.slice(0, 10).map((error, index) => (
|
||
<ListItem key={index} sx={{ px: 0 }}>
|
||
<ListItemIcon>
|
||
<ErrorIcon color="error" fontSize="small" />
|
||
</ListItemIcon>
|
||
<ListItemText primary={error} primaryTypographyProps={{ variant: 'body2' }} />
|
||
</ListItem>
|
||
))}
|
||
</List>
|
||
{importStats.errors.length > 10 && (
|
||
<Typography variant="body2" color="text.secondary">
|
||
{t('import.moreErrors', '还有 {{count}} 个错误未显示...', {
|
||
count: importStats.errors.length - 10
|
||
})}
|
||
</Typography>
|
||
)}
|
||
</Paper>
|
||
)}
|
||
|
||
{/* 完成提示 */}
|
||
{completed && (
|
||
<Alert severity="success" sx={{ mt: 2 }}>
|
||
{t('import.importSuccess', '数据集导入完成！成功导入 {{success}} 条记录。', {
|
||
success: importStats.success
|
||
})}
|
||
</Alert>
|
||
)}
|
||
</Box>
|
||
);
|
||
}
|