345 lines
9.7 KiB
JavaScript
345 lines
9.7 KiB
JavaScript
'use client';
|
||
|
||
import { useState, useCallback } from 'react';
|
||
import {
|
||
Box,
|
||
Typography,
|
||
Button,
|
||
Paper,
|
||
List,
|
||
ListItem,
|
||
ListItemIcon,
|
||
ListItemText,
|
||
LinearProgress,
|
||
Alert
|
||
} from '@mui/material';
|
||
import { CloudUpload as UploadIcon, Description as FileIcon, CheckCircle as CheckIcon } from '@mui/icons-material';
|
||
import { useTranslation } from 'react-i18next';
|
||
// import { useDropzone } from 'react-dropzone';
|
||
|
||
/**
|
||
* 文件上传步骤组件
|
||
*/
|
||
export default function FileUploadStep({ onDataLoaded, onError }) {
|
||
const { t } = useTranslation();
|
||
const [uploading, setUploading] = useState(false);
|
||
const [uploadedFiles, setUploadedFiles] = useState([]);
|
||
|
||
// 健壮的CSV解析函数，支持多行字段和引号转义
|
||
const parseCSV = text => {
|
||
const result = [];
|
||
const lines = [];
|
||
let currentLine = '';
|
||
let inQuotes = false;
|
||
|
||
// 逐字符解析，正确处理引号内的换行符
|
||
for (let i = 0; i < text.length; i++) {
|
||
const char = text[i];
|
||
const nextChar = text[i + 1];
|
||
|
||
if (char === '"') {
|
||
if (inQuotes && nextChar === '"') {
|
||
// 转义的引号
|
||
currentLine += '"';
|
||
i++; // 跳过下一个引号
|
||
} else {
|
||
// 切换引号状态
|
||
inQuotes = !inQuotes;
|
||
}
|
||
} else if (char === '\n' && !inQuotes) {
|
||
// 行结束（不在引号内）
|
||
if (currentLine.trim()) {
|
||
lines.push(currentLine);
|
||
}
|
||
currentLine = '';
|
||
} else {
|
||
currentLine += char;
|
||
}
|
||
}
|
||
|
||
// 添加最后一行
|
||
if (currentLine.trim()) {
|
||
lines.push(currentLine);
|
||
}
|
||
|
||
if (lines.length < 2) {
|
||
throw new Error('CSV文件格式不正确，至少需要标题行和一行数据');
|
||
}
|
||
|
||
// 解析标题行
|
||
const headers = parseCSVLine(lines[0]);
|
||
|
||
// 解析数据行
|
||
for (let i = 1; i < lines.length; i++) {
|
||
const values = parseCSVLine(lines[i]);
|
||
if (values.length > 0) {
|
||
const obj = {};
|
||
headers.forEach((header, index) => {
|
||
obj[header] = values[index] || '';
|
||
});
|
||
result.push(obj);
|
||
}
|
||
}
|
||
|
||
return result;
|
||
};
|
||
|
||
// 解析单行CSV，处理逗号分隔和引号转义
|
||
const parseCSVLine = line => {
|
||
const result = [];
|
||
let current = '';
|
||
let inQuotes = false;
|
||
|
||
for (let i = 0; i < line.length; i++) {
|
||
const char = line[i];
|
||
const nextChar = line[i + 1];
|
||
|
||
if (char === '"') {
|
||
if (inQuotes && nextChar === '"') {
|
||
// 转义的引号
|
||
current += '"';
|
||
i++; // 跳过下一个引号
|
||
} else {
|
||
// 切换引号状态
|
||
inQuotes = !inQuotes;
|
||
}
|
||
} else if (char === ',' && !inQuotes) {
|
||
// 字段分隔符（不在引号内）
|
||
result.push(current.trim());
|
||
current = '';
|
||
} else {
|
||
current += char;
|
||
}
|
||
}
|
||
|
||
// 添加最后一个字段
|
||
result.push(current.trim());
|
||
|
||
return result;
|
||
};
|
||
|
||
// 检测并转换ShareGPT格式为Alpaca格式
|
||
const convertShareGPTToAlpaca = item => {
|
||
// 检查是否包含conversations字段且格式正确
|
||
if (item.conversations && Array.isArray(item.conversations)) {
|
||
const conversations = item.conversations;
|
||
|
||
// 查找system、human、gpt消息
|
||
let systemMessage = '';
|
||
let instruction = '';
|
||
let output = '';
|
||
|
||
for (const conv of conversations) {
|
||
if (conv.from === 'system' && conv.value) {
|
||
systemMessage = conv.value;
|
||
} else if (conv.from === 'human' && conv.value) {
|
||
instruction = conv.value;
|
||
} else if (conv.from === 'gpt' && conv.value) {
|
||
output = conv.value;
|
||
break; // 只取第一轮对话
|
||
}
|
||
}
|
||
|
||
// 如果有system消息，将其作为instruction的前缀
|
||
if (systemMessage && instruction) {
|
||
instruction = `${systemMessage}\n\n${instruction}`;
|
||
} else if (systemMessage && !instruction) {
|
||
instruction = systemMessage;
|
||
}
|
||
|
||
// 转换为Alpaca格式
|
||
return {
|
||
instruction: instruction || '',
|
||
input: '', // ShareGPT格式通常没有单独的input字段
|
||
output: output || '',
|
||
// 保留其他字段
|
||
...Object.fromEntries(Object.entries(item).filter(([key]) => key !== 'conversations'))
|
||
};
|
||
}
|
||
|
||
return item; // 如果不是ShareGPT格式，返回原始数据
|
||
};
|
||
|
||
const parseFileContent = async file => {
|
||
const text = await file.text();
|
||
const extension = file.name.split('.').pop().toLowerCase();
|
||
|
||
try {
|
||
let data = [];
|
||
|
||
if (extension === 'json') {
|
||
const parsed = JSON.parse(text);
|
||
data = Array.isArray(parsed) ? parsed : [parsed];
|
||
} else if (extension === 'jsonl') {
|
||
data = text
|
||
.split('\n')
|
||
.filter(line => line.trim())
|
||
.map(line => JSON.parse(line));
|
||
} else if (extension === 'csv') {
|
||
// 更健壮的CSV解析，支持多行字段和引号转义
|
||
data = parseCSV(text);
|
||
if (data.length === 0) {
|
||
throw new Error('CSV文件格式不正确或没有数据');
|
||
}
|
||
} else {
|
||
throw new Error('不支持的文件格式');
|
||
}
|
||
|
||
if (data.length === 0) {
|
||
throw new Error('文件中没有找到有效数据');
|
||
}
|
||
|
||
// 检测并转换ShareGPT格式为Alpaca格式
|
||
data = data.map(convertShareGPTToAlpaca);
|
||
|
||
// 生成预览数据（取前3条记录，每个字段值截取前100字符）
|
||
const previewData = data.slice(0, 3).map(item => {
|
||
const preview = {};
|
||
Object.keys(item).forEach(key => {
|
||
const value = String(item[key] || '');
|
||
preview[key] = value.length > 100 ? value.substring(0, 100) + '...' : value;
|
||
});
|
||
return preview;
|
||
});
|
||
|
||
return {
|
||
data,
|
||
preview: previewData,
|
||
source: {
|
||
type: 'file',
|
||
fileName: file.name,
|
||
fileSize: file.size,
|
||
totalRecords: data.length
|
||
}
|
||
};
|
||
} catch (error) {
|
||
throw new Error(`解析文件失败: ${error.message}`);
|
||
}
|
||
};
|
||
|
||
const handleFileSelect = async event => {
|
||
const files = event.target.files;
|
||
if (!files || files.length === 0) return;
|
||
|
||
const file = files[0];
|
||
setUploading(true);
|
||
|
||
try {
|
||
const result = await parseFileContent(file);
|
||
setUploadedFiles([
|
||
{
|
||
name: file.name,
|
||
size: file.size,
|
||
status: 'success'
|
||
}
|
||
]);
|
||
|
||
onDataLoaded(result.data, result.preview, result.source);
|
||
} catch (error) {
|
||
setUploadedFiles([
|
||
{
|
||
name: file.name,
|
||
size: file.size,
|
||
status: 'error',
|
||
error: error.message
|
||
}
|
||
]);
|
||
onError(error.message);
|
||
} finally {
|
||
setUploading(false);
|
||
}
|
||
};
|
||
|
||
const formatFileSize = bytes => {
|
||
if (bytes === 0) return '0 Bytes';
|
||
const k = 1024;
|
||
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
|
||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||
};
|
||
|
||
return (
|
||
<Box>
|
||
<Typography variant="h6" gutterBottom>
|
||
{t('import.uploadFile', '上传文件')}
|
||
</Typography>
|
||
|
||
<Typography variant="body2" color="text.secondary" sx={{ mb: 3 }}>
|
||
{t('import.supportedFormats', '支持 JSON、JSONL、CSV 格式文件')}
|
||
</Typography>
|
||
|
||
{/* 文件上传区域 */}
|
||
<Paper
|
||
sx={{
|
||
p: 4,
|
||
textAlign: 'center',
|
||
cursor: 'pointer',
|
||
border: '2px dashed',
|
||
borderColor: 'divider',
|
||
backgroundColor: 'background.paper',
|
||
transition: 'all 0.2s ease',
|
||
mb: 3,
|
||
'&:hover': {
|
||
borderColor: 'primary.main',
|
||
backgroundColor: 'action.hover'
|
||
}
|
||
}}
|
||
onClick={() => document.getElementById('file-upload-input').click()}
|
||
>
|
||
<input
|
||
id="file-upload-input"
|
||
type="file"
|
||
accept=".json,.jsonl,.csv"
|
||
onChange={handleFileSelect}
|
||
style={{ display: 'none' }}
|
||
/>
|
||
<UploadIcon sx={{ fontSize: 48, color: 'text.secondary', mb: 2 }} />
|
||
<Typography variant="h6" gutterBottom>
|
||
{t('import.dragDropFile', '拖拽文件到此处或点击选择文件')}
|
||
</Typography>
|
||
<Typography variant="body2" color="text.secondary">
|
||
{t('import.maxFileSize', '最大文件大小: 50MB')}
|
||
</Typography>
|
||
</Paper>
|
||
|
||
{/* 上传进度 */}
|
||
{uploading && (
|
||
<Box sx={{ mb: 3 }}>
|
||
<Typography variant="body2" gutterBottom>
|
||
{t('import.processingFile', '正在处理文件...')}
|
||
</Typography>
|
||
<LinearProgress />
|
||
</Box>
|
||
)}
|
||
|
||
{/* 已上传文件列表 */}
|
||
{uploadedFiles.length > 0 && (
|
||
<Box>
|
||
<Typography variant="subtitle2" gutterBottom>
|
||
{t('import.uploadedFiles', '已上传文件')}
|
||
</Typography>
|
||
<List>
|
||
{uploadedFiles.map((file, index) => (
|
||
<ListItem key={index} sx={{ px: 0 }}>
|
||
<ListItemIcon>
|
||
{file.status === 'success' ? <CheckIcon color="success" /> : <FileIcon color="error" />}
|
||
</ListItemIcon>
|
||
<ListItemText
|
||
primary={file.name}
|
||
secondary={file.status === 'success' ? `${formatFileSize(file.size)}` : file.error}
|
||
/>
|
||
</ListItem>
|
||
))}
|
||
</List>
|
||
|
||
{uploadedFiles.some(f => f.status === 'error') && (
|
||
<Alert severity="error" sx={{ mt: 2 }}>
|
||
{t('import.uploadError', '文件上传失败，请检查文件格式是否正确')}
|
||
</Alert>
|
||
)}
|
||
</Box>
|
||
)}
|
||
</Box>
|
||
);
|
||
}
|