YG-Soft/YG-Datasets
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first-update

Browse Source
This commit is contained in:
DESKTOP-72TV0V4\caoxiaozhu
2026-03-17 14:36:31 +08:00
parent 72f08aee7c
commit 4eddf05e79
516 changed files with 115270 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

202
easy-dataset-main/app/projects/[projectId]/datasets/[datasetId]/page.js Normal file
View File

@@ -0,0 +1,202 @@
'use client';
import { Container, Box, Typography, Alert, Snackbar, Paper } from '@mui/material';
import { useEffect } from 'react';
import ChunkViewDialog from '@/components/text-split/ChunkViewDialog';
import DatasetHeader from '@/components/datasets/DatasetHeader';
import DatasetMetadata from '@/components/datasets/DatasetMetadata';
import EditableField from '@/components/datasets/EditableField';
import OptimizeDialog from '@/components/datasets/OptimizeDialog';
import DatasetRatingSection from '@/components/datasets/DatasetRatingSection';
import useDatasetDetails from '@/app/projects/[projectId]/datasets/[datasetId]/useDatasetDetails';
import { useTranslation } from 'react-i18next';
/**
* 数据集详情页面
*/
export default function DatasetDetailsPage({ params }) {
const { projectId, datasetId } = params;
const { t } = useTranslation();
// 使用自定义Hook管理状态和逻辑
const {
currentDataset,
loading,
editingAnswer,
editingCot,
editingQuestion,
answerValue,
cotValue,
questionValue,
snackbar,
confirming,
unconfirming,
optimizeDialog,
viewDialogOpen,
viewChunk,
datasetsAllCount,
datasetsConfirmCount,
answerTokens,
cotTokens,
shortcutsEnabled,
setShortcutsEnabled,
setSnackbar,
setAnswerValue,
setCotValue,
setQuestionValue,
setEditingAnswer,
setEditingCot,
setEditingQuestion,
handleNavigate,
handleConfirm,
handleUnconfirm,
handleSave,
handleDelete,
handleOpenOptimizeDialog,
handleCloseOptimizeDialog,
handleOptimize,
handleViewChunk,
handleCloseViewDialog
} = useDatasetDetails(projectId, datasetId);
// 加载状态
if (loading) {
return (
<Container maxWidth="lg" sx={{ mt: 4 }}>
<Box sx={{ display: 'flex', justifyContent: 'center', alignItems: 'center', height: '70vh' }}>
<Alert severity="info">{t('datasets.loadingDataset')}</Alert>
</Box>
</Container>
);
}
// 无数据状态
if (!currentDataset) {
return (
<Container maxWidth="lg" sx={{ mt: 4 }}>
<Alert severity="error">{t('datasets.datasetNotFound')}</Alert>
</Container>
);
}
return (
<Container maxWidth="xl" sx={{ mt: 4, mb: 4 }}>
{/* 顶部导航栏 */}
<DatasetHeader
projectId={projectId}
datasetsAllCount={datasetsAllCount}
datasetsConfirmCount={datasetsConfirmCount}
confirming={confirming}
unconfirming={unconfirming}
currentDataset={currentDataset}
shortcutsEnabled={shortcutsEnabled}
setShortcutsEnabled={setShortcutsEnabled}
onNavigate={handleNavigate}
onConfirm={handleConfirm}
onUnconfirm={handleUnconfirm}
onDelete={handleDelete}
/>
{/* 主要布局：左右分栏 */}
<Box sx={{ display: 'flex', gap: 3, alignItems: 'flex-start' }}>
{/* 左侧主要内容区域 */}
<Box sx={{ flex: 1, minWidth: 0 }}>
<Paper sx={{ p: 3 }}>
<EditableField
label={t('datasets.question')}
value={questionValue}
editing={editingQuestion}
onEdit={() => setEditingQuestion(true)}
onChange={e => setQuestionValue(e.target.value)}
onSave={() => handleSave('question', questionValue)}
dataset={currentDataset}
onCancel={() => {
setEditingQuestion(false);
setQuestionValue(currentDataset.question);
}}
/>
<EditableField
label={t('datasets.answer')}
value={answerValue}
editing={editingAnswer}
onEdit={() => setEditingAnswer(true)}
onChange={e => setAnswerValue(e.target.value)}
onSave={() => handleSave('answer', answerValue)}
onCancel={() => {
setEditingAnswer(false);
setAnswerValue(currentDataset.answer);
}}
dataset={currentDataset}
onOptimize={handleOpenOptimizeDialog}
tokenCount={answerTokens}
optimizing={optimizeDialog.loading}
/>
<EditableField
label={t('datasets.cot')}
value={cotValue}
editing={editingCot}
onEdit={() => setEditingCot(true)}
onChange={e => setCotValue(e.target.value)}
onSave={() => handleSave('cot', cotValue)}
dataset={currentDataset}
onCancel={() => {
setEditingCot(false);
setCotValue(currentDataset.cot || '');
}}
tokenCount={cotTokens}
/>
</Paper>
</Box>
{/* 右侧固定侧边栏 */}
<Box
sx={{
width: 360,
position: 'sticky',
top: 24,
maxHeight: 'calc(100vh - 48px)',
overflowY: 'auto'
}}
>
{/* 数据集元数据信息 */}
<DatasetMetadata currentDataset={currentDataset} onViewChunk={handleViewChunk} />
{/* 评分、标签、备注区域 */}
<DatasetRatingSection
dataset={currentDataset}
projectId={projectId}
onUpdate={() => {
// 更新成功后刷新数据，保持页面状态同步
// 这里可以调用 useDatasetDetails 的刷新逻辑
}}
currentDataset={currentDataset}
/>
</Box>
</Box>
{/* 消息提示 */}
<Snackbar
open={snackbar.open}
autoHideDuration={2000}
onClose={() => setSnackbar(prev => ({ ...prev, open: false }))}
anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }}
>
<Alert
onClose={() => setSnackbar(prev => ({ ...prev, open: false }))}
severity={snackbar.severity}
sx={{ width: '100%' }}
>
{snackbar.message}
</Alert>
</Snackbar>
{/* AI优化对话框 */}
<OptimizeDialog open={optimizeDialog.open} onClose={handleCloseOptimizeDialog} onConfirm={handleOptimize} />
{/* 文本块详情对话框 */}
<ChunkViewDialog open={viewDialogOpen} chunk={viewChunk} onClose={handleCloseViewDialog} />
</Container>
);
}

471
easy-dataset-main/app/projects/[projectId]/datasets/[datasetId]/useDatasetDetails.js Normal file
View File

@@ -0,0 +1,471 @@
'use client';
import { useState, useEffect, useRef, useCallback } from 'react';
import { useRouter } from 'next/navigation';
import { useAtomValue } from 'jotai/index';
import { selectedModelInfoAtom } from '@/lib/store';
import axios from 'axios';
import { toast } from 'sonner';
import i18n from '@/lib/i18n';
/**
* 数据集详情页面业务逻辑 Hook
*/
export default function useDatasetDetails(projectId, datasetId) {
const router = useRouter();
const [datasets, setDatasets] = useState([]);
const [currentDataset, setCurrentDataset] = useState(null);
const [loading, setLoading] = useState(true);
const [editingAnswer, setEditingAnswer] = useState(false);
const [editingCot, setEditingCot] = useState(false);
const [editingQuestion, setEditingQuestion] = useState(false);
const [answerValue, setAnswerValue] = useState('');
const [cotValue, setCotValue] = useState('');
const [questionValue, setQuestionValue] = useState('');
const [snackbar, setSnackbar] = useState({
open: false,
message: '',
severity: 'success'
});
const [confirming, setConfirming] = useState(false);
const [unconfirming, setUnconfirming] = useState(false);
const [optimizeDialog, setOptimizeDialog] = useState({
open: false,
loading: false
});
const [viewDialogOpen, setViewDialogOpen] = useState(false);
const [viewChunk, setViewChunk] = useState(null);
const [datasetsAllCount, setDatasetsAllCount] = useState(0);
const [datasetsConfirmCount, setDatasetsConfirmCount] = useState(0);
const [answerTokens, setAnswerTokens] = useState(0);
const [cotTokens, setCotTokens] = useState(0);
const model = useAtomValue(selectedModelInfoAtom);
const [shortcutsEnabled, setShortcutsEnabled] = useState(() => {
const storedValue = localStorage.getItem('shortcutsEnabled');
return storedValue !== null ? storedValue === 'true' : false;
});
// 输入环境判断，避免在输入框/可编辑区域误触快捷键
const isEditableTarget = el => {
if (!el) return false;
const tag = el.tagName?.toLowerCase();
if (tag && ['input', 'textarea', 'select'].includes(tag)) return true;
if (el.isContentEditable) return true;
// 兼容嵌套的可编辑区域与常见富文本编辑器
return !!el.closest?.('[contenteditable="true"], .ProseMirror, .ql-editor');
};
// 简单节流，避免连续触发
const lastShortcutRef = useRef(0);
// 异步获取Token数量
const fetchTokenCount = async () => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets/${datasetId}/token-count`);
if (response.ok) {
const data = await response.json();
if (data.answerTokens !== undefined) {
setAnswerTokens(data.answerTokens);
}
if (data.cotTokens !== undefined) {
setCotTokens(data.cotTokens);
}
}
} catch (error) {
console.error('获取Token数量失败:', error);
// Token加载失败不阻塞主界面或显示错误提示
}
};
// 获取数据集详情
const fetchDatasets = async () => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets/${datasetId}`);
if (!response.ok) throw new Error('获取数据集详情失败');
const data = await response.json();
setCurrentDataset(data.datasets);
setCotValue(data.datasets?.cot);
setAnswerValue(data.datasets?.answer);
setQuestionValue(data.datasets?.question);
setDatasetsAllCount(data.total);
setDatasetsConfirmCount(data.confirmedCount);
// 数据加载完成后，异步获取Token数量
fetchTokenCount();
} catch (error) {
setSnackbar({
open: true,
message: error.message,
severity: 'error'
});
} finally {
setLoading(false);
}
};
// 确认并保存数据集
const handleConfirm = async () => {
try {
setConfirming(true);
const response = await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
confirmed: true
})
});
if (!response.ok) {
throw new Error('操作失败');
}
setCurrentDataset(prev => ({ ...prev, confirmed: true }));
setSnackbar({
open: true,
message: '操作成功',
severity: 'success'
});
// 导航到下一个数据集
handleNavigate('next');
} catch (error) {
setSnackbar({
open: true,
message: error.message || '操作失败',
severity: 'error'
});
} finally {
setConfirming(false);
}
};
// 取消确认数据集
const handleUnconfirm = async () => {
try {
setUnconfirming(true);
const response = await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
confirmed: false
})
});
if (!response.ok) {
throw new Error('操作失败');
}
setCurrentDataset(prev => ({ ...prev, confirmed: false }));
setSnackbar({
open: true,
message: '已取消确认',
severity: 'success'
});
} catch (error) {
setSnackbar({
open: true,
message: error.message || '取消确认失败',
severity: 'error'
});
} finally {
setUnconfirming(false);
}
};
// 导航到其他数据集
const handleNavigate = async direction => {
const response = await axios.get(`/api/projects/${projectId}/datasets/${datasetId}?operateType=${direction}`);
if (response.data) {
router.push(`/projects/${projectId}/datasets/${response.data.id}`);
} else {
toast.warning(`已经是${direction === 'next' ? '最后' : '第'}一条数据了`);
}
};
// 保存编辑
const handleSave = async (field, value) => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
method: 'PATCH',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
[field]: value
})
});
if (!response.ok) {
throw new Error('保存失败');
}
const data = await response.json();
setCurrentDataset(prev => ({ ...prev, [field]: value }));
setSnackbar({
open: true,
message: '保存成功',
severity: 'success'
});
// 重置编辑状态
if (field === 'answer') setEditingAnswer(false);
if (field === 'cot') setEditingCot(false);
if (field === 'question') setEditingQuestion(false);
} catch (error) {
setSnackbar({
open: true,
message: error.message || '保存失败',
severity: 'error'
});
}
};
// 删除数据集
const handleDelete = async () => {
if (!confirm('确定要删除这条数据吗？此操作不可撤销。')) return;
try {
// 尝试获取下一个数据集，在删除前先确保有可导航的目标
const nextResponse = await axios.get(`/api/projects/${projectId}/datasets/${datasetId}?operateType=next`);
const hasNextDataset = !!nextResponse.data;
const nextDatasetId = hasNextDataset ? nextResponse.data.id : null;
// 删除当前数据集
const deleteResponse = await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
method: 'DELETE'
});
if (!deleteResponse.ok) {
throw new Error('删除失败');
}
// 导航逻辑：有下一个就跳转下一个，没有则返回列表页
if (hasNextDataset) {
router.push(`/projects/${projectId}/datasets/${nextDatasetId}`);
} else {
// 没有更多数据集，返回列表页面
router.push(`/projects/${projectId}/datasets`);
}
toast.success('删除成功');
} catch (error) {
setSnackbar({
open: true,
message: error.message || '删除失败',
severity: 'error'
});
}
};
// 优化对话框相关操作
const handleOpenOptimizeDialog = () => {
setOptimizeDialog({
open: true,
loading: false
});
};
const handleCloseOptimizeDialog = () => {
setOptimizeDialog(prev => {
// 如果正在优化，不允许关闭
if (prev.loading) {
return prev;
}
return {
open: false,
loading: false
};
});
};
// 优化操作
const handleOptimize = async advice => {
if (!model) {
setSnackbar({
open: true,
message: '请先选择模型，可以在顶部导航栏选择',
severity: 'error'
});
return;
}
// 立即关闭对话框，并设置优化中状态
setOptimizeDialog(prev => {
const newState = {
open: false,
loading: true
};
return newState;
});
toast.info('已开始优化，请稍候...');
// 异步后台处理，不等待结果
(async () => {
try {
const language = i18n.language === 'zh-CN' ? '中文' : 'en';
const response = await fetch(`/api/projects/${projectId}/datasets/optimize`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
datasetId,
model,
advice,
language
})
});
if (!response.ok) {
const errorData = await response.json();
throw new Error(errorData.error || '优化失败');
}
// 优化成功后，重新查询数据以获取最新状态
await fetchDatasets();
// 优化可能改变了文本内容，重新获取Token计数
fetchTokenCount();
toast.success('AI智能优化成功');
} catch (error) {
toast.error(error.message);
} finally {
setOptimizeDialog({
open: false,
loading: false
});
}
})();
};
// 查看文本块详情
const handleViewChunk = async chunkContent => {
try {
setViewChunk(chunkContent);
setViewDialogOpen(true);
} catch (error) {
console.error('查看文本块出错', error);
setSnackbar({
open: true,
message: error.message,
severity: 'error'
});
setViewDialogOpen(false);
}
};
// 关闭文本块详情对话框
const handleCloseViewDialog = () => {
setViewDialogOpen(false);
};
// 初始化和快捷键事件
useEffect(() => {
fetchDatasets();
}, [projectId, datasetId]);
// 快捷键状态变化
useEffect(() => {
localStorage.setItem('shortcutsEnabled', shortcutsEnabled);
}, [shortcutsEnabled]);
// 监听键盘事件
useEffect(() => {
const handleKeyDown = event => {
if (!shortcutsEnabled) return;
// 在输入框或可编辑区域时不触发
const activeEl = typeof document !== 'undefined' ? document.activeElement : null;
if (isEditableTarget(event.target) || isEditableTarget(activeEl)) {
return;
}
// 仅要求 Shift 修饰键，降低误触且更简单
if (!event.shiftKey) return;
// 简单节流，过滤极短时间内重复触发
const now = Date.now();
if (now - (lastShortcutRef.current || 0) < 250) {
return;
}
lastShortcutRef.current = now;
switch (event.key) {
case 'ArrowLeft': // 上一个（Shift + ArrowLeft）
event.preventDefault();
handleNavigate('prev');
break;
case 'ArrowRight': // 下一个（Shift + ArrowRight）
event.preventDefault();
handleNavigate('next');
break;
case 'y': // 确认（Shift + Y）
case 'Y':
if (!confirming && currentDataset && !currentDataset.confirmed) {
event.preventDefault();
handleConfirm();
}
break;
case 'd': // 删除（Shift + D）
case 'D':
event.preventDefault();
handleDelete();
break;
default:
break;
}
};
window.addEventListener('keydown', handleKeyDown);
return () => {
window.removeEventListener('keydown', handleKeyDown);
};
}, [shortcutsEnabled, confirming, currentDataset]);
return {
loading,
currentDataset,
answerValue,
cotValue,
questionValue,
editingAnswer,
editingCot,
editingQuestion,
confirming,
unconfirming,
snackbar,
optimizeDialog,
viewDialogOpen,
viewChunk,
datasetsAllCount,
datasetsConfirmCount,
answerTokens,
cotTokens,
shortcutsEnabled,
setShortcutsEnabled,
setSnackbar,
setAnswerValue,
setCotValue,
setQuestionValue,
setEditingAnswer,
setEditingCot,
setEditingQuestion,
handleNavigate,
handleConfirm,
handleUnconfirm,
handleSave,
handleDelete,
handleOpenOptimizeDialog,
handleCloseOptimizeDialog,
handleOptimize,
handleViewChunk,
handleCloseViewDialog
};
}

33
easy-dataset-main/app/projects/[projectId]/datasets/components/ActionBar.js Normal file
View File

@@ -0,0 +1,33 @@
'use client';
import { Box, Button } from '@mui/material';
import AssessmentIcon from '@mui/icons-material/Assessment';
import FileUploadIcon from '@mui/icons-material/FileUpload';
import FileDownloadIcon from '@mui/icons-material/FileDownload';
import { useTranslation } from 'react-i18next';
const ActionBar = ({ onBatchEvaluate, onImport, onExport, batchEvaluating = false }) => {
const { t } = useTranslation();
return (
<Box sx={{ display: 'flex', gap: 2 }}>
<Button
variant="outlined"
startIcon={<AssessmentIcon />}
sx={{ borderRadius: 2 }}
onClick={onBatchEvaluate}
disabled={batchEvaluating}
>
{batchEvaluating ? t('datasets.evaluating', '评估中...') : t('datasets.batchEvaluate', '批量评估')}
</Button>
<Button variant="outlined" startIcon={<FileUploadIcon />} sx={{ borderRadius: 2 }} onClick={onImport}>
{t('import.title', '导入')}
</Button>
<Button variant="outlined" startIcon={<FileDownloadIcon />} sx={{ borderRadius: 2 }} onClick={onExport}>
{t('export.title')}
</Button>
</Box>
);
};
export default ActionBar;

422
easy-dataset-main/app/projects/[projectId]/datasets/components/DatasetList.js Normal file
View File

@@ -0,0 +1,422 @@
'use client';
import {
Box,
Typography,
IconButton,
Table,
TableBody,
TableCell,
TableContainer,
TableHead,
TableRow,
Chip,
Divider,
useTheme,
alpha,
Tooltip,
Checkbox,
TablePagination,
TextField,
Card,
CircularProgress
} from '@mui/material';
import DeleteIcon from '@mui/icons-material/Delete';
import VisibilityIcon from '@mui/icons-material/Visibility';
import AssessmentIcon from '@mui/icons-material/Assessment';
import StarIcon from '@mui/icons-material/Star';
import { useTranslation } from 'react-i18next';
import { getRatingConfigI18n, formatScore } from '@/components/datasets/utils/ratingUtils';
// 数据集列表组件
const DatasetList = ({
datasets,
onViewDetails,
onDelete,
onEvaluate,
page,
rowsPerPage,
onPageChange,
onRowsPerPageChange,
total,
selectedIds,
onSelectAll,
onSelectItem,
evaluatingIds = [],
loading = false
}) => {
const theme = useTheme();
const { t } = useTranslation();
const bgColor = theme.palette.mode === 'dark' ? theme.palette.primary.dark : theme.palette.primary.light;
const color =
theme.palette.mode === 'dark'
? theme.palette.getContrastText(theme.palette.primary.main)
: theme.palette.getContrastText(theme.palette.primary.contrastText);
const RatingChip = ({ score }) => {
const config = getRatingConfigI18n(score, t);
return (
<Chip
icon={<StarIcon sx={{ fontSize: '14px !important' }} />}
label={`${formatScore(score)} ${config.label}`}
size="small"
sx={{
backgroundColor: config.backgroundColor,
color: config.color,
fontWeight: 'medium',
'& .MuiChip-icon': {
color: config.color
}
}}
/>
);
};
return (
<Card elevation={2}>
<Box sx={{ position: 'relative' }}>
<TableContainer sx={{ overflowX: 'auto' }}>
<Table sx={{ minWidth: 900 }}>
<TableHead>
<TableRow>
<TableCell
padding="checkbox"
sx={{
backgroundColor: bgColor,
color: color,
borderBottom: `2px solid ${theme.palette.divider}`
}}
>
<Checkbox
color="primary"
indeterminate={selectedIds.length > 0 && selectedIds.length < total}
checked={total > 0 && selectedIds.length === total}
onChange={onSelectAll}
/>
</TableCell>
<TableCell
sx={{
backgroundColor: bgColor,
color: color,
fontWeight: 'bold',
padding: '16px 8px',
borderBottom: `2px solid ${theme.palette.divider}`,
minWidth: 200
}}
>
{t('datasets.question')}
</TableCell>
<TableCell
sx={{
backgroundColor: bgColor,
color: color,
fontWeight: 'bold',
padding: '16px 8px',
borderBottom: `2px solid ${theme.palette.divider}`,
width: 120
}}
>
{t('datasets.rating', '评分')}
</TableCell>
<TableCell
sx={{
backgroundColor: bgColor,
color: color,
fontWeight: 'bold',
padding: '16px 8px',
borderBottom: `2px solid ${theme.palette.divider}`,
width: 100
}}
>
{t('datasets.model')}
</TableCell>
<TableCell
sx={{
backgroundColor: bgColor,
color: color,
fontWeight: 'bold',
padding: '16px 8px',
borderBottom: `2px solid ${theme.palette.divider}`,
width: 100
}}
>
{t('datasets.domainTag')}
</TableCell>
<TableCell
sx={{
backgroundColor: bgColor,
color: color,
fontWeight: 'bold',
padding: '16px 8px',
borderBottom: `2px solid ${theme.palette.divider}`,
width: 120
}}
>
{t('datasets.createdAt')}
</TableCell>
<TableCell
sx={{
backgroundColor: bgColor,
color: color,
fontWeight: 'bold',
padding: '16px 8px',
borderBottom: `2px solid ${theme.palette.divider}`,
width: 120
}}
>
{t('common.actions')}
</TableCell>
</TableRow>
</TableHead>
<TableBody>
{datasets.map((dataset, index) => (
<>
<TableRow
key={dataset.id}
sx={{
'&:nth-of-type(odd)': { backgroundColor: alpha(theme.palette.primary.light, 0.05) },
'&:hover': { backgroundColor: alpha(theme.palette.primary.light, 0.1) },
cursor: 'pointer'
}}
onClick={() => onViewDetails(dataset.id)}
>
<TableCell
padding="checkbox"
sx={{
borderLeft: `4px solid ${theme.palette.primary.main}`
}}
>
<Checkbox
color="primary"
checked={selectedIds.includes(dataset.id)}
onChange={e => {
e.stopPropagation();
onSelectItem(dataset.id);
}}
onClick={e => e.stopPropagation()}
/>
</TableCell>
<TableCell sx={{ py: 2 }}>
<Box>
<Typography
variant="body2"
fontWeight="medium"
sx={{
overflow: 'hidden',
textOverflow: 'ellipsis',
display: '-webkit-box',
WebkitLineClamp: 2,
WebkitBoxOrient: 'vertical',
lineHeight: 1.4,
mb: 0.5
}}
>
{dataset.question}
</Typography>
{dataset.confirmed && (
<Chip
label={t('datasets.confirmed')}
size="small"
sx={{
backgroundColor: alpha(theme.palette.success.main, 0.1),
color: theme.palette.success.dark,
fontWeight: 'medium',
height: 20,
fontSize: '0.7rem',
mt: 1
}}
/>
)}
</Box>
</TableCell>
<TableCell>
<RatingChip score={dataset.score || 0} />
</TableCell>
<TableCell>
<Chip
label={dataset.model}
size="small"
sx={{
backgroundColor: alpha(theme.palette.info.main, 0.1),
color: theme.palette.info.dark,
fontWeight: 'medium',
maxWidth: '100%',
'& .MuiChip-label': {
overflow: 'hidden',
textOverflow: 'ellipsis'
}
}}
/>
</TableCell>
<TableCell>
{dataset.questionLabel ? (
<Chip
label={dataset.questionLabel}
size="small"
sx={{
backgroundColor: alpha(theme.palette.primary.main, 0.1),
color: theme.palette.primary.dark,
fontWeight: 'medium',
maxWidth: '100%',
'& .MuiChip-label': {
overflow: 'hidden',
textOverflow: 'ellipsis'
}
}}
/>
) : (
<Typography variant="body2" color="text.disabled" fontSize="0.75rem">
{t('datasets.noTag')}
</Typography>
)}
</TableCell>
<TableCell>
<Typography variant="body2" color="text.secondary" fontSize="0.75rem">
{new Date(dataset.createAt).toLocaleDateString('zh-CN')}
</Typography>
</TableCell>
<TableCell>
<Box sx={{ display: 'flex', gap: 0.5 }}>
<Tooltip title={t('datasets.viewDetails')}>
<IconButton
size="small"
onClick={e => {
e.stopPropagation();
onViewDetails(dataset.id);
}}
sx={{
color: theme.palette.primary.main,
'&:hover': { backgroundColor: alpha(theme.palette.primary.main, 0.1) }
}}
>
<VisibilityIcon fontSize="small" />
</IconButton>
</Tooltip>
<Tooltip title={t('datasets.evaluate')}>
<IconButton
size="small"
disabled={evaluatingIds.includes(dataset.id)}
onClick={e => {
e.stopPropagation();
onEvaluate && onEvaluate(dataset);
}}
sx={{
color: theme.palette.secondary.main,
'&:hover': { backgroundColor: alpha(theme.palette.secondary.main, 0.1) }
}}
>
{evaluatingIds.includes(dataset.id) ? (
<CircularProgress size={20} sx={{ color: theme.palette.secondary.main }} />
) : (
<AssessmentIcon fontSize="small" />
)}
</IconButton>
</Tooltip>
<Tooltip title={t('common.delete')}>
<IconButton
size="small"
onClick={e => {
e.stopPropagation();
onDelete(dataset);
}}
sx={{
color: theme.palette.error.main,
'&:hover': { backgroundColor: alpha(theme.palette.error.main, 0.1) }
}}
>
<DeleteIcon fontSize="small" />
</IconButton>
</Tooltip>
</Box>
</TableCell>
</TableRow>
</>
))}
{datasets.length === 0 && (
<TableRow>
<TableCell colSpan={8} align="center" sx={{ py: 6 }}>
<Typography variant="body1" color="text.secondary">
{t('datasets.noData')}
</Typography>
</TableCell>
</TableRow>
)}
</TableBody>
</Table>
</TableContainer>
{loading && (
<Box
sx={{
position: 'absolute',
inset: 0,
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
justifyContent: 'center',
backgroundColor: alpha(theme.palette.background.paper, 0.6),
backdropFilter: 'blur(2px)',
zIndex: 1
}}
>
<CircularProgress size={32} />
<Typography variant="body2" sx={{ mt: 1 }} color="text.secondary">
{t('datasets.loading')}
</Typography>
</Box>
)}
</Box>
<Divider />
<Box
sx={{
display: 'flex',
alignItems: 'center',
justifyContent: 'space-between',
px: 2,
py: 1,
borderTop: `1px solid ${theme.palette.divider}`
}}
>
<TablePagination
component="div"
count={total}
page={page - 1}
onPageChange={onPageChange}
rowsPerPage={rowsPerPage}
onRowsPerPageChange={onRowsPerPageChange}
labelRowsPerPage={t('datasets.rowsPerPage')}
labelDisplayedRows={({ from, to, count }) => t('datasets.pagination', { from, to, count })}
sx={{
'.MuiTablePagination-selectLabel, .MuiTablePagination-displayedRows': {
fontWeight: 'medium'
},
border: 'none'
}}
/>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
<Typography variant="body2">{t('common.jumpTo')}:</Typography>
<TextField
size="small"
type="number"
inputProps={{
min: 1,
max: Math.ceil(total / rowsPerPage),
style: { padding: '4px 8px', width: '50px' }
}}
onKeyPress={e => {
if (e.key === 'Enter') {
const pageNum = parseInt(e.target.value, 10);
if (pageNum >= 1 && pageNum <= Math.ceil(total / rowsPerPage)) {
onPageChange(null, pageNum - 1);
e.target.value = '';
}
}
}}
/>
</Box>
</Box>
</Card>
);
};
export default DatasetList;

105
easy-dataset-main/app/projects/[projectId]/datasets/components/DeleteConfirmDialog.js Normal file
View File

@@ -0,0 +1,105 @@
'use client';
import {
Dialog,
DialogTitle,
DialogContent,
DialogActions,
Typography,
Paper,
Box,
LinearProgress,
Button,
useTheme,
alpha
} from '@mui/material';
import { useTranslation } from 'react-i18next';
const DeleteConfirmDialog = ({ open, datasets, onClose, onConfirm, batch, progress, deleting }) => {
const theme = useTheme();
const { t } = useTranslation();
const dataset = datasets?.[0];
return (
<Dialog
open={open}
onClose={onClose}
PaperProps={{
elevation: 3,
sx: { borderRadius: 2 }
}}
>
<DialogTitle sx={{ pb: 1 }}>
<Typography variant="h6" fontWeight="bold">
{t('common.confirmDelete')}
</Typography>
</DialogTitle>
<DialogContent sx={{ pb: 2, pt: 1 }}>
<Typography variant="body1" sx={{ mb: 2 }}>
{batch
? t('datasets.batchconfirmDeleteMessage', {
count: datasets.length
})
: t('common.confirmDeleteDataSet')}
</Typography>
{batch ? (
''
) : (
<Paper
variant="outlined"
sx={{
p: 2,
backgroundColor: alpha(theme.palette.warning.light, 0.1),
borderColor: theme.palette.warning.light
}}
>
<Typography variant="subtitle2" color="text.secondary" fontWeight="bold">
{t('datasets.question')}：
</Typography>
<Typography variant="body2">{dataset?.question}</Typography>
</Paper>
)}
{deleting && progress ? (
<Box sx={{ mb: 3 }}>
<Box sx={{ display: 'flex', alignItems: 'center', mb: 1 }}>
<Typography variant="body1" sx={{ mr: 1 }}>
{progress.percentage}%
</Typography>
<Box sx={{ width: '100%' }}>
<LinearProgress
variant="determinate"
value={progress.percentage}
sx={{
height: 8,
borderRadius: 4,
backgroundColor: alpha(theme.palette.primary.main, 0.1),
'& .MuiLinearProgress-bar': {
borderRadius: 4,
backgroundColor: theme.palette.primary.main
}
}}
/>
</Box>
</Box>
<Typography variant="body2" color="text.secondary">
{t('datasets.deletingProgress', '正在删除 {{completed}}/{{total}} 个数据集...', {
completed: progress.completed,
total: progress.total
})}
</Typography>
</Box>
) : null}
</DialogContent>
<DialogActions sx={{ px: 3, pb: 2 }}>
<Button onClick={onClose} disabled={deleting} sx={{ borderRadius: 2 }}>
{t('common.cancel')}
</Button>
<Button onClick={onConfirm} variant="contained" color="error" disabled={deleting} sx={{ borderRadius: 2 }}>
{deleting ? t('common.deleting') : t('common.delete')}
</Button>
</DialogActions>
</Dialog>
);
};
export default DeleteConfirmDialog;

198
easy-dataset-main/app/projects/[projectId]/datasets/components/FilterDialog.js Normal file
View File

@@ -0,0 +1,198 @@
'use client';
import {
Dialog,
DialogTitle,
DialogContent,
DialogActions,
Box,
Typography,
Select,
MenuItem,
Slider,
TextField,
Button,
InputAdornment
} from '@mui/material';
import SearchIcon from '@mui/icons-material/Search';
import { useTranslation } from 'react-i18next';
const FilterDialog = ({
open,
onClose,
filterConfirmed,
filterHasCot,
filterIsDistill,
filterScoreRange,
filterCustomTag,
filterNoteKeyword,
filterChunkName,
availableTags,
onFilterConfirmedChange,
onFilterHasCotChange,
onFilterIsDistillChange,
onFilterScoreRangeChange,
onFilterCustomTagChange,
onFilterNoteKeywordChange,
onFilterChunkNameChange,
onResetFilters,
onApplyFilters
}) => {
const { t } = useTranslation();
return (
<Dialog open={open} onClose={onClose} maxWidth="xs" fullWidth>
<DialogTitle>{t('datasets.filtersTitle')}</DialogTitle>
<DialogContent>
<Box sx={{ mb: 3, mt: 1 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterConfirmationStatus')}
</Typography>
<Select
value={filterConfirmed}
onChange={e => onFilterConfirmedChange(e.target.value)}
fullWidth
size="small"
sx={{ mt: 1 }}
>
<MenuItem value="all">{t('datasets.filterAll')}</MenuItem>
<MenuItem value="confirmed">{t('datasets.filterConfirmed')}</MenuItem>
<MenuItem value="unconfirmed">{t('datasets.filterUnconfirmed')}</MenuItem>
</Select>
</Box>
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterCotStatus')}
</Typography>
<Select
value={filterHasCot}
onChange={e => onFilterHasCotChange(e.target.value)}
fullWidth
size="small"
sx={{ mt: 1 }}
>
<MenuItem value="all">{t('datasets.filterAll')}</MenuItem>
<MenuItem value="yes">{t('datasets.filterHasCot')}</MenuItem>
<MenuItem value="no">{t('datasets.filterNoCot')}</MenuItem>
</Select>
</Box>
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterDistill')}
</Typography>
<Select
value={filterIsDistill}
onChange={e => onFilterIsDistillChange(e.target.value)}
fullWidth
size="small"
sx={{ mt: 1 }}
>
<MenuItem value="all">{t('datasets.filterAll')}</MenuItem>
<MenuItem value="yes">{t('datasets.filterDistillYes')}</MenuItem>
<MenuItem value="no">{t('datasets.filterDistillNo')}</MenuItem>
</Select>
</Box>
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterScoreRange')}
</Typography>
<Box sx={{ px: 1, mt: 2 }}>
<Slider
value={filterScoreRange}
onChange={(_, newValue) => onFilterScoreRangeChange(newValue)}
valueLabelDisplay="auto"
min={0}
max={5}
step={0.5}
marks={[
{ value: 0, label: '0' },
{ value: 2.5, label: '2.5' },
{ value: 5, label: '5' }
]}
sx={{ mt: 1 }}
/>
<Typography variant="caption" color="text.secondary">
{t('datasets.scoreRange', '{{min}} - {{max}} 分', {
min: filterScoreRange[0],
max: filterScoreRange[1]
})}
</Typography>
</Box>
</Box>
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterCustomTag')}
</Typography>
<Select
value={filterCustomTag}
onChange={e => onFilterCustomTagChange(e.target.value)}
fullWidth
size="small"
sx={{ mt: 1 }}
>
<MenuItem value="">{t('datasets.filterAll')}</MenuItem>
{availableTags.map(tag => (
<MenuItem key={tag} value={tag}>
{tag}
</MenuItem>
))}
</Select>
</Box>
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterNoteKeyword')}
</Typography>
<TextField
value={filterNoteKeyword}
onChange={e => onFilterNoteKeywordChange(e.target.value)}
placeholder={t('datasets.filterNoteKeywordPlaceholder')}
fullWidth
size="small"
sx={{ mt: 1 }}
InputProps={{
startAdornment: (
<InputAdornment position="start">
<SearchIcon fontSize="small" />
</InputAdornment>
)
}}
/>
</Box>
<Box sx={{ mb: 2 }}>
<Typography variant="subtitle2" gutterBottom>
{t('datasets.filterChunkName')}
</Typography>
<TextField
value={filterChunkName}
onChange={e => onFilterChunkNameChange(e.target.value)}
placeholder={t('datasets.filterChunkNamePlaceholder')}
fullWidth
size="small"
sx={{ mt: 1 }}
InputProps={{
startAdornment: (
<InputAdornment position="start">
<SearchIcon fontSize="small" />
</InputAdornment>
)
}}
/>
</Box>
</DialogContent>
<DialogActions>
<Button onClick={onResetFilters}>{t('datasets.resetFilters')}</Button>
<Button onClick={onApplyFilters} variant="contained">
{t('datasets.applyFilters')}
</Button>
</DialogActions>
</Dialog>
);
};
export default FilterDialog;

68
easy-dataset-main/app/projects/[projectId]/datasets/components/SearchBar.js Normal file
View File

@@ -0,0 +1,68 @@
'use client';
import { Box, Paper, IconButton, InputBase, Select, MenuItem, Button, Badge } from '@mui/material';
import SearchIcon from '@mui/icons-material/Search';
import FilterListIcon from '@mui/icons-material/FilterList';
import { useTranslation } from 'react-i18next';
const SearchBar = ({
searchQuery,
searchField,
onSearchQueryChange,
onSearchFieldChange,
onMoreFiltersClick,
activeFilterCount = 0
}) => {
const { t } = useTranslation();
return (
<Box sx={{ display: 'flex', gap: 2 }}>
<Paper
component="form"
sx={{
p: '2px 4px',
display: 'flex',
alignItems: 'center',
width: 400,
borderRadius: 2
}}
>
<IconButton sx={{ p: '10px' }} aria-label="search">
<SearchIcon />
</IconButton>
<InputBase
sx={{ ml: 1, flex: 1 }}
placeholder={t('datasets.searchPlaceholder')}
value={searchQuery}
onChange={e => onSearchQueryChange(e.target.value)}
endAdornment={
<Select
value={searchField}
onChange={e => onSearchFieldChange(e.target.value)}
variant="standard"
sx={{
minWidth: 90,
'& .MuiInput-underline:before': { borderBottom: 'none' },
'& .MuiInput-underline:after': { borderBottom: 'none' },
'& .MuiInput-underline:hover:not(.Mui-disabled):before': { borderBottom: 'none' }
}}
disableUnderline
>
<MenuItem value="question">{t('datasets.fieldQuestion')}</MenuItem>
<MenuItem value="answer">{t('datasets.fieldAnswer')}</MenuItem>
<MenuItem value="cot">{t('datasets.fieldCOT')}</MenuItem>
<MenuItem value="questionLabel">{t('datasets.fieldLabel')}</MenuItem>
</Select>
}
/>
</Paper>
<Badge badgeContent={activeFilterCount} color="error" overlap="circular">
<Button variant="outlined" onClick={onMoreFiltersClick} startIcon={<FilterListIcon />} sx={{ borderRadius: 2 }}>
{t('datasets.moreFilters')}
</Button>
</Badge>
</Box>
);
};
export default SearchBar;

165
easy-dataset-main/app/projects/[projectId]/datasets/hooks/useDatasetEvaluation.js Normal file
View File

@@ -0,0 +1,165 @@
'use client';
import { useState } from 'react';
import { useRouter } from 'next/navigation';
import { useTranslation } from 'react-i18next';
import { toast } from 'sonner';
import { useAtomValue } from 'jotai';
import { selectedModelInfoAtom } from '@/lib/store';
/**
* 数据集评估相关的自定义 Hook
* 封装单个评估和批量评估的逻辑
*/
const useDatasetEvaluation = (projectId, onEvaluationComplete) => {
const router = useRouter();
const { t } = useTranslation();
const model = useAtomValue(selectedModelInfoAtom);
// 评估状态管理
const [evaluatingIds, setEvaluatingIds] = useState([]);
const [batchEvaluating, setBatchEvaluating] = useState(false);
/**
* 检查模型是否已配置
*/
const checkModelConfiguration = () => {
if (!model || !model.modelName) {
toast.error(t('datasets.selectModelFirst', '请先选择模型'));
return false;
}
return true;
};
/**
* 处理单个数据集评估
* @param {Object} dataset - 要评估的数据集对象
*/
const handleEvaluateDataset = async dataset => {
// 检查模型配置
if (!checkModelConfiguration()) {
return;
}
try {
// 添加到评估中的ID列表
setEvaluatingIds(prev => [...prev, dataset.id]);
// 调用评估接口
const evaluateResponse = await fetch(`/api/projects/${projectId}/datasets/${dataset.id}/evaluate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
language: 'zh-CN'
})
});
const result = await evaluateResponse.json();
if (result.success) {
toast.success(
t('datasets.evaluateSuccess', '评估完成！评分：{{score}}/5', {
score: result.data.score
})
);
// 调用回调函数通知评估完成（通常用于刷新数据列表）
if (onEvaluationComplete) {
await onEvaluationComplete();
}
} else {
toast.error(result.message || t('datasets.evaluateFailed', '评估失败'));
}
} catch (error) {
console.error('评估失败:', error);
toast.error(
t('datasets.evaluateError', '评估失败: {{error}}', {
error: error.message
})
);
} finally {
// 从评估中的ID列表移除
setEvaluatingIds(prev => prev.filter(id => id !== dataset.id));
}
};
/**
* 处理批量评估
*/
const handleBatchEvaluate = async () => {
// 检查模型配置
if (!checkModelConfiguration()) {
return;
}
try {
setBatchEvaluating(true);
// 调用批量评估接口
const response = await fetch(`/api/projects/${projectId}/datasets/batch-evaluate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
language: 'zh-CN'
})
});
const result = await response.json();
if (result.success) {
toast.success(t('datasets.batchEvaluateStarted', '批量评估任务已启动，将在后台进行处理'));
// 跳转到任务页面查看进度
router.push(`/projects/${projectId}/tasks`);
} else {
toast.error(result.message || t('datasets.batchEvaluateStartFailed', '启动批量评估失败'));
}
} catch (error) {
console.error('批量评估失败:', error);
toast.error(
t('datasets.batchEvaluateFailed', '批量评估失败: {{error}}', {
error: error.message
})
);
} finally {
setBatchEvaluating(false);
}
};
/**
* 检查指定数据集是否正在评估中
* @param {string} datasetId - 数据集ID
* @returns {boolean} 是否正在评估中
*/
const isEvaluating = datasetId => {
return evaluatingIds.includes(datasetId);
};
/**
* 获取当前正在评估的数据集数量
* @returns {number} 正在评估的数据集数量
*/
const getEvaluatingCount = () => {
return evaluatingIds.length;
};
return {
// 状态
evaluatingIds,
batchEvaluating,
// 方法
handleEvaluateDataset,
handleBatchEvaluate,
// 工具方法
isEvaluating,
getEvaluatingCount,
// 模型信息（便于组件使用）
model
};
};
export default useDatasetEvaluation;

487
easy-dataset-main/app/projects/[projectId]/datasets/hooks/useDatasetExport.js Normal file
View File

@@ -0,0 +1,487 @@
'use client';
import { useTranslation } from 'react-i18next';
import { toast } from 'sonner';
import axios from 'axios';
const useDatasetExport = projectId => {
const { t } = useTranslation();
// 优化的流式导出 - 使用 WritableStream 避免内存溢出
const exportDatasetsStreaming = async (exportOptions, onProgress) => {
try {
const batchSize = exportOptions.batchSize || 1000;
let offset = 0;
let hasMore = true;
let totalProcessed = 0;
let isFirstBatch = true;
// 确定文件格式
const fileFormat = exportOptions.fileFormat || 'json';
const formatType = exportOptions.formatType || 'alpaca';
// 生成文件名
const formatSuffixMap = {
alpaca: 'alpaca',
multilingualthinking: 'multilingual-thinking',
sharegpt: 'sharegpt',
custom: 'custom'
};
const formatSuffix = formatSuffixMap[formatType] || formatType || 'export';
const balanceSuffix = exportOptions.balanceMode ? '-balanced' : '';
const dateStr = new Date().toISOString().slice(0, 10);
const fileName = `datasets-${projectId}-${formatSuffix}${balanceSuffix}-${dateStr}.${fileFormat}`;
// 创建可写流
let fileStream;
let writer;
try {
// 使用 showSaveFilePicker API（现代浏览器）
if (window.showSaveFilePicker) {
const handle = await window.showSaveFilePicker({
suggestedName: fileName,
types: [
{
description: 'Dataset File',
accept: {
'application/json': [`.${fileFormat}`]
}
}
]
});
fileStream = await handle.createWritable();
} else {
// 降级方案：使用内存缓冲区（但分块处理）
fileStream = null;
}
} catch (err) {
// 用户取消或不支持，使用降级方案
fileStream = null;
}
// 如果不支持流式写入，使用分块累积方案
let chunks = [];
let chunkCount = 0;
const MAX_CHUNKS_IN_MEMORY = 5; // 最多在内存中保留5批数据
// 写入文件头（JSON数组开始或CSV表头）
if (fileFormat === 'json') {
if (fileStream) {
await fileStream.write('[\n');
} else {
chunks.push('[\n');
}
} else if (fileFormat === 'csv') {
// 写入CSV表头
const headers = getCSVHeaders(formatType, exportOptions);
const headerLine = headers.join(',') + '\n';
if (fileStream) {
await fileStream.write(headerLine);
} else {
chunks.push(headerLine);
}
}
// 分批获取和写入数据
while (hasMore) {
const apiUrl = `/api/projects/${projectId}/datasets/export`;
const requestBody = {
batchMode: true,
offset: offset,
batchSize: batchSize
};
// 如果有选中的数据集 ID，传递 ID 列表
if (exportOptions.selectedIds && exportOptions.selectedIds.length > 0) {
requestBody.selectedIds = exportOptions.selectedIds;
} else if (exportOptions.confirmedOnly) {
requestBody.status = 'confirmed';
}
// 检查是否是平衡导出模式
if (exportOptions.balanceMode && exportOptions.balanceConfig) {
requestBody.balanceMode = true;
requestBody.balanceConfig = exportOptions.balanceConfig;
}
const response = await axios.post(apiUrl, requestBody);
const batchResult = response.data;
// 如果需要包含文本块内容，批量查询并填充
if (exportOptions.customFields?.includeChunk && batchResult.data.length > 0) {
const chunkNames = batchResult.data.map(item => item.chunkName).filter(name => name);
if (chunkNames.length > 0) {
try {
const chunkResponse = await axios.post(`/api/projects/${projectId}/chunks/batch-content`, {
chunkNames
});
const chunkContentMap = chunkResponse.data;
batchResult.data.forEach(item => {
if (item.chunkName && chunkContentMap[item.chunkName]) {
item.chunkContent = chunkContentMap[item.chunkName];
}
});
} catch (chunkError) {
console.error('获取文本块内容失败:', chunkError);
}
}
}
// 转换当前批次数据
const formattedBatch = formatDataBatch(batchResult.data, exportOptions);
// 写入当前批次
if (fileFormat === 'json') {
// 保持与原逻辑一致：JSON 导出为“格式化后的 JSON 数组”（2空格缩进）
// 每条记录单独 stringify + 缩进，并在数组级别拼接，避免一次性 stringify 全量数据导致内存暴涨
const batchContent = formattedBatch
.map(item => {
const pretty = JSON.stringify(item, null, 2);
// 将对象的每一行整体再缩进 2 个空格，以符合数组元素缩进
return ' ' + pretty.replace(/\n/g, '\n ');
})
.join(',\n');
const content = isFirstBatch ? batchContent : ',\n' + batchContent;
if (fileStream) {
await fileStream.write(content);
} else {
chunks.push(content);
chunkCount++;
}
} else if (fileFormat === 'jsonl') {
const batchContent = formattedBatch.map(item => JSON.stringify(item)).join('\n') + '\n';
if (fileStream) {
await fileStream.write(batchContent);
} else {
chunks.push(batchContent);
chunkCount++;
}
} else if (fileFormat === 'csv') {
const batchContent = formatBatchToCSV(formattedBatch, formatType, exportOptions);
if (fileStream) {
await fileStream.write(batchContent);
} else {
chunks.push(batchContent);
chunkCount++;
}
}
// 如果使用内存缓冲且累积了足够多的块，触发部分下载
if (!fileStream && chunkCount >= MAX_CHUNKS_IN_MEMORY) {
// 这里我们仍然需要等到最后才能下载，但至少限制了内存使用
// 可以考虑使用 Blob 分片
}
hasMore = batchResult.hasMore;
offset = batchResult.offset;
totalProcessed += batchResult.data.length;
isFirstBatch = false;
// 通知进度更新
if (onProgress) {
onProgress({
processed: totalProcessed,
currentBatch: batchResult.data.length,
hasMore
});
}
// 避免过快请求
if (hasMore) {
await new Promise(resolve => setTimeout(resolve, 50));
}
}
// 写入文件尾
if (fileFormat === 'json') {
if (fileStream) {
await fileStream.write('\n]\n');
await fileStream.close();
} else {
chunks.push('\n]\n');
}
} else {
if (fileStream) {
await fileStream.close();
}
}
// 如果使用内存缓冲方案，现在触发下载
if (!fileStream) {
downloadFromChunks(chunks, fileName);
}
toast.success(t('datasets.exportSuccess'));
return true;
} catch (error) {
console.error('Streaming export failed:', error);
toast.error(error.message || t('datasets.exportFailed'));
return false;
}
};
// 从内存块下载文件（优化版本，使用 Blob 流）
const downloadFromChunks = (chunks, fileName) => {
// 使用 Blob 构造函数，它会自动处理大数据
const blob = new Blob(chunks, { type: 'application/octet-stream' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = fileName;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
// 延迟释放 URL，确保下载开始
setTimeout(() => URL.revokeObjectURL(url), 1000);
};
// 获取CSV表头
const getCSVHeaders = (formatType, exportOptions) => {
if (formatType === 'alpaca') {
return ['instruction', 'input', 'output', 'system'];
} else if (formatType === 'sharegpt') {
return ['messages'];
} else if (formatType === 'multilingualthinking') {
return ['reasoning_language', 'developer', 'user', 'analysis', 'final', 'messages'];
} else if (formatType === 'custom') {
const { questionField, answerField, cotField, includeLabels, includeChunk, questionOnly } =
exportOptions.customFields;
const headers = [questionField];
if (!questionOnly) {
headers.push(answerField);
if (exportOptions.includeCOT && cotField) {
headers.push(cotField);
}
}
if (includeLabels) headers.push('label');
if (includeChunk) headers.push('chunk');
return headers;
}
return [];
};
// 格式化数据批次
const formatDataBatch = (dataBatch, exportOptions) => {
const formatType = exportOptions.formatType || 'alpaca';
if (formatType === 'alpaca') {
if (exportOptions.alpacaFieldType === 'instruction') {
return dataBatch.map(({ question, answer, cot }) => ({
instruction: question,
input: '',
output: cot && exportOptions.includeCOT ? `<think>${cot}</think>\n${answer}` : answer,
system: exportOptions.systemPrompt || ''
}));
} else {
return dataBatch.map(({ question, answer, cot }) => ({
instruction: exportOptions.customInstruction || '',
input: question,
output: cot && exportOptions.includeCOT ? `<think>${cot}</think>\n${answer}` : answer,
system: exportOptions.systemPrompt || ''
}));
}
} else if (formatType === 'sharegpt') {
return dataBatch.map(({ question, answer, cot }) => {
const messages = [];
if (exportOptions.systemPrompt) {
messages.push({ role: 'system', content: exportOptions.systemPrompt });
}
messages.push({
role: 'user',
content: question
});
messages.push({
role: 'assistant',
content: cot && exportOptions.includeCOT ? `<think>${cot}</think>\n${answer}` : answer
});
return { messages };
});
} else if (formatType === 'multilingualthinking') {
return dataBatch.map(({ question, answer, cot }) => ({
reasoning_language: exportOptions.reasoningLanguage || 'English',
developer: exportOptions.systemPrompt || '',
user: question,
analysis: exportOptions.includeCOT && cot ? cot : null,
final: answer,
messages: [
{
content: exportOptions.systemPrompt || '',
role: 'system',
thinking: null
},
{
content: question,
role: 'user',
thinking: null
},
{
content: answer,
role: 'assistant',
thinking: exportOptions.includeCOT && cot ? cot : null
}
]
}));
} else if (formatType === 'custom') {
const { questionField, answerField, cotField, includeLabels, includeChunk, questionOnly } =
exportOptions.customFields;
return dataBatch.map(({ question, answer, cot, questionLabel: labels, chunkContent }) => {
const item = { [questionField]: question };
if (!questionOnly) {
item[answerField] = answer;
if (cot && exportOptions.includeCOT && cotField) {
item[cotField] = cot;
}
}
if (includeLabels && labels && labels.length > 0) {
item.label = labels.split(' ')[1];
}
if (includeChunk && chunkContent) {
item.chunk = chunkContent;
}
return item;
});
}
return dataBatch;
};
// 将批次格式化为CSV行
const formatBatchToCSV = (formattedBatch, formatType, exportOptions) => {
const headers = getCSVHeaders(formatType, exportOptions);
return (
formattedBatch
.map(item => {
return headers
.map(header => {
let field = item[header]?.toString() || '';
// 对于复杂对象，转换为JSON字符串
if (typeof item[header] === 'object') {
field = JSON.stringify(item[header]);
}
// CSV转义
if (field.includes(',') || field.includes('\n') || field.includes('"')) {
field = `"${field.replace(/"/g, '""')}"`;
}
return field;
})
.join(',');
})
.join('\n') + '\n'
);
};
// 处理和下载数据的通用函数（保留用于小数据量）
const processAndDownloadData = async (dataToExport, exportOptions) => {
const formattedData = formatDataBatch(dataToExport, exportOptions);
let content;
let fileExtension;
const fileFormat = exportOptions.fileFormat || 'json';
if (fileFormat === 'jsonl') {
content = formattedData.map(item => JSON.stringify(item)).join('\n');
fileExtension = 'jsonl';
} else if (fileFormat === 'csv') {
const headers = getCSVHeaders(exportOptions.formatType, exportOptions);
const csvRows = [
headers.join(','),
...formattedData.map(item =>
headers
.map(header => {
let field = item[header]?.toString() || '';
if (typeof item[header] === 'object') {
field = JSON.stringify(item[header]);
}
if (field.includes(',') || field.includes('\n') || field.includes('"')) {
field = `"${field.replace(/"/g, '""')}"`;
}
return field;
})
.join(',')
)
];
content = csvRows.join('\n');
fileExtension = 'csv';
} else {
content = JSON.stringify(formattedData, null, 2);
fileExtension = 'json';
}
const blob = new Blob([content], { type: 'application/json' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
const formatSuffixMap = {
alpaca: 'alpaca',
multilingualthinking: 'multilingual-thinking',
sharegpt: 'sharegpt',
custom: 'custom'
};
const formatSuffix = formatSuffixMap[exportOptions.formatType] || exportOptions.formatType || 'export';
const balanceSuffix = exportOptions.balanceMode ? '-balanced' : '';
const dateStr = new Date().toISOString().slice(0, 10);
a.download = `datasets-${projectId}-${formatSuffix}${balanceSuffix}-${dateStr}.${fileExtension}`;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(url);
};
// 导出数据集（保持向后兼容的原有功能）
const exportDatasets = async exportOptions => {
try {
const apiUrl = `/api/projects/${projectId}/datasets/export`;
const requestBody = {};
if (exportOptions.selectedIds && exportOptions.selectedIds.length > 0) {
requestBody.selectedIds = exportOptions.selectedIds;
} else if (exportOptions.confirmedOnly) {
requestBody.status = 'confirmed';
}
if (exportOptions.balanceMode && exportOptions.balanceConfig) {
requestBody.balanceMode = true;
requestBody.balanceConfig = exportOptions.balanceConfig;
}
const response = await axios.post(apiUrl, requestBody);
let dataToExport = response.data;
await processAndDownloadData(dataToExport, exportOptions);
toast.success(t('datasets.exportSuccess'));
return true;
} catch (error) {
toast.error(error.message);
return false;
}
};
// 导出平衡数据集
const exportBalancedDataset = async exportOptions => {
const balancedOptions = {
...exportOptions,
balanceMode: true,
balanceConfig: exportOptions.balanceConfig
};
return await exportDatasets(balancedOptions);
};
return {
exportDatasets,
exportBalancedDataset,
exportDatasetsStreaming
};
};
export default useDatasetExport;
export { useDatasetExport };

171
easy-dataset-main/app/projects/[projectId]/datasets/hooks/useDatasetFilters.js Normal file
View File

@@ -0,0 +1,171 @@
'use client';
import { useState, useEffect } from 'react';
/**
* 数据集筛选条件持久化 Hook
* 负责筛选条件的保存、恢复和管理
* @param {string} projectId - 项目ID
* @returns {Object} 筛选条件和相关方法
*/
export function useDatasetFilters(projectId) {
const [filterConfirmed, setFilterConfirmed] = useState('all');
const [filterHasCot, setFilterHasCot] = useState('all');
const [filterIsDistill, setFilterIsDistill] = useState('all');
const [filterScoreRange, setFilterScoreRange] = useState([0, 5]);
const [filterCustomTag, setFilterCustomTag] = useState('');
const [filterNoteKeyword, setFilterNoteKeyword] = useState('');
const [filterChunkName, setFilterChunkName] = useState('');
const [searchQuery, setSearchQuery] = useState('');
const [searchField, setSearchField] = useState('question');
const [page, setPage] = useState(1);
const [rowsPerPage, setRowsPerPage] = useState(10);
const [isInitialized, setIsInitialized] = useState(false);
// 从 localStorage 恢复筛选条件
useEffect(() => {
if (typeof window !== 'undefined') {
try {
const savedFilters = localStorage.getItem(`datasets-filters-${projectId}`);
if (savedFilters) {
const filters = JSON.parse(savedFilters);
setFilterConfirmed(filters.filterConfirmed || 'all');
setFilterHasCot(filters.filterHasCot || 'all');
setFilterIsDistill(filters.filterIsDistill || 'all');
setFilterScoreRange(filters.filterScoreRange || [0, 5]);
setFilterCustomTag(filters.filterCustomTag || '');
setFilterNoteKeyword(filters.filterNoteKeyword || '');
setFilterChunkName(filters.filterChunkName || '');
setSearchQuery(filters.searchQuery || '');
setSearchField(filters.searchField || 'question');
setPage(filters.page || 1);
setRowsPerPage(filters.rowsPerPage || 10);
}
} catch (error) {
console.error('恢复筛选条件失败:', error);
}
setIsInitialized(true);
}
}, [projectId]);
// 保存筛选条件到 localStorage
useEffect(() => {
if (typeof window !== 'undefined' && isInitialized) {
try {
const filters = {
filterConfirmed,
filterHasCot,
filterIsDistill,
filterScoreRange,
filterCustomTag,
filterNoteKeyword,
filterChunkName,
searchQuery,
searchField,
page,
rowsPerPage
};
localStorage.setItem(`datasets-filters-${projectId}`, JSON.stringify(filters));
} catch (error) {
console.error('保存筛选条件失败:', error);
}
}
}, [
projectId,
filterConfirmed,
filterHasCot,
filterIsDistill,
filterScoreRange,
filterCustomTag,
filterNoteKeyword,
filterChunkName,
searchQuery,
searchField,
page,
rowsPerPage,
isInitialized
]);
/**
* 重置所有筛选条件为默认值
*/
const resetFilters = () => {
setFilterConfirmed('all');
setFilterHasCot('all');
setFilterIsDistill('all');
setFilterScoreRange([0, 5]);
setFilterCustomTag('');
setFilterNoteKeyword('');
setFilterChunkName('');
setSearchQuery('');
setSearchField('question');
setPage(1);
setRowsPerPage(10);
};
/**
* 清除 localStorage 中的筛选条件
*/
const clearSavedFilters = () => {
if (typeof window !== 'undefined') {
try {
localStorage.removeItem(`datasets-filters-${projectId}`);
} catch (error) {
console.error('清除筛选条件失败:', error);
}
}
};
/**
* 计算当前活跃的筛选条件数量
* @returns {number} 活跃筛选条件的数量
*/
const getActiveFilterCount = () => {
let count = 0;
if (filterConfirmed !== 'all') count++;
if (filterHasCot !== 'all') count++;
if (filterIsDistill !== 'all') count++;
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) count++;
if (filterCustomTag) count++;
if (filterNoteKeyword) count++;
if (filterChunkName) count++;
return count;
};
return {
// 筛选条件状态
filterConfirmed,
setFilterConfirmed,
filterHasCot,
setFilterHasCot,
filterIsDistill,
setFilterIsDistill,
filterScoreRange,
setFilterScoreRange,
filterCustomTag,
setFilterCustomTag,
filterNoteKeyword,
setFilterNoteKeyword,
filterChunkName,
setFilterChunkName,
searchQuery,
setSearchQuery,
searchField,
setSearchField,
// 分页状态
page,
setPage,
rowsPerPage,
setRowsPerPage,
// 初始化状态
isInitialized,
// 工具方法
resetFilters,
clearSavedFilters,
getActiveFilterCount
};
}
export default useDatasetFilters;

596
easy-dataset-main/app/projects/[projectId]/datasets/page.js Normal file
View File

@@ -0,0 +1,596 @@
'use client';
import { useState, useEffect, useCallback } from 'react';
import { Container, Box, Typography, Button, Card, useTheme, alpha } from '@mui/material';
import DeleteIcon from '@mui/icons-material/Delete';
import { useRouter } from 'next/navigation';
import ExportDatasetDialog from '@/components/ExportDatasetDialog';
import ExportProgressDialog from '@/components/ExportProgressDialog';
import ImportDatasetDialog from '@/components/datasets/ImportDatasetDialog';
import { useTranslation } from 'react-i18next';
import DatasetList from './components/DatasetList';
import SearchBar from './components/SearchBar';
import ActionBar from './components/ActionBar';
import FilterDialog from './components/FilterDialog';
import DeleteConfirmDialog from './components/DeleteConfirmDialog';
import useDatasetExport from './hooks/useDatasetExport';
import useDatasetEvaluation from './hooks/useDatasetEvaluation';
import useDatasetFilters from './hooks/useDatasetFilters';
import { processInParallel } from '@/lib/util/async';
import axios from 'axios';
import { useDebounce } from '@/hooks/useDebounce';
import { toast } from 'sonner';
// 主页面组件
export default function DatasetsPage({ params }) {
const { projectId } = params;
const router = useRouter();
const theme = useTheme();
const [datasets, setDatasets] = useState({ data: [], total: 0, confirmedCount: 0 });
const [loading, setLoading] = useState(true);
const [deleteDialog, setDeleteDialog] = useState({
open: false,
datasets: null,
batch: false,
deleting: false
});
const [exportDialog, setExportDialog] = useState({ open: false });
const [importDialog, setImportDialog] = useState({ open: false });
const [selectedIds, setselectedIds] = useState([]);
const [availableTags, setAvailableTags] = useState([]);
const [filterDialogOpen, setFilterDialogOpen] = useState(false);
const { t } = useTranslation();
// 使用 useDatasetFilters Hook 管理筛选条件
const {
filterConfirmed,
setFilterConfirmed,
filterHasCot,
setFilterHasCot,
filterIsDistill,
setFilterIsDistill,
filterScoreRange,
setFilterScoreRange,
filterCustomTag,
setFilterCustomTag,
filterNoteKeyword,
setFilterNoteKeyword,
filterChunkName,
setFilterChunkName,
searchQuery,
setSearchQuery,
searchField,
setSearchField,
page,
setPage,
rowsPerPage,
setRowsPerPage,
isInitialized,
getActiveFilterCount
} = useDatasetFilters(projectId);
const debouncedSearchQuery = useDebounce(searchQuery);
// 删除进度状态
const [deleteProgress, setDeteleProgress] = useState({
total: 0, // 总删除问题数量
completed: 0, // 已删除完成的数量
percentage: 0 // 进度百分比
});
// 导出进度状态
const [exportProgress, setExportProgress] = useState({
show: false, // 是否显示进度
processed: 0, // 已处理数量
total: 0, // 总数量
hasMore: true // 是否还有更多数据
});
// 3. 添加打开导出对话框的处理函数
const handleOpenExportDialog = () => {
setExportDialog({ open: true });
};
// 4. 添加关闭导出对话框的处理函数
const handleCloseExportDialog = () => {
setExportDialog({ open: false });
};
// 5. 添加打开导入对话框的处理函数
const handleOpenImportDialog = () => {
setImportDialog({ open: true });
};
// 6. 添加关闭导入对话框的处理函数
const handleCloseImportDialog = () => {
setImportDialog({ open: false });
};
// 7. 导入成功后的处理函数
const handleImportSuccess = () => {
// 刷新数据集列表
getDatasetsList();
toast.success(t('import.importSuccess', '数据集导入成功'));
};
// 获取数据集列表
const getDatasetsList = useCallback(
async ({ pageOverride } = {}) => {
const effectivePage = pageOverride ?? page;
try {
setLoading(true);
let url = `/api/projects/${projectId}/datasets?page=${effectivePage}&size=${rowsPerPage}`;
if (filterConfirmed !== 'all') {
url += `&status=${filterConfirmed}`;
}
if (debouncedSearchQuery) {
url += `&input=${encodeURIComponent(debouncedSearchQuery)}&field=${searchField}`;
}
if (filterHasCot !== 'all') {
url += `&hasCot=${filterHasCot}`;
}
if (filterIsDistill !== 'all') {
url += `&isDistill=${filterIsDistill}`;
}
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) {
url += `&scoreRange=${filterScoreRange[0]}-${filterScoreRange[1]}`;
}
if (filterCustomTag) {
url += `&customTag=${encodeURIComponent(filterCustomTag)}`;
}
if (filterNoteKeyword) {
url += `&noteKeyword=${encodeURIComponent(filterNoteKeyword)}`;
}
if (filterChunkName) {
url += `&chunkName=${encodeURIComponent(filterChunkName)}`;
}
const response = await axios.get(url);
setDatasets(response.data || { data: [], total: 0, confirmedCount: 0 });
} catch (error) {
toast.error(error.message);
} finally {
setLoading(false);
}
},
[
debouncedSearchQuery,
filterConfirmed,
filterCustomTag,
filterHasCot,
filterIsDistill,
filterNoteKeyword,
filterChunkName,
filterScoreRange,
page,
projectId,
rowsPerPage,
searchField
]
);
useEffect(() => {
if (!isInitialized) return;
getDatasetsList();
// 获取项目中所有使用过的标签
const fetchAvailableTags = async () => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets/tags`);
if (response.ok) {
const data = await response.json();
setAvailableTags(data.tags || []);
}
} catch (error) {
console.error('获取标签失败:', error);
}
};
fetchAvailableTags();
}, [projectId, page, rowsPerPage, debouncedSearchQuery, searchField, isInitialized]);
// 处理页码变化
const handlePageChange = (_event, newPage) => {
// MUI TablePagination 的页码从 0 开始，而我们的 API 从 1 开始
setPage(newPage + 1);
};
// 处理每页行数变化
const handleRowsPerPageChange = event => {
setPage(1);
setRowsPerPage(parseInt(event.target.value, 10));
};
// 打开删除确认框
const handleOpenDeleteDialog = dataset => {
setDeleteDialog({
open: true,
datasets: [dataset]
});
};
// 关闭删除确认框
const handleCloseDeleteDialog = () => {
setDeleteDialog({
open: false,
dataset: null
});
};
const handleBatchDeleteDataset = async () => {
const datasetsArray = selectedIds.map(id => ({ id }));
setDeleteDialog({
open: true,
datasets: datasetsArray,
batch: true,
count: selectedIds.length
});
};
const resetProgress = () => {
setDeteleProgress({
total: deleteDialog.count,
completed: 0,
percentage: 0
});
};
const handleDeleteConfirm = async () => {
if (deleteDialog.batch) {
setDeleteDialog({
...deleteDialog,
deleting: true
});
await handleBatchDelete();
resetProgress();
} else {
const [dataset] = deleteDialog.datasets;
if (!dataset) return;
await handleDelete(dataset);
}
setselectedIds([]);
// 刷新数据
getDatasetsList();
// 关闭确认框
handleCloseDeleteDialog();
};
// 批量删除数据集
const handleBatchDelete = async () => {
try {
await processInParallel(
selectedIds,
async datasetId => {
await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
method: 'DELETE'
});
},
3,
(cur, total) => {
setDeteleProgress({
total,
completed: cur,
percentage: Math.floor((cur / total) * 100)
});
}
);
toast.success(t('common.deleteSuccess'));
} catch (error) {
console.error('批量删除失败:', error);
toast.error(error.message || t('common.deleteFailed'));
}
};
// 删除数据集
const handleDelete = async dataset => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets?id=${dataset.id}`, {
method: 'DELETE'
});
if (!response.ok) throw new Error(t('datasets.deleteFailed'));
toast.success(t('datasets.deleteSuccess'));
} catch (error) {
toast.error(error.message || t('datasets.deleteFailed'));
}
};
// 使用自定义 Hook 处理数据集导出逻辑
const { exportDatasets, exportDatasetsStreaming } = useDatasetExport(projectId);
// 使用自定义 Hook 处理数据集评估逻辑
const { evaluatingIds, batchEvaluating, handleEvaluateDataset, handleBatchEvaluate } = useDatasetEvaluation(
projectId,
getDatasetsList
);
// 处理导出数据集 - 智能选择导出方式
const handleExportDatasets = async exportOptions => {
try {
// 如果是平衡导出，则忽略选中项，按 balanceConfig 导出
const exportOptionsWithSelection = exportOptions.balanceMode
? { ...exportOptions }
: { ...exportOptions, ...(selectedIds.length > 0 && { selectedIds }) };
// 获取数据总量：
// 平衡导出时，按 balanceConfig 的总量计算；
// 其他情况：如果有选中数据集则使用选中数量，否则使用当前筛选条件下的数据总量
const balancedTotal = Array.isArray(exportOptions.balanceConfig)
? exportOptions.balanceConfig.reduce((sum, c) => sum + (parseInt(c.maxCount) || 0), 0)
: 0;
const totalCount = exportOptions.balanceMode
? balancedTotal
: selectedIds.length > 0
? selectedIds.length
: datasets.total || 0;
// 设置阈值：超过1000条数据使用流式导出
const STREAMING_THRESHOLD = 1000;
// 检查是否需要包含文本块内容
const needsChunkContent = exportOptions.formatType === 'custom' && exportOptions.customFields?.includeChunk;
let success = false;
// 如果数据量大于阈值或需要查询文本块内容，使用流式导出
if (totalCount > STREAMING_THRESHOLD || needsChunkContent) {
// 使用流式导出，显示进度
setExportProgress({ show: true, processed: 0, total: totalCount });
success = await exportDatasetsStreaming(exportOptionsWithSelection, progress => {
setExportProgress(prev => ({
...prev,
processed: progress.processed,
hasMore: progress.hasMore
}));
});
// 隐藏进度
setExportProgress({ show: false, processed: 0, total: 0 });
} else {
// 使用传统导出方式
success = await exportDatasets(exportOptionsWithSelection);
}
if (success) {
// 关闭export对话框
handleCloseExportDialog();
}
} catch (error) {
console.error('Export failed:', error);
setExportProgress({ show: false, processed: 0, total: 0 });
}
};
// 查看详情
const handleViewDetails = id => {
router.push(`/projects/${projectId}/datasets/${id}`);
};
// 处理全选/取消全选
const handleSelectAll = async event => {
if (event.target.checked) {
// 获取所有符合当前筛选条件的数据，不受分页限制
let url = `/api/projects/${projectId}/datasets?selectedAll=1`;
if (filterConfirmed !== 'all') {
url += `&status=${filterConfirmed}`;
}
if (debouncedSearchQuery) {
url += `&input=${encodeURIComponent(debouncedSearchQuery)}&field=${searchField}`;
}
if (filterHasCot !== 'all') {
url += `&hasCot=${filterHasCot}`;
}
if (filterIsDistill !== 'all') {
url += `&isDistill=${filterIsDistill}`;
}
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) {
url += `&scoreRange=${filterScoreRange[0]}-${filterScoreRange[1]}`;
}
if (filterCustomTag) {
url += `&customTag=${encodeURIComponent(filterCustomTag)}`;
}
if (filterNoteKeyword) {
url += `&noteKeyword=${encodeURIComponent(filterNoteKeyword)}`;
}
const response = await axios.get(url);
setselectedIds(response.data.map(dataset => dataset.id));
} else {
setselectedIds([]);
}
};
// 处理单个选择
const handleSelectItem = id => {
setselectedIds(prev => {
if (prev.includes(id)) {
return prev.filter(item => item !== id);
} else {
return [...prev, id];
}
});
};
const handleResetFilters = useCallback(() => {
setFilterConfirmed('all');
setFilterHasCot('all');
setFilterIsDistill('all');
setFilterScoreRange([0, 5]);
setFilterCustomTag('');
setFilterNoteKeyword('');
setFilterChunkName('');
setPage(1);
getDatasetsList({ pageOverride: 1 });
}, [
getDatasetsList,
setFilterConfirmed,
setFilterHasCot,
setFilterIsDistill,
setFilterScoreRange,
setFilterCustomTag,
setFilterNoteKeyword,
setFilterChunkName,
setPage
]);
const handleApplyFilters = useCallback(() => {
setFilterDialogOpen(false);
setPage(1);
getDatasetsList({ pageOverride: 1 });
}, [getDatasetsList, setFilterDialogOpen, setPage]);
const handleCloseFilterDialog = useCallback(() => setFilterDialogOpen(false), [setFilterDialogOpen]);
return (
<Container maxWidth="xl" sx={{ mt: 4, mb: 6 }}>
<Card
elevation={0}
sx={{
mb: 4,
p: 3,
backgroundColor: alpha(theme.palette.primary.light, 0.05),
borderRadius: 2
}}
>
<Box
sx={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
flexWrap: 'wrap',
gap: 2
}}
>
<SearchBar
searchQuery={searchQuery}
searchField={searchField}
onSearchQueryChange={value => {
setSearchQuery(value);
setPage(1);
}}
onSearchFieldChange={value => {
setSearchField(value);
setPage(1);
}}
onMoreFiltersClick={() => setFilterDialogOpen(true)}
activeFilterCount={getActiveFilterCount()}
/>
<ActionBar
batchEvaluating={batchEvaluating}
onBatchEvaluate={handleBatchEvaluate}
onImport={handleOpenImportDialog}
onExport={handleOpenExportDialog}
/>
</Box>
</Card>
{selectedIds.length ? (
<Box
sx={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
flexWrap: 'wrap',
marginTop: '10px',
gap: 2
}}
>
<Typography variant="body1" color="text.secondary">
{t('datasets.selected', {
count: selectedIds.length
})}
</Typography>
<Button
variant="outlined"
color="error"
startIcon={<DeleteIcon />}
sx={{ borderRadius: 2 }}
onClick={handleBatchDeleteDataset}
>
{t('datasets.batchDelete')}
</Button>
</Box>
) : (
''
)}
<DatasetList
datasets={datasets.data || []}
onViewDetails={handleViewDetails}
onDelete={handleOpenDeleteDialog}
onEvaluate={handleEvaluateDataset}
page={page}
rowsPerPage={rowsPerPage}
onPageChange={handlePageChange}
onRowsPerPageChange={handleRowsPerPageChange}
total={datasets.total || 0}
selectedIds={selectedIds}
onSelectAll={handleSelectAll}
onSelectItem={handleSelectItem}
evaluatingIds={evaluatingIds}
loading={loading}
/>
<DeleteConfirmDialog
open={deleteDialog.open}
datasets={deleteDialog.datasets || []}
onClose={handleCloseDeleteDialog}
onConfirm={handleDeleteConfirm}
batch={deleteDialog.batch}
progress={deleteProgress}
deleting={deleteDialog.deleting}
/>
<FilterDialog
open={filterDialogOpen}
onClose={handleCloseFilterDialog}
filterConfirmed={filterConfirmed}
filterHasCot={filterHasCot}
filterIsDistill={filterIsDistill}
filterScoreRange={filterScoreRange}
filterCustomTag={filterCustomTag}
filterNoteKeyword={filterNoteKeyword}
filterChunkName={filterChunkName}
availableTags={availableTags}
onFilterConfirmedChange={setFilterConfirmed}
onFilterHasCotChange={setFilterHasCot}
onFilterIsDistillChange={setFilterIsDistill}
onFilterScoreRangeChange={setFilterScoreRange}
onFilterCustomTagChange={setFilterCustomTag}
onFilterNoteKeywordChange={setFilterNoteKeyword}
onFilterChunkNameChange={setFilterChunkName}
onResetFilters={handleResetFilters}
onApplyFilters={handleApplyFilters}
/>
<ExportDatasetDialog
open={exportDialog.open}
onClose={handleCloseExportDialog}
onExport={handleExportDatasets}
projectId={projectId}
/>
<ImportDatasetDialog
open={importDialog.open}
onClose={handleCloseImportDialog}
onImportSuccess={handleImportSuccess}
projectId={projectId}
/>
{/* 导出进度对话框 */}
<ExportProgressDialog open={exportProgress.show} progress={exportProgress} />
</Container>
);
}