Files
YG-Datasets/easy-dataset-main/app/projects/[projectId]/datasets/page.js

597 lines
18 KiB
JavaScript
Raw Normal View History

2026-03-17 14:36:31 +08:00
'use client';
import { useState, useEffect, useCallback } from 'react';
import { Container, Box, Typography, Button, Card, useTheme, alpha } from '@mui/material';
import DeleteIcon from '@mui/icons-material/Delete';
import { useRouter } from 'next/navigation';
import ExportDatasetDialog from '@/components/ExportDatasetDialog';
import ExportProgressDialog from '@/components/ExportProgressDialog';
import ImportDatasetDialog from '@/components/datasets/ImportDatasetDialog';
import { useTranslation } from 'react-i18next';
import DatasetList from './components/DatasetList';
import SearchBar from './components/SearchBar';
import ActionBar from './components/ActionBar';
import FilterDialog from './components/FilterDialog';
import DeleteConfirmDialog from './components/DeleteConfirmDialog';
import useDatasetExport from './hooks/useDatasetExport';
import useDatasetEvaluation from './hooks/useDatasetEvaluation';
import useDatasetFilters from './hooks/useDatasetFilters';
import { processInParallel } from '@/lib/util/async';
import axios from 'axios';
import { useDebounce } from '@/hooks/useDebounce';
import { toast } from 'sonner';
// 主页面组件
export default function DatasetsPage({ params }) {
const { projectId } = params;
const router = useRouter();
const theme = useTheme();
const [datasets, setDatasets] = useState({ data: [], total: 0, confirmedCount: 0 });
const [loading, setLoading] = useState(true);
const [deleteDialog, setDeleteDialog] = useState({
open: false,
datasets: null,
batch: false,
deleting: false
});
const [exportDialog, setExportDialog] = useState({ open: false });
const [importDialog, setImportDialog] = useState({ open: false });
const [selectedIds, setselectedIds] = useState([]);
const [availableTags, setAvailableTags] = useState([]);
const [filterDialogOpen, setFilterDialogOpen] = useState(false);
const { t } = useTranslation();
// 使用 useDatasetFilters Hook 管理筛选条件
const {
filterConfirmed,
setFilterConfirmed,
filterHasCot,
setFilterHasCot,
filterIsDistill,
setFilterIsDistill,
filterScoreRange,
setFilterScoreRange,
filterCustomTag,
setFilterCustomTag,
filterNoteKeyword,
setFilterNoteKeyword,
filterChunkName,
setFilterChunkName,
searchQuery,
setSearchQuery,
searchField,
setSearchField,
page,
setPage,
rowsPerPage,
setRowsPerPage,
isInitialized,
getActiveFilterCount
} = useDatasetFilters(projectId);
const debouncedSearchQuery = useDebounce(searchQuery);
// 删除进度状态
const [deleteProgress, setDeteleProgress] = useState({
total: 0, // 总删除问题数量
completed: 0, // 已删除完成的数量
percentage: 0 // 进度百分比
});
// 导出进度状态
const [exportProgress, setExportProgress] = useState({
show: false, // 是否显示进度
processed: 0, // 已处理数量
total: 0, // 总数量
hasMore: true // 是否还有更多数据
});
// 3. 添加打开导出对话框的处理函数
const handleOpenExportDialog = () => {
setExportDialog({ open: true });
};
// 4. 添加关闭导出对话框的处理函数
const handleCloseExportDialog = () => {
setExportDialog({ open: false });
};
// 5. 添加打开导入对话框的处理函数
const handleOpenImportDialog = () => {
setImportDialog({ open: true });
};
// 6. 添加关闭导入对话框的处理函数
const handleCloseImportDialog = () => {
setImportDialog({ open: false });
};
// 7. 导入成功后的处理函数
const handleImportSuccess = () => {
// 刷新数据集列表
getDatasetsList();
toast.success(t('import.importSuccess', '数据集导入成功'));
};
// 获取数据集列表
const getDatasetsList = useCallback(
async ({ pageOverride } = {}) => {
const effectivePage = pageOverride ?? page;
try {
setLoading(true);
let url = `/api/projects/${projectId}/datasets?page=${effectivePage}&size=${rowsPerPage}`;
if (filterConfirmed !== 'all') {
url += `&status=${filterConfirmed}`;
}
if (debouncedSearchQuery) {
url += `&input=${encodeURIComponent(debouncedSearchQuery)}&field=${searchField}`;
}
if (filterHasCot !== 'all') {
url += `&hasCot=${filterHasCot}`;
}
if (filterIsDistill !== 'all') {
url += `&isDistill=${filterIsDistill}`;
}
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) {
url += `&scoreRange=${filterScoreRange[0]}-${filterScoreRange[1]}`;
}
if (filterCustomTag) {
url += `&customTag=${encodeURIComponent(filterCustomTag)}`;
}
if (filterNoteKeyword) {
url += `&noteKeyword=${encodeURIComponent(filterNoteKeyword)}`;
}
if (filterChunkName) {
url += `&chunkName=${encodeURIComponent(filterChunkName)}`;
}
const response = await axios.get(url);
setDatasets(response.data || { data: [], total: 0, confirmedCount: 0 });
} catch (error) {
toast.error(error.message);
} finally {
setLoading(false);
}
},
[
debouncedSearchQuery,
filterConfirmed,
filterCustomTag,
filterHasCot,
filterIsDistill,
filterNoteKeyword,
filterChunkName,
filterScoreRange,
page,
projectId,
rowsPerPage,
searchField
]
);
useEffect(() => {
if (!isInitialized) return;
getDatasetsList();
// 获取项目中所有使用过的标签
const fetchAvailableTags = async () => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets/tags`);
if (response.ok) {
const data = await response.json();
setAvailableTags(data.tags || []);
}
} catch (error) {
console.error('获取标签失败:', error);
}
};
fetchAvailableTags();
}, [projectId, page, rowsPerPage, debouncedSearchQuery, searchField, isInitialized]);
// 处理页码变化
const handlePageChange = (_event, newPage) => {
// MUI TablePagination 的页码从 0 开始,而我们的 API 从 1 开始
setPage(newPage + 1);
};
// 处理每页行数变化
const handleRowsPerPageChange = event => {
setPage(1);
setRowsPerPage(parseInt(event.target.value, 10));
};
// 打开删除确认框
const handleOpenDeleteDialog = dataset => {
setDeleteDialog({
open: true,
datasets: [dataset]
});
};
// 关闭删除确认框
const handleCloseDeleteDialog = () => {
setDeleteDialog({
open: false,
dataset: null
});
};
const handleBatchDeleteDataset = async () => {
const datasetsArray = selectedIds.map(id => ({ id }));
setDeleteDialog({
open: true,
datasets: datasetsArray,
batch: true,
count: selectedIds.length
});
};
const resetProgress = () => {
setDeteleProgress({
total: deleteDialog.count,
completed: 0,
percentage: 0
});
};
const handleDeleteConfirm = async () => {
if (deleteDialog.batch) {
setDeleteDialog({
...deleteDialog,
deleting: true
});
await handleBatchDelete();
resetProgress();
} else {
const [dataset] = deleteDialog.datasets;
if (!dataset) return;
await handleDelete(dataset);
}
setselectedIds([]);
// 刷新数据
getDatasetsList();
// 关闭确认框
handleCloseDeleteDialog();
};
// 批量删除数据集
const handleBatchDelete = async () => {
try {
await processInParallel(
selectedIds,
async datasetId => {
await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
method: 'DELETE'
});
},
3,
(cur, total) => {
setDeteleProgress({
total,
completed: cur,
percentage: Math.floor((cur / total) * 100)
});
}
);
toast.success(t('common.deleteSuccess'));
} catch (error) {
console.error('批量删除失败:', error);
toast.error(error.message || t('common.deleteFailed'));
}
};
// 删除数据集
const handleDelete = async dataset => {
try {
const response = await fetch(`/api/projects/${projectId}/datasets?id=${dataset.id}`, {
method: 'DELETE'
});
if (!response.ok) throw new Error(t('datasets.deleteFailed'));
toast.success(t('datasets.deleteSuccess'));
} catch (error) {
toast.error(error.message || t('datasets.deleteFailed'));
}
};
// 使用自定义 Hook 处理数据集导出逻辑
const { exportDatasets, exportDatasetsStreaming } = useDatasetExport(projectId);
// 使用自定义 Hook 处理数据集评估逻辑
const { evaluatingIds, batchEvaluating, handleEvaluateDataset, handleBatchEvaluate } = useDatasetEvaluation(
projectId,
getDatasetsList
);
// 处理导出数据集 - 智能选择导出方式
const handleExportDatasets = async exportOptions => {
try {
// 如果是平衡导出,则忽略选中项,按 balanceConfig 导出
const exportOptionsWithSelection = exportOptions.balanceMode
? { ...exportOptions }
: { ...exportOptions, ...(selectedIds.length > 0 && { selectedIds }) };
// 获取数据总量:
// 平衡导出时,按 balanceConfig 的总量计算;
// 其他情况:如果有选中数据集则使用选中数量,否则使用当前筛选条件下的数据总量
const balancedTotal = Array.isArray(exportOptions.balanceConfig)
? exportOptions.balanceConfig.reduce((sum, c) => sum + (parseInt(c.maxCount) || 0), 0)
: 0;
const totalCount = exportOptions.balanceMode
? balancedTotal
: selectedIds.length > 0
? selectedIds.length
: datasets.total || 0;
// 设置阈值超过1000条数据使用流式导出
const STREAMING_THRESHOLD = 1000;
// 检查是否需要包含文本块内容
const needsChunkContent = exportOptions.formatType === 'custom' && exportOptions.customFields?.includeChunk;
let success = false;
// 如果数据量大于阈值或需要查询文本块内容,使用流式导出
if (totalCount > STREAMING_THRESHOLD || needsChunkContent) {
// 使用流式导出,显示进度
setExportProgress({ show: true, processed: 0, total: totalCount });
success = await exportDatasetsStreaming(exportOptionsWithSelection, progress => {
setExportProgress(prev => ({
...prev,
processed: progress.processed,
hasMore: progress.hasMore
}));
});
// 隐藏进度
setExportProgress({ show: false, processed: 0, total: 0 });
} else {
// 使用传统导出方式
success = await exportDatasets(exportOptionsWithSelection);
}
if (success) {
// 关闭export对话框
handleCloseExportDialog();
}
} catch (error) {
console.error('Export failed:', error);
setExportProgress({ show: false, processed: 0, total: 0 });
}
};
// 查看详情
const handleViewDetails = id => {
router.push(`/projects/${projectId}/datasets/${id}`);
};
// 处理全选/取消全选
const handleSelectAll = async event => {
if (event.target.checked) {
// 获取所有符合当前筛选条件的数据,不受分页限制
let url = `/api/projects/${projectId}/datasets?selectedAll=1`;
if (filterConfirmed !== 'all') {
url += `&status=${filterConfirmed}`;
}
if (debouncedSearchQuery) {
url += `&input=${encodeURIComponent(debouncedSearchQuery)}&field=${searchField}`;
}
if (filterHasCot !== 'all') {
url += `&hasCot=${filterHasCot}`;
}
if (filterIsDistill !== 'all') {
url += `&isDistill=${filterIsDistill}`;
}
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) {
url += `&scoreRange=${filterScoreRange[0]}-${filterScoreRange[1]}`;
}
if (filterCustomTag) {
url += `&customTag=${encodeURIComponent(filterCustomTag)}`;
}
if (filterNoteKeyword) {
url += `&noteKeyword=${encodeURIComponent(filterNoteKeyword)}`;
}
const response = await axios.get(url);
setselectedIds(response.data.map(dataset => dataset.id));
} else {
setselectedIds([]);
}
};
// 处理单个选择
const handleSelectItem = id => {
setselectedIds(prev => {
if (prev.includes(id)) {
return prev.filter(item => item !== id);
} else {
return [...prev, id];
}
});
};
const handleResetFilters = useCallback(() => {
setFilterConfirmed('all');
setFilterHasCot('all');
setFilterIsDistill('all');
setFilterScoreRange([0, 5]);
setFilterCustomTag('');
setFilterNoteKeyword('');
setFilterChunkName('');
setPage(1);
getDatasetsList({ pageOverride: 1 });
}, [
getDatasetsList,
setFilterConfirmed,
setFilterHasCot,
setFilterIsDistill,
setFilterScoreRange,
setFilterCustomTag,
setFilterNoteKeyword,
setFilterChunkName,
setPage
]);
const handleApplyFilters = useCallback(() => {
setFilterDialogOpen(false);
setPage(1);
getDatasetsList({ pageOverride: 1 });
}, [getDatasetsList, setFilterDialogOpen, setPage]);
const handleCloseFilterDialog = useCallback(() => setFilterDialogOpen(false), [setFilterDialogOpen]);
return (
<Container maxWidth="xl" sx={{ mt: 4, mb: 6 }}>
<Card
elevation={0}
sx={{
mb: 4,
p: 3,
backgroundColor: alpha(theme.palette.primary.light, 0.05),
borderRadius: 2
}}
>
<Box
sx={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
flexWrap: 'wrap',
gap: 2
}}
>
<SearchBar
searchQuery={searchQuery}
searchField={searchField}
onSearchQueryChange={value => {
setSearchQuery(value);
setPage(1);
}}
onSearchFieldChange={value => {
setSearchField(value);
setPage(1);
}}
onMoreFiltersClick={() => setFilterDialogOpen(true)}
activeFilterCount={getActiveFilterCount()}
/>
<ActionBar
batchEvaluating={batchEvaluating}
onBatchEvaluate={handleBatchEvaluate}
onImport={handleOpenImportDialog}
onExport={handleOpenExportDialog}
/>
</Box>
</Card>
{selectedIds.length ? (
<Box
sx={{
display: 'flex',
justifyContent: 'space-between',
alignItems: 'center',
flexWrap: 'wrap',
marginTop: '10px',
gap: 2
}}
>
<Typography variant="body1" color="text.secondary">
{t('datasets.selected', {
count: selectedIds.length
})}
</Typography>
<Button
variant="outlined"
color="error"
startIcon={<DeleteIcon />}
sx={{ borderRadius: 2 }}
onClick={handleBatchDeleteDataset}
>
{t('datasets.batchDelete')}
</Button>
</Box>
) : (
''
)}
<DatasetList
datasets={datasets.data || []}
onViewDetails={handleViewDetails}
onDelete={handleOpenDeleteDialog}
onEvaluate={handleEvaluateDataset}
page={page}
rowsPerPage={rowsPerPage}
onPageChange={handlePageChange}
onRowsPerPageChange={handleRowsPerPageChange}
total={datasets.total || 0}
selectedIds={selectedIds}
onSelectAll={handleSelectAll}
onSelectItem={handleSelectItem}
evaluatingIds={evaluatingIds}
loading={loading}
/>
<DeleteConfirmDialog
open={deleteDialog.open}
datasets={deleteDialog.datasets || []}
onClose={handleCloseDeleteDialog}
onConfirm={handleDeleteConfirm}
batch={deleteDialog.batch}
progress={deleteProgress}
deleting={deleteDialog.deleting}
/>
<FilterDialog
open={filterDialogOpen}
onClose={handleCloseFilterDialog}
filterConfirmed={filterConfirmed}
filterHasCot={filterHasCot}
filterIsDistill={filterIsDistill}
filterScoreRange={filterScoreRange}
filterCustomTag={filterCustomTag}
filterNoteKeyword={filterNoteKeyword}
filterChunkName={filterChunkName}
availableTags={availableTags}
onFilterConfirmedChange={setFilterConfirmed}
onFilterHasCotChange={setFilterHasCot}
onFilterIsDistillChange={setFilterIsDistill}
onFilterScoreRangeChange={setFilterScoreRange}
onFilterCustomTagChange={setFilterCustomTag}
onFilterNoteKeywordChange={setFilterNoteKeyword}
onFilterChunkNameChange={setFilterChunkName}
onResetFilters={handleResetFilters}
onApplyFilters={handleApplyFilters}
/>
<ExportDatasetDialog
open={exportDialog.open}
onClose={handleCloseExportDialog}
onExport={handleExportDatasets}
projectId={projectId}
/>
<ImportDatasetDialog
open={importDialog.open}
onClose={handleCloseImportDialog}
onImportSuccess={handleImportSuccess}
projectId={projectId}
/>
{/* 导出进度对话框 */}
<ExportProgressDialog open={exportProgress.show} progress={exportProgress} />
</Container>
);
}