597 lines
18 KiB
JavaScript
597 lines
18 KiB
JavaScript
'use client';
|
||
|
||
import { useState, useEffect, useCallback } from 'react';
|
||
import { Container, Box, Typography, Button, Card, useTheme, alpha } from '@mui/material';
|
||
import DeleteIcon from '@mui/icons-material/Delete';
|
||
import { useRouter } from 'next/navigation';
|
||
import ExportDatasetDialog from '@/components/ExportDatasetDialog';
|
||
import ExportProgressDialog from '@/components/ExportProgressDialog';
|
||
import ImportDatasetDialog from '@/components/datasets/ImportDatasetDialog';
|
||
import { useTranslation } from 'react-i18next';
|
||
import DatasetList from './components/DatasetList';
|
||
import SearchBar from './components/SearchBar';
|
||
import ActionBar from './components/ActionBar';
|
||
import FilterDialog from './components/FilterDialog';
|
||
import DeleteConfirmDialog from './components/DeleteConfirmDialog';
|
||
import useDatasetExport from './hooks/useDatasetExport';
|
||
import useDatasetEvaluation from './hooks/useDatasetEvaluation';
|
||
import useDatasetFilters from './hooks/useDatasetFilters';
|
||
import { processInParallel } from '@/lib/util/async';
|
||
import axios from 'axios';
|
||
import { useDebounce } from '@/hooks/useDebounce';
|
||
import { toast } from 'sonner';
|
||
|
||
// 主页面组件
|
||
export default function DatasetsPage({ params }) {
|
||
const { projectId } = params;
|
||
const router = useRouter();
|
||
const theme = useTheme();
|
||
const [datasets, setDatasets] = useState({ data: [], total: 0, confirmedCount: 0 });
|
||
const [loading, setLoading] = useState(true);
|
||
const [deleteDialog, setDeleteDialog] = useState({
|
||
open: false,
|
||
datasets: null,
|
||
batch: false,
|
||
deleting: false
|
||
});
|
||
const [exportDialog, setExportDialog] = useState({ open: false });
|
||
const [importDialog, setImportDialog] = useState({ open: false });
|
||
const [selectedIds, setselectedIds] = useState([]);
|
||
const [availableTags, setAvailableTags] = useState([]);
|
||
const [filterDialogOpen, setFilterDialogOpen] = useState(false);
|
||
const { t } = useTranslation();
|
||
|
||
// 使用 useDatasetFilters Hook 管理筛选条件
|
||
const {
|
||
filterConfirmed,
|
||
setFilterConfirmed,
|
||
filterHasCot,
|
||
setFilterHasCot,
|
||
filterIsDistill,
|
||
setFilterIsDistill,
|
||
filterScoreRange,
|
||
setFilterScoreRange,
|
||
filterCustomTag,
|
||
setFilterCustomTag,
|
||
filterNoteKeyword,
|
||
setFilterNoteKeyword,
|
||
filterChunkName,
|
||
setFilterChunkName,
|
||
searchQuery,
|
||
setSearchQuery,
|
||
searchField,
|
||
setSearchField,
|
||
page,
|
||
setPage,
|
||
rowsPerPage,
|
||
setRowsPerPage,
|
||
isInitialized,
|
||
getActiveFilterCount
|
||
} = useDatasetFilters(projectId);
|
||
|
||
const debouncedSearchQuery = useDebounce(searchQuery);
|
||
// 删除进度状态
|
||
const [deleteProgress, setDeteleProgress] = useState({
|
||
total: 0, // 总删除问题数量
|
||
completed: 0, // 已删除完成的数量
|
||
percentage: 0 // 进度百分比
|
||
});
|
||
// 导出进度状态
|
||
const [exportProgress, setExportProgress] = useState({
|
||
show: false, // 是否显示进度
|
||
processed: 0, // 已处理数量
|
||
total: 0, // 总数量
|
||
hasMore: true // 是否还有更多数据
|
||
});
|
||
|
||
// 3. 添加打开导出对话框的处理函数
|
||
const handleOpenExportDialog = () => {
|
||
setExportDialog({ open: true });
|
||
};
|
||
|
||
// 4. 添加关闭导出对话框的处理函数
|
||
const handleCloseExportDialog = () => {
|
||
setExportDialog({ open: false });
|
||
};
|
||
|
||
// 5. 添加打开导入对话框的处理函数
|
||
const handleOpenImportDialog = () => {
|
||
setImportDialog({ open: true });
|
||
};
|
||
|
||
// 6. 添加关闭导入对话框的处理函数
|
||
const handleCloseImportDialog = () => {
|
||
setImportDialog({ open: false });
|
||
};
|
||
|
||
// 7. 导入成功后的处理函数
|
||
const handleImportSuccess = () => {
|
||
// 刷新数据集列表
|
||
getDatasetsList();
|
||
toast.success(t('import.importSuccess', '数据集导入成功'));
|
||
};
|
||
|
||
// 获取数据集列表
|
||
const getDatasetsList = useCallback(
|
||
async ({ pageOverride } = {}) => {
|
||
const effectivePage = pageOverride ?? page;
|
||
try {
|
||
setLoading(true);
|
||
let url = `/api/projects/${projectId}/datasets?page=${effectivePage}&size=${rowsPerPage}`;
|
||
|
||
if (filterConfirmed !== 'all') {
|
||
url += `&status=${filterConfirmed}`;
|
||
}
|
||
|
||
if (debouncedSearchQuery) {
|
||
url += `&input=${encodeURIComponent(debouncedSearchQuery)}&field=${searchField}`;
|
||
}
|
||
|
||
if (filterHasCot !== 'all') {
|
||
url += `&hasCot=${filterHasCot}`;
|
||
}
|
||
|
||
if (filterIsDistill !== 'all') {
|
||
url += `&isDistill=${filterIsDistill}`;
|
||
}
|
||
|
||
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) {
|
||
url += `&scoreRange=${filterScoreRange[0]}-${filterScoreRange[1]}`;
|
||
}
|
||
|
||
if (filterCustomTag) {
|
||
url += `&customTag=${encodeURIComponent(filterCustomTag)}`;
|
||
}
|
||
|
||
if (filterNoteKeyword) {
|
||
url += `¬eKeyword=${encodeURIComponent(filterNoteKeyword)}`;
|
||
}
|
||
|
||
if (filterChunkName) {
|
||
url += `&chunkName=${encodeURIComponent(filterChunkName)}`;
|
||
}
|
||
|
||
const response = await axios.get(url);
|
||
setDatasets(response.data || { data: [], total: 0, confirmedCount: 0 });
|
||
} catch (error) {
|
||
toast.error(error.message);
|
||
} finally {
|
||
setLoading(false);
|
||
}
|
||
},
|
||
[
|
||
debouncedSearchQuery,
|
||
filterConfirmed,
|
||
filterCustomTag,
|
||
filterHasCot,
|
||
filterIsDistill,
|
||
filterNoteKeyword,
|
||
filterChunkName,
|
||
filterScoreRange,
|
||
page,
|
||
projectId,
|
||
rowsPerPage,
|
||
searchField
|
||
]
|
||
);
|
||
|
||
useEffect(() => {
|
||
if (!isInitialized) return;
|
||
|
||
getDatasetsList();
|
||
// 获取项目中所有使用过的标签
|
||
const fetchAvailableTags = async () => {
|
||
try {
|
||
const response = await fetch(`/api/projects/${projectId}/datasets/tags`);
|
||
if (response.ok) {
|
||
const data = await response.json();
|
||
setAvailableTags(data.tags || []);
|
||
}
|
||
} catch (error) {
|
||
console.error('获取标签失败:', error);
|
||
}
|
||
};
|
||
fetchAvailableTags();
|
||
}, [projectId, page, rowsPerPage, debouncedSearchQuery, searchField, isInitialized]);
|
||
|
||
// 处理页码变化
|
||
const handlePageChange = (_event, newPage) => {
|
||
// MUI TablePagination 的页码从 0 开始，而我们的 API 从 1 开始
|
||
setPage(newPage + 1);
|
||
};
|
||
|
||
// 处理每页行数变化
|
||
const handleRowsPerPageChange = event => {
|
||
setPage(1);
|
||
setRowsPerPage(parseInt(event.target.value, 10));
|
||
};
|
||
|
||
// 打开删除确认框
|
||
const handleOpenDeleteDialog = dataset => {
|
||
setDeleteDialog({
|
||
open: true,
|
||
datasets: [dataset]
|
||
});
|
||
};
|
||
|
||
// 关闭删除确认框
|
||
const handleCloseDeleteDialog = () => {
|
||
setDeleteDialog({
|
||
open: false,
|
||
dataset: null
|
||
});
|
||
};
|
||
|
||
const handleBatchDeleteDataset = async () => {
|
||
const datasetsArray = selectedIds.map(id => ({ id }));
|
||
setDeleteDialog({
|
||
open: true,
|
||
datasets: datasetsArray,
|
||
batch: true,
|
||
count: selectedIds.length
|
||
});
|
||
};
|
||
|
||
const resetProgress = () => {
|
||
setDeteleProgress({
|
||
total: deleteDialog.count,
|
||
completed: 0,
|
||
percentage: 0
|
||
});
|
||
};
|
||
|
||
const handleDeleteConfirm = async () => {
|
||
if (deleteDialog.batch) {
|
||
setDeleteDialog({
|
||
...deleteDialog,
|
||
deleting: true
|
||
});
|
||
await handleBatchDelete();
|
||
resetProgress();
|
||
} else {
|
||
const [dataset] = deleteDialog.datasets;
|
||
if (!dataset) return;
|
||
await handleDelete(dataset);
|
||
}
|
||
setselectedIds([]);
|
||
// 刷新数据
|
||
getDatasetsList();
|
||
// 关闭确认框
|
||
handleCloseDeleteDialog();
|
||
};
|
||
|
||
// 批量删除数据集
|
||
const handleBatchDelete = async () => {
|
||
try {
|
||
await processInParallel(
|
||
selectedIds,
|
||
async datasetId => {
|
||
await fetch(`/api/projects/${projectId}/datasets?id=${datasetId}`, {
|
||
method: 'DELETE'
|
||
});
|
||
},
|
||
3,
|
||
(cur, total) => {
|
||
setDeteleProgress({
|
||
total,
|
||
completed: cur,
|
||
percentage: Math.floor((cur / total) * 100)
|
||
});
|
||
}
|
||
);
|
||
|
||
toast.success(t('common.deleteSuccess'));
|
||
} catch (error) {
|
||
console.error('批量删除失败:', error);
|
||
toast.error(error.message || t('common.deleteFailed'));
|
||
}
|
||
};
|
||
|
||
// 删除数据集
|
||
const handleDelete = async dataset => {
|
||
try {
|
||
const response = await fetch(`/api/projects/${projectId}/datasets?id=${dataset.id}`, {
|
||
method: 'DELETE'
|
||
});
|
||
if (!response.ok) throw new Error(t('datasets.deleteFailed'));
|
||
|
||
toast.success(t('datasets.deleteSuccess'));
|
||
} catch (error) {
|
||
toast.error(error.message || t('datasets.deleteFailed'));
|
||
}
|
||
};
|
||
|
||
// 使用自定义 Hook 处理数据集导出逻辑
|
||
const { exportDatasets, exportDatasetsStreaming } = useDatasetExport(projectId);
|
||
|
||
// 使用自定义 Hook 处理数据集评估逻辑
|
||
const { evaluatingIds, batchEvaluating, handleEvaluateDataset, handleBatchEvaluate } = useDatasetEvaluation(
|
||
projectId,
|
||
getDatasetsList
|
||
);
|
||
|
||
// 处理导出数据集 - 智能选择导出方式
|
||
const handleExportDatasets = async exportOptions => {
|
||
try {
|
||
// 如果是平衡导出，则忽略选中项，按 balanceConfig 导出
|
||
const exportOptionsWithSelection = exportOptions.balanceMode
|
||
? { ...exportOptions }
|
||
: { ...exportOptions, ...(selectedIds.length > 0 && { selectedIds }) };
|
||
|
||
// 获取数据总量：
|
||
// 平衡导出时，按 balanceConfig 的总量计算；
|
||
// 其他情况：如果有选中数据集则使用选中数量，否则使用当前筛选条件下的数据总量
|
||
const balancedTotal = Array.isArray(exportOptions.balanceConfig)
|
||
? exportOptions.balanceConfig.reduce((sum, c) => sum + (parseInt(c.maxCount) || 0), 0)
|
||
: 0;
|
||
const totalCount = exportOptions.balanceMode
|
||
? balancedTotal
|
||
: selectedIds.length > 0
|
||
? selectedIds.length
|
||
: datasets.total || 0;
|
||
|
||
// 设置阈值：超过1000条数据使用流式导出
|
||
const STREAMING_THRESHOLD = 1000;
|
||
|
||
// 检查是否需要包含文本块内容
|
||
const needsChunkContent = exportOptions.formatType === 'custom' && exportOptions.customFields?.includeChunk;
|
||
|
||
let success = false;
|
||
|
||
// 如果数据量大于阈值或需要查询文本块内容，使用流式导出
|
||
if (totalCount > STREAMING_THRESHOLD || needsChunkContent) {
|
||
// 使用流式导出，显示进度
|
||
setExportProgress({ show: true, processed: 0, total: totalCount });
|
||
|
||
success = await exportDatasetsStreaming(exportOptionsWithSelection, progress => {
|
||
setExportProgress(prev => ({
|
||
...prev,
|
||
processed: progress.processed,
|
||
hasMore: progress.hasMore
|
||
}));
|
||
});
|
||
|
||
// 隐藏进度
|
||
setExportProgress({ show: false, processed: 0, total: 0 });
|
||
} else {
|
||
// 使用传统导出方式
|
||
success = await exportDatasets(exportOptionsWithSelection);
|
||
}
|
||
|
||
if (success) {
|
||
// 关闭export对话框
|
||
handleCloseExportDialog();
|
||
}
|
||
} catch (error) {
|
||
console.error('Export failed:', error);
|
||
setExportProgress({ show: false, processed: 0, total: 0 });
|
||
}
|
||
};
|
||
|
||
// 查看详情
|
||
const handleViewDetails = id => {
|
||
router.push(`/projects/${projectId}/datasets/${id}`);
|
||
};
|
||
|
||
// 处理全选/取消全选
|
||
const handleSelectAll = async event => {
|
||
if (event.target.checked) {
|
||
// 获取所有符合当前筛选条件的数据，不受分页限制
|
||
let url = `/api/projects/${projectId}/datasets?selectedAll=1`;
|
||
|
||
if (filterConfirmed !== 'all') {
|
||
url += `&status=${filterConfirmed}`;
|
||
}
|
||
|
||
if (debouncedSearchQuery) {
|
||
url += `&input=${encodeURIComponent(debouncedSearchQuery)}&field=${searchField}`;
|
||
}
|
||
|
||
if (filterHasCot !== 'all') {
|
||
url += `&hasCot=${filterHasCot}`;
|
||
}
|
||
|
||
if (filterIsDistill !== 'all') {
|
||
url += `&isDistill=${filterIsDistill}`;
|
||
}
|
||
|
||
if (filterScoreRange[0] > 0 || filterScoreRange[1] < 5) {
|
||
url += `&scoreRange=${filterScoreRange[0]}-${filterScoreRange[1]}`;
|
||
}
|
||
|
||
if (filterCustomTag) {
|
||
url += `&customTag=${encodeURIComponent(filterCustomTag)}`;
|
||
}
|
||
|
||
if (filterNoteKeyword) {
|
||
url += `¬eKeyword=${encodeURIComponent(filterNoteKeyword)}`;
|
||
}
|
||
|
||
const response = await axios.get(url);
|
||
setselectedIds(response.data.map(dataset => dataset.id));
|
||
} else {
|
||
setselectedIds([]);
|
||
}
|
||
};
|
||
|
||
// 处理单个选择
|
||
const handleSelectItem = id => {
|
||
setselectedIds(prev => {
|
||
if (prev.includes(id)) {
|
||
return prev.filter(item => item !== id);
|
||
} else {
|
||
return [...prev, id];
|
||
}
|
||
});
|
||
};
|
||
|
||
const handleResetFilters = useCallback(() => {
|
||
setFilterConfirmed('all');
|
||
setFilterHasCot('all');
|
||
setFilterIsDistill('all');
|
||
setFilterScoreRange([0, 5]);
|
||
setFilterCustomTag('');
|
||
setFilterNoteKeyword('');
|
||
setFilterChunkName('');
|
||
setPage(1);
|
||
getDatasetsList({ pageOverride: 1 });
|
||
}, [
|
||
getDatasetsList,
|
||
setFilterConfirmed,
|
||
setFilterHasCot,
|
||
setFilterIsDistill,
|
||
setFilterScoreRange,
|
||
setFilterCustomTag,
|
||
setFilterNoteKeyword,
|
||
setFilterChunkName,
|
||
setPage
|
||
]);
|
||
|
||
const handleApplyFilters = useCallback(() => {
|
||
setFilterDialogOpen(false);
|
||
setPage(1);
|
||
getDatasetsList({ pageOverride: 1 });
|
||
}, [getDatasetsList, setFilterDialogOpen, setPage]);
|
||
const handleCloseFilterDialog = useCallback(() => setFilterDialogOpen(false), [setFilterDialogOpen]);
|
||
|
||
return (
|
||
<Container maxWidth="xl" sx={{ mt: 4, mb: 6 }}>
|
||
<Card
|
||
elevation={0}
|
||
sx={{
|
||
mb: 4,
|
||
p: 3,
|
||
backgroundColor: alpha(theme.palette.primary.light, 0.05),
|
||
borderRadius: 2
|
||
}}
|
||
>
|
||
<Box
|
||
sx={{
|
||
display: 'flex',
|
||
justifyContent: 'space-between',
|
||
alignItems: 'center',
|
||
flexWrap: 'wrap',
|
||
gap: 2
|
||
}}
|
||
>
|
||
<SearchBar
|
||
searchQuery={searchQuery}
|
||
searchField={searchField}
|
||
onSearchQueryChange={value => {
|
||
setSearchQuery(value);
|
||
setPage(1);
|
||
}}
|
||
onSearchFieldChange={value => {
|
||
setSearchField(value);
|
||
setPage(1);
|
||
}}
|
||
onMoreFiltersClick={() => setFilterDialogOpen(true)}
|
||
activeFilterCount={getActiveFilterCount()}
|
||
/>
|
||
<ActionBar
|
||
batchEvaluating={batchEvaluating}
|
||
onBatchEvaluate={handleBatchEvaluate}
|
||
onImport={handleOpenImportDialog}
|
||
onExport={handleOpenExportDialog}
|
||
/>
|
||
</Box>
|
||
</Card>
|
||
{selectedIds.length ? (
|
||
<Box
|
||
sx={{
|
||
display: 'flex',
|
||
justifyContent: 'space-between',
|
||
alignItems: 'center',
|
||
flexWrap: 'wrap',
|
||
marginTop: '10px',
|
||
gap: 2
|
||
}}
|
||
>
|
||
<Typography variant="body1" color="text.secondary">
|
||
{t('datasets.selected', {
|
||
count: selectedIds.length
|
||
})}
|
||
</Typography>
|
||
<Button
|
||
variant="outlined"
|
||
color="error"
|
||
startIcon={<DeleteIcon />}
|
||
sx={{ borderRadius: 2 }}
|
||
onClick={handleBatchDeleteDataset}
|
||
>
|
||
{t('datasets.batchDelete')}
|
||
</Button>
|
||
</Box>
|
||
) : (
|
||
''
|
||
)}
|
||
|
||
<DatasetList
|
||
datasets={datasets.data || []}
|
||
onViewDetails={handleViewDetails}
|
||
onDelete={handleOpenDeleteDialog}
|
||
onEvaluate={handleEvaluateDataset}
|
||
page={page}
|
||
rowsPerPage={rowsPerPage}
|
||
onPageChange={handlePageChange}
|
||
onRowsPerPageChange={handleRowsPerPageChange}
|
||
total={datasets.total || 0}
|
||
selectedIds={selectedIds}
|
||
onSelectAll={handleSelectAll}
|
||
onSelectItem={handleSelectItem}
|
||
evaluatingIds={evaluatingIds}
|
||
loading={loading}
|
||
/>
|
||
|
||
<DeleteConfirmDialog
|
||
open={deleteDialog.open}
|
||
datasets={deleteDialog.datasets || []}
|
||
onClose={handleCloseDeleteDialog}
|
||
onConfirm={handleDeleteConfirm}
|
||
batch={deleteDialog.batch}
|
||
progress={deleteProgress}
|
||
deleting={deleteDialog.deleting}
|
||
/>
|
||
|
||
<FilterDialog
|
||
open={filterDialogOpen}
|
||
onClose={handleCloseFilterDialog}
|
||
filterConfirmed={filterConfirmed}
|
||
filterHasCot={filterHasCot}
|
||
filterIsDistill={filterIsDistill}
|
||
filterScoreRange={filterScoreRange}
|
||
filterCustomTag={filterCustomTag}
|
||
filterNoteKeyword={filterNoteKeyword}
|
||
filterChunkName={filterChunkName}
|
||
availableTags={availableTags}
|
||
onFilterConfirmedChange={setFilterConfirmed}
|
||
onFilterHasCotChange={setFilterHasCot}
|
||
onFilterIsDistillChange={setFilterIsDistill}
|
||
onFilterScoreRangeChange={setFilterScoreRange}
|
||
onFilterCustomTagChange={setFilterCustomTag}
|
||
onFilterNoteKeywordChange={setFilterNoteKeyword}
|
||
onFilterChunkNameChange={setFilterChunkName}
|
||
onResetFilters={handleResetFilters}
|
||
onApplyFilters={handleApplyFilters}
|
||
/>
|
||
|
||
<ExportDatasetDialog
|
||
open={exportDialog.open}
|
||
onClose={handleCloseExportDialog}
|
||
onExport={handleExportDatasets}
|
||
projectId={projectId}
|
||
/>
|
||
|
||
<ImportDatasetDialog
|
||
open={importDialog.open}
|
||
onClose={handleCloseImportDialog}
|
||
onImportSuccess={handleImportSuccess}
|
||
projectId={projectId}
|
||
/>
|
||
|
||
{/* 导出进度对话框 */}
|
||
<ExportProgressDialog open={exportProgress.show} progress={exportProgress} />
|
||
</Container>
|
||
);
|
||
}
|