202 lines
8.1 KiB
JavaScript
202 lines
8.1 KiB
JavaScript
|
/**
|
|||
|
|
* 训练服务模块
|
|||
|
|
* 处理训练任务相关的操作和进度跟踪
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
// 训练进度缓存
|
|||
|
|
window.trainingProgressCache = window.trainingProgressCache || {};
|
|||
|
|
// 使用 window 避免重复声明
|
|||
|
|
if (typeof window._progressRefreshTimer === 'undefined') {
|
|||
|
|
window._progressRefreshTimer = null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 渲染训练进度
|
|||
|
|
function renderTrainingProgress(val, row) {
|
|||
|
|
const progressData = window.trainingProgressCache[row.id];
|
|||
|
|
if (progressData && progressData.status === 'running') {
|
|||
|
|
if (progressData.progress > 0) {
|
|||
|
|
return `
|
|||
|
|
<div class="flex flex-col">
|
|||
|
|
<span class="text-sm font-medium text-primary">${progressData.progress}%</span>
|
|||
|
|
<span class="text-xs text-gray-500">${progressData.step || ''} ${progressData.speed || ''}</span>
|
|||
|
|
<span class="text-xs text-gray-400">ETA: ${progressData.eta || '--:--'}</span>
|
|||
|
|
</div>
|
|||
|
|
`;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return `${val || 0}%`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 刷新训练进度
|
|||
|
|
async function refreshTrainingProgress() {
|
|||
|
|
if (typeof window.currentPage !== 'string' || window.currentPage !== 'fine-tune') return;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const response = await fetch(`${window.API_BASE}/fine-tune`);
|
|||
|
|
const result = await response.json();
|
|||
|
|
|
|||
|
|
if (result.code === 0 && result.data) {
|
|||
|
|
// 刷新运行中或已完成的任务（有进度信息）
|
|||
|
|
const activeTasks = result.data.filter(task =>
|
|||
|
|
task.status === 'running' || task.status === 'pending'
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
for (const task of activeTasks) {
|
|||
|
|
try {
|
|||
|
|
// 并行获取进度和PID状态
|
|||
|
|
const [progressResponse, statusResponse] = await Promise.all([
|
|||
|
|
fetch(`${window.API_BASE}/fine-tune/progress/${task.id}`),
|
|||
|
|
fetch(`${window.API_BASE}/fine-tune/${task.id}`)
|
|||
|
|
]);
|
|||
|
|
const progressResult = await progressResponse.json();
|
|||
|
|
const statusResult = await statusResponse.json();
|
|||
|
|
|
|||
|
|
if (progressResult.code === 0 && progressResult.data) {
|
|||
|
|
window.trainingProgressCache[task.id] = progressResult.data;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 如果状态已改变（PID已结束），更新表格中的状态显示
|
|||
|
|
if (statusResult.code === 0 && statusResult.data) {
|
|||
|
|
const actualStatus = statusResult.data.status;
|
|||
|
|
if (task.status !== actualStatus) {
|
|||
|
|
// 找到对应的行并更新状态
|
|||
|
|
const row = document.querySelector(`tr[data-id="${task.id}"]`);
|
|||
|
|
if (row) {
|
|||
|
|
const statusCell = row.querySelector('td:nth-child(3)');
|
|||
|
|
if (statusCell) {
|
|||
|
|
statusCell.innerHTML = `<span class="px-2 py-1 rounded text-xs ${actualStatus === 'running' ? 'bg-green-100 text-green-700' : actualStatus === 'failed' ? 'bg-red-100 text-red-700' : 'bg-blue-100 text-blue-700'}">${actualStatus}</span>`;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (e) {
|
|||
|
|
console.warn(`获取任务 ${task.id} 信息失败:`, e);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.warn('刷新训练进度失败:', error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 检查并更新任务状态（用于 fine-tune 页面）
|
|||
|
|
async function checkAndUpdateTaskStatus() {
|
|||
|
|
if (typeof window.currentPage !== 'string' || window.currentPage !== 'fine-tune') return;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const response = await fetch(`${window.API_BASE}/fine-tune`);
|
|||
|
|
const result = await response.json();
|
|||
|
|
|
|||
|
|
if (result.code === 0 && result.data) {
|
|||
|
|
// 获取所有 running 状态的任务
|
|||
|
|
const runningTasks = result.data.filter(task => task.status === 'running');
|
|||
|
|
|
|||
|
|
for (const task of runningTasks) {
|
|||
|
|
try {
|
|||
|
|
// 调用 status API 获取实际状态（会检查 PID）
|
|||
|
|
const statusResponse = await fetch(`${window.API_BASE}/fine-tune/${task.id}`);
|
|||
|
|
const statusResult = await statusResponse.json();
|
|||
|
|
|
|||
|
|
if (statusResult.code === 0 && statusResult.data) {
|
|||
|
|
const actualStatus = statusResult.data.status;
|
|||
|
|
// 如果实际状态不是 running，更新表格显示
|
|||
|
|
if (actualStatus !== 'running') {
|
|||
|
|
const row = document.querySelector(`tr[data-id="${task.id}"]`);
|
|||
|
|
if (row) {
|
|||
|
|
const statusCell = row.querySelector('td:nth-child(3)');
|
|||
|
|
if (statusCell) {
|
|||
|
|
const statusClass = actualStatus === 'failed'
|
|||
|
|
? 'bg-red-100 text-red-700'
|
|||
|
|
: 'bg-blue-100 text-blue-700';
|
|||
|
|
statusCell.innerHTML = `<span class="px-2 py-1 rounded text-xs ${statusClass}">${actualStatus}</span>`;
|
|||
|
|
console.log(`[Status] 任务 ${task.id} 状态已更新: running -> ${actualStatus}`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (e) {
|
|||
|
|
console.warn(`检查任务 ${task.id} 状态失败:`, e);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.warn('检查任务状态失败:', error);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 启动训练进度自动刷新
|
|||
|
|
function startProgressRefresh() {
|
|||
|
|
stopProgressRefresh();
|
|||
|
|
window._progressRefreshTimer = setInterval(() => {
|
|||
|
|
refreshTrainingProgress();
|
|||
|
|
checkAndUpdateTaskStatus();
|
|||
|
|
}, 5000);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 停止训练进度刷新
|
|||
|
|
function stopProgressRefresh() {
|
|||
|
|
if (window._progressRefreshTimer) {
|
|||
|
|
clearInterval(window._progressRefreshTimer);
|
|||
|
|
window._progressRefreshTimer = null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 停止训练任务
|
|||
|
|
async function stopItem(taskId) {
|
|||
|
|
window.showConfirm('确认停止', '确定要停止这个训练任务吗？进程将被终止。', async () => {
|
|||
|
|
try {
|
|||
|
|
const response = await fetch(`${window.API_BASE}/fine-tune/stop/${taskId}`, {
|
|||
|
|
method: 'POST'
|
|||
|
|
});
|
|||
|
|
const result = await response.json();
|
|||
|
|
if (result.code === 0) {
|
|||
|
|
window.showMessage('成功', '训练任务已停止', 'success');
|
|||
|
|
// 刷新当前页面
|
|||
|
|
const activeLink = document.querySelector('.nav-link.sidebar-item-active');
|
|||
|
|
if (activeLink && typeof window.loadPage === 'function') {
|
|||
|
|
window.loadPage(activeLink.dataset.page);
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
window.showMessage('错误', result.message || '停止失败', 'error');
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
window.showMessage('错误', '停止失败: ' + error.message, 'error');
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 查看训练日志 - 跳转到日志页面
|
|||
|
|
async function viewTrainingLog(taskId, taskName) {
|
|||
|
|
window.loadPage('logs');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 查看调优任务日志 - 跳转到training-log.html页面
|
|||
|
|
function viewFineTuneLogs(taskId, taskName) {
|
|||
|
|
// 保存 taskId 到 sessionStorage
|
|||
|
|
sessionStorage.setItem('trainingLogTaskId', taskId.toString());
|
|||
|
|
sessionStorage.setItem('trainingLogTaskName', taskName);
|
|||
|
|
// 跳转到日志页面
|
|||
|
|
window.navigateToPage('training-log');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 跳转到训练日志二级页面
|
|||
|
|
function navigateToTrainingLog(taskId) {
|
|||
|
|
// 传递 taskId 到 sessionStorage
|
|||
|
|
sessionStorage.setItem('trainingLogTaskId', taskId.toString());
|
|||
|
|
// 跳转到日志页面
|
|||
|
|
window.navigateToPage('training-log');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 导出训练服务
|
|||
|
|
window.TrainingService = {
|
|||
|
|
renderTrainingProgress,
|
|||
|
|
refreshTrainingProgress,
|
|||
|
|
checkAndUpdateTaskStatus,
|
|||
|
|
startProgressRefresh,
|
|||
|
|
stopProgressRefresh,
|
|||
|
|
stopItem,
|
|||
|
|
viewTrainingLog,
|
|||
|
|
viewFineTuneLogs,
|
|||
|
|
navigateToTrainingLog
|
|||
|
|
};
|