Files
YG_FT_Platform/web/pages/training-log.html

741 lines
32 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>训练日志 / 远光软件微调平台</title>
<script src="../lib/tailwindcss/tailwind.js"></script>
<link href="../lib/font-awesome/css/font-awesome.min.css" rel="stylesheet">
<script src="../lib/chart.js/chart.min.js"></script>
<script>
// 确保 Chart.js 已加载
if (typeof Chart === 'undefined') {
console.error('Chart.js 未加载,尝试动态加载...');
// 备用:尝试动态加载
const script = document.createElement('script');
script.src = '../lib/chart.js/chart.umd.min.js';
script.onload = function() {
console.log('Chart.js 动态加载成功');
window.chartJsLoaded = true;
};
script.onerror = function() {
console.error('Chart.js 加载失败');
};
document.head.appendChild(script);
} else {
console.log('Chart.js 已加载');
window.chartJsLoaded = true;
}
</script>
<style>
.bg-primary { background-color: #1890ff; }
.text-primary { color: #1890ff; }
.border-primary { border-color: #1890ff; }
:root { --primary: #1890ff; --danger: #f5222d; --success: #52c41a; }
/* 日志样式 */
.log-content {
font-family: 'Consolas', 'Monaco', monospace;
font-size: 12px;
line-height: 1.5;
white-space: pre-wrap;
word-wrap: break-word;
}
.log-content .error { color: #dc3545; }
.log-content .warning { color: #d97706; }
.log-content .info { color: #0891b2; }
.log-content .success { color: #16a34a; }
.log-content .progress { color: #7c3aed; font-weight: bold; }
.log-line { padding: 1px 8px; }
.log-line:hover { background-color: rgba(24, 144, 255, 0.1); }
</style>
</head>
<body class="bg-gray-50 p-6">
<!-- 页面标题 -->
<div class="bg-white rounded-lg shadow-sm w-full p-4 border-b border-gray-100 mb-4">
<div class="flex items-center justify-between">
<div class="flex items-center text-sm">
<span class="text-gray-800 font-medium">训练日志</span>
</div>
<div class="flex items-center space-x-3">
<button onclick="toggleTB()" id="tbBtn" class="bg-purple-500 text-white px-4 py-2 rounded hover:bg-purple-600 transition-colors text-sm">
<i class="fa fa-bar-chart mr-1"></i>TensorBoard
</button>
</div>
</div>
</div>
<!-- 任务信息 -->
<div class="bg-white rounded-lg shadow-sm p-6 mb-6">
<div class="flex items-center justify-between mb-4">
<h2 class="text-lg font-medium text-gray-800" id="taskName">加载中...</h2>
<span id="taskStatus" class="px-3 py-1 rounded-full text-sm bg-gray-100 text-gray-600">加载中</span>
</div>
<div class="grid grid-cols-2 md:grid-cols-5 gap-4 text-sm">
<div>
<div class="text-gray-500 text-xs">基础模型</div>
<div id="baseModel" class="font-medium text-gray-800">-</div>
</div>
<div>
<div class="text-gray-500 text-xs">数据集</div>
<div id="dataset" class="font-medium text-gray-800">-</div>
</div>
<div>
<div class="text-gray-500 text-xs">创建时间</div>
<div id="createTime" class="font-medium text-gray-800">-</div>
</div>
<div>
<div class="text-gray-500 text-xs">进程ID</div>
<div id="processId" class="font-medium text-gray-800">-</div>
</div>
<div>
<div class="text-gray-500 text-xs">最后更新</div>
<div id="lastUpdate" class="font-medium text-gray-800">-</div>
</div>
</div>
</div>
<!-- 训练曲线图表 -->
<div id="chartsContainer" class="bg-white rounded-lg shadow-sm p-6 mb-6">
<h3 class="text-base font-medium text-gray-800 mb-4">训练曲线</h3>
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
<div>
<canvas id="lossChart" class="w-full h-48"></canvas>
</div>
<div>
<canvas id="gradNormChart" class="w-full h-48"></canvas>
</div>
<div>
<canvas id="learningRateChart" class="w-full h-48"></canvas>
</div>
</div>
</div>
<!-- 日志内容 -->
<div class="bg-white rounded-lg shadow-sm">
<div class="flex items-center justify-between p-4 border-b border-gray-100">
<h3 class="text-base font-medium text-gray-800">实时日志</h3>
<div class="flex items-center space-x-4">
<input type="text" id="logSearchInput" placeholder="搜索日志..."
class="px-3 py-1.5 border border-gray-300 rounded text-sm focus:outline-none focus:border-primary w-48"
oninput="searchLog()">
</div>
</div>
<div class="p-4">
<div id="logMatchCount" class="text-xs text-gray-500 mb-2"></div>
<div id="logContent" class="log-content bg-gray-50 rounded p-4 max-h-[400px] overflow-y-auto text-xs">
加载日志中...
</div>
</div>
</div>
<script>
let taskId = null;
let taskInfo = null;
let trainingLogFullContent = '';
// 训练曲线数据
const lossData = { labels: [], values: [] };
const gradNormData = { labels: [], values: [] };
const learningRateData = { labels: [], values: [] };
// 图表实例
let lossChart, gradNormChart, learningRateChart;
// 初始化图表
function initCharts() {
if (typeof Chart === 'undefined') {
console.error('[Charts] Chart 未定义,无法初始化图表');
document.getElementById('chartsContainer').innerHTML = '<div class="text-center p-4 text-red-500"><i class="fa fa-exclamation-triangle mr-2"></i>图表库加载失败,请刷新页面重试</div>';
return;
}
console.log('[Charts] 开始初始化图表...');
const commonOptions = {
responsive: true,
maintainAspectRatio: false,
animation: false,
scales: {
x: {
title: { display: true, text: 'Step' },
grid: { color: 'rgba(0,0,0,0.05)' }
},
y: {
title: { display: true, text: 'Value' },
grid: { color: 'rgba(0,0,0,0.05)' }
}
},
plugins: {
legend: { display: false }
}
};
// Loss 图表
const lossCtx = document.getElementById('lossChart').getContext('2d');
lossChart = new Chart(lossCtx, {
type: 'line',
data: {
labels: lossData.labels,
datasets: [{
label: 'Loss',
data: lossData.values,
borderColor: '#ef4444',
backgroundColor: 'rgba(239, 68, 68, 0.1)',
fill: true,
tension: 0.3,
pointRadius: 3
}]
},
options: {
...commonOptions,
plugins: { ...commonOptions.plugins, title: { display: true, text: 'Loss', color: '#ef4444', font: { size: 14 } } }
}
});
// Grad Norm 图表
const gradNormCtx = document.getElementById('gradNormChart').getContext('2d');
gradNormChart = new Chart(gradNormCtx, {
type: 'line',
data: {
labels: gradNormData.labels,
datasets: [{
label: 'Grad Norm',
data: gradNormData.values,
borderColor: '#3b82f6',
backgroundColor: 'rgba(59, 130, 246, 0.1)',
fill: true,
tension: 0.3,
pointRadius: 3
}]
},
options: {
...commonOptions,
plugins: { ...commonOptions.plugins, title: { display: true, text: 'Grad Norm', color: '#3b82f6', font: { size: 14 } } }
}
});
// Learning Rate 图表
const lrCtx = document.getElementById('learningRateChart').getContext('2d');
learningRateChart = new Chart(lrCtx, {
type: 'line',
data: {
labels: learningRateData.labels,
datasets: [{
label: 'Learning Rate',
data: learningRateData.values,
borderColor: '#22c55e',
backgroundColor: 'rgba(34, 197, 94, 0.1)',
fill: true,
tension: 0.3,
pointRadius: 3
}]
},
options: {
...commonOptions,
scales: {
...commonOptions.scales,
y: {
...commonOptions.scales.y,
type: 'logarithmic',
title: { display: true, text: 'Learning Rate (log)' }
}
},
plugins: { ...commonOptions.plugins, title: { display: true, text: 'Learning Rate', color: '#22c55e', font: { size: 14 } } }
}
});
}
// 解析日志中的指标数据
function parseMetricsFromLog(logContent) {
// 匹配 {'loss': x.xxxx, 'grad_norm': x.xxxx, 'learning_rate': x.xxxx, 'epoch': x.xx}
const metricRegex = /\{'loss':\s*([\d.]+),\s*'grad_norm':\s*([\d.]+),\s*'learning_rate':\s*([\d.e+-]+),\s*'epoch':\s*([\d.]+)\}/g;
let match;
let step = 0;
while ((match = metricRegex.exec(logContent)) !== null) {
const loss = parseFloat(match[1]);
const gradNorm = parseFloat(match[2]);
const learningRate = parseFloat(match[3]);
const epoch = parseFloat(match[4]);
// 更新数据
if (!lossData.values.includes(loss)) {
step++;
lossData.labels.push(step);
lossData.values.push(loss);
gradNormData.labels.push(step);
gradNormData.values.push(gradNorm);
learningRateData.labels.push(step);
learningRateData.values.push(learningRate);
}
}
// 更新图表
if (lossChart) {
lossChart.data.labels = lossData.labels;
lossChart.data.datasets[0].data = lossData.values;
lossChart.update('none');
}
if (gradNormChart) {
gradNormChart.data.labels = gradNormData.labels;
gradNormChart.data.datasets[0].data = gradNormData.values;
gradNormChart.update('none');
}
if (learningRateChart) {
learningRateChart.data.labels = learningRateData.labels;
learningRateChart.data.datasets[0].data = learningRateData.values;
learningRateChart.update('none');
}
}
// 带超时的 fetch
async function fetchWithTimeout(url, options = {}, timeout = 10000) {
const controller = new AbortController();
const id = setTimeout(() => controller.abort(), timeout);
try {
const response = await fetch(url, {
...options,
signal: controller.signal
});
clearTimeout(id);
return response;
} catch (error) {
clearTimeout(id);
throw new Error(`请求超时或失败: ${error.message}`);
}
}
// 获取URL参数
function getQueryParam(name) {
const urlParams = new URLSearchParams(window.location.search);
return urlParams.get(name);
}
// 获取任务ID优先从URL参数其次从sessionStorage
function getTaskId() {
let id = getQueryParam('id');
if (!id) {
try {
id = sessionStorage.getItem('trainingLogTaskId');
} catch (e) {}
}
return id;
}
// 返回模型调优列表
function goBack() {
if (window.parent && window.parent.navigateToPage) {
window.parent.navigateToPage('fine-tune');
} else {
window.location.href = 'main.html?page=fine-tune';
}
}
// 初始化
async function init() {
console.log('[Init] 开始初始化...');
taskId = getTaskId();
console.log('[Init] taskId:', taskId);
if (!taskId) {
document.getElementById('taskName').textContent = '未指定任务ID';
document.getElementById('logContent').innerHTML = '<span class="text-gray-400">请先从模型调优列表点击查看日志</span>';
return;
}
console.log('[Init] 加载任务信息...');
await loadTaskInfo();
console.log('[Init] 加载日志内容...');
await loadLogContent();
// 自动刷新每5秒
setInterval(async () => {
await loadTaskInfo();
await loadLogContent();
}, 5000);
}
// 加载任务信息
async function loadTaskInfo() {
try {
console.log('[Task] Fetching task info from:', `${API_BASE}/fine-tune/${taskId}`);
const response = await fetchWithTimeout(`${API_BASE}/fine-tune/${taskId}`);
console.log('[Task] Response status:', response.status);
const result = await response.json();
console.log('[Task] API result:', result);
if (result.code === 0 && result.data) {
taskInfo = result.data;
console.log('[Task] taskInfo:', taskInfo);
console.log('[Task] process_id:', taskInfo.process_id);
await updateTaskInfo();
} else {
console.error('[Task] API返回错误:', result.message);
}
} catch (error) {
console.error('[Task] 获取任务信息失败:', error);
document.getElementById('taskStatus').textContent = '获取失败';
document.getElementById('taskStatus').className = 'px-3 py-1 rounded-full text-sm bg-red-100 text-red-700';
}
}
// 更新任务信息显示
async function updateTaskInfo() {
if (!taskInfo) return;
document.getElementById('taskName').textContent = taskInfo.name || '未知任务';
// 更新状态
const statusElement = document.getElementById('taskStatus');
const actualStatus = taskInfo.status ? taskInfo.status.toLowerCase() : 'unknown';
const statusMap = {
'pending': { text: '等待中', class: 'bg-gray-100 text-gray-600' },
'running': { text: '训练中', class: 'bg-blue-100 text-blue-600' },
'completed': { text: '已完成', class: 'bg-green-100 text-green-600' },
'failed': { text: '失败', class: 'bg-red-100 text-red-700' },
'stopped': { text: '已停止', class: 'bg-orange-100 text-orange-600' }
};
const statusConfig = statusMap[actualStatus] || { text: actualStatus, class: 'bg-gray-100 text-gray-600' };
statusElement.textContent = statusConfig.text;
statusElement.className = `px-3 py-1 rounded-full text-sm ${statusConfig.class}`;
// 更新进度
const progressElement = document.getElementById('taskProgress');
if (progressElement && taskInfo.progress !== undefined) {
progressElement.textContent = `${taskInfo.progress}%`;
}
// 获取并显示GPU信息如果有
try {
const gpuResponse = await fetchWithTimeout(`${API_BASE}/fine-tune/progress/${taskId}`);
const gpuResult = await gpuResponse.json();
if (gpuResult.code === 0 && gpuResult.data) {
const gpuElement = document.getElementById('taskGPU');
if (gpuElement && gpuResult.data.gpu_info) {
gpuElement.textContent = gpuResult.data.gpu_info;
}
}
} catch (e) {
console.log('[Task] 获取GPU信息失败:', e);
}
// 更新数据集信息
const datasetElement = document.getElementById('dataset');
if (datasetElement && taskInfo.train_dataset_id) {
try {
const datasetResponse = await fetchWithTimeout(`${API_BASE}/dataset-manage/${taskInfo.train_dataset_id}`);
const datasetResult = await datasetResponse.json();
if (datasetResult.code === 0 && datasetResult.data) {
datasetElement.textContent = datasetResult.data.name;
} else {
datasetElement.textContent = `数据集${taskInfo.train_dataset_id}`;
}
} catch (e) {
datasetElement.textContent = `数据集${taskInfo.train_dataset_id}`;
}
} else if (datasetElement) {
datasetElement.textContent = '-';
}
// 更新最后更新时间
const lastUpdateElement = document.getElementById('lastUpdate');
if (lastUpdateElement && taskInfo.update_time) {
try {
const updateTime = new Date(taskInfo.update_time);
lastUpdateElement.textContent = updateTime.toLocaleString('zh-CN');
} catch (e) {
lastUpdateElement.textContent = taskInfo.update_time || '-';
}
}
// 其他信息
document.getElementById('processId').textContent = taskInfo.process_id || '-';
document.getElementById('createTime').textContent = taskInfo.create_time ?
new Date(taskInfo.create_time).toLocaleString('zh-CN') : '-';
// 获取模型名称
if (taskInfo.base_model) {
loadModelName(taskInfo.base_model);
}
}
// 加载模型名称
async function loadModelName(modelId) {
try {
const response = await fetchWithTimeout(`${API_BASE}/model-manage`);
const result = await response.json();
if (result.code === 0 && result.data) {
const model = result.data.find(m => m.id == modelId);
document.getElementById('baseModel').textContent = model ? model.name : `模型${modelId}`;
}
} catch (e) {
document.getElementById('baseModel').textContent = `模型${modelId}`;
}
}
// 加载日志内容
async function loadLogContent() {
console.log('[Log] loadLogContent called');
console.log('[Log] taskInfo:', taskInfo);
console.log('[Log] taskInfo.process_id:', taskInfo ? taskInfo.process_id : 'taskInfo is null');
// 检查 taskInfo 是否存在
if (!taskInfo) {
console.log('[Log] taskInfo 为空,等待任务信息加载...');
// 尝试重新加载任务信息
await loadTaskInfo();
if (!taskInfo) {
document.getElementById('logContent').innerHTML = '<span class="text-gray-400">无法获取任务信息</span>';
return;
}
}
// 检查 process_id 和 task_name
const processId = taskInfo.process_id;
const taskName = taskInfo.name || '';
if (!processId && !taskName) {
const msg = '<span class="text-gray-400">暂无日志文件 (任务未开始或无进程ID)</span>';
document.getElementById('logContent').innerHTML = msg;
return;
}
try {
console.log('[Log] Fetching training log files...');
const response = await fetchWithTimeout(`${API_BASE}/training-log-files`);
const result = await response.json();
if (result.code === 0 && result.data) {
console.log('[Log] Training log files:', result.data);
// 优先使用进程ID匹配文件名
let selectedFile = null;
if (processId) {
const pidStr = processId.toString();
for (const file of result.data) {
console.log(`[Log] Checking file: ${file.file}, PID: ${file.pid}, Match: ${file.file.startsWith(pidStr + '_') || file.file.includes(pidStr)}`);
if (file.file.startsWith(pidStr + '_') || file.file.includes(`_${pidStr}_`) || file.file.endsWith(`_${pidStr}.log`)) {
selectedFile = file.file;
break;
}
}
}
// 如果没找到,尝试使用任务名称匹配
if (!selectedFile && taskName) {
for (const file of result.data) {
if (file.file.includes(taskName)) {
selectedFile = file.file;
break;
}
}
}
// 如果仍然没有找到,使用第一个文件
if (!selectedFile && result.data.length > 0) {
selectedFile = result.data[0].file;
console.log('[Log] No matching file found, using first available file:', selectedFile);
}
if (selectedFile) {
console.log('[Log] Selected log file:', selectedFile);
await loadLogFileContent(selectedFile);
} else {
document.getElementById('logContent').innerHTML = '<span class="text-gray-400">未找到匹配的日志文件</span>';
}
} else {
document.getElementById('logContent').innerHTML = '<span class="text-gray-400">获取日志列表失败: ' + (result.message || '未知错误') + '</span>';
}
} catch (error) {
console.error('[Log] 获取日志列表失败:', error);
document.getElementById('logContent').innerHTML = '<span class="text-red-500">加载日志失败: ' + error.message + '</span>';
}
}
// 加载日志文件内容
async function loadLogFileContent(fileName) {
console.log('[Log] Loading log file:', fileName);
try {
const response = await fetchWithTimeout(`${API_BASE}/training-log-content?file=${encodeURIComponent(fileName)}`);
const result = await response.json();
console.log('[Log] Log content API response:', result);
if (result.code === 0 && result.data) {
trainingLogFullContent = result.data.content || '';
console.log('[Log] Log content length:', trainingLogFullContent.length);
renderLogContent();
// 解析并更新图表
parseMetricsFromLog(trainingLogFullContent);
} else if (result.code === 2) {
// 文件被锁定,正在训练中
document.getElementById('logContent').innerHTML = `
<div class="text-orange-500 p-4 text-center">
<i class="fa fa-spinner fa-spin fa-2x mb-2"></i>
<p class="text-lg">日志文件正在被训练进程占用</p>
<p class="text-sm text-gray-500 mt-1">${result.message || '训练结束后可查看完整内容'}</p>
<p class="text-xs text-gray-400 mt-2">页面将自动刷新...</p>
</div>
`;
} else {
document.getElementById('logContent').innerHTML = '<span class="text-red-500">加载日志失败: ' + (result.message || '未知错误') + '</span>';
}
} catch (error) {
console.error('[Log] 获取日志内容失败:', error);
document.getElementById('logContent').innerHTML = '<span class="text-red-500">加载日志失败: ' + error.message + '</span>';
}
}
// 渲染日志内容
function renderLogContent() {
const logContent = document.getElementById('logContent');
const searchInput = document.getElementById('logSearchInput');
const searchText = searchInput ? searchInput.value.toLowerCase() : '';
if (!trainingLogFullContent) {
logContent.innerHTML = '<span class="text-gray-400">暂无日志内容</span>';
return;
}
const lines = trainingLogFullContent.split('\n');
let html = '';
let matchCount = 0;
// 只显示最后500行以提高性能
const displayLines = lines.slice(-500);
for (const line of displayLines) {
if (!line.trim()) continue;
// 搜索过滤
if (searchText && !line.toLowerCase().includes(searchText)) {
continue;
}
// 级别过滤(不再使用)
let cssClass = '';
if (line.includes('[ERROR') || line.includes('error:') || line.includes('Error:')) {
cssClass = 'error';
} else if (line.includes('[WARNING') || line.includes('warning:') || line.includes('Warning:')) {
cssClass = 'warning';
} else if (line.includes('[INFO') || line.includes('info:') || line.includes('Info:')) {
cssClass = 'info';
}
// 进度条格式高亮
if (/\d+%/.test(line)) {
cssClass = cssClass ? cssClass + ' progress' : 'progress';
}
html += `<div class="log-line ${cssClass}">${escapeHtml(line)}</div>`;
matchCount++;
}
if (matchCount === 0) {
html = '<div class="text-gray-400 p-4">没有匹配的日志</div>';
}
logContent.innerHTML = html;
logContent.scrollTop = logContent.scrollHeight;
// 更新匹配数量
document.getElementById('logMatchCount').textContent =
searchText ? `找到 ${matchCount}` : '';
}
// 搜索日志
function searchLog() {
console.log('[Search] 搜索触发trainingLogFullContent:', trainingLogFullContent ? '已加载' : '未加载');
const searchInput = document.getElementById('logSearchInput');
console.log('[Search] 搜索文本:', searchInput ? searchInput.value : '输入框未找到');
renderLogContent();
}
// HTML转义
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
// 页面加载完成后初始化
function startApp() {
console.log('[App] startApp called');
console.log('[App] Chart available:', typeof Chart !== 'undefined');
// 等待 Chart.js 加载完成最多等待5秒
let waitCount = 0;
const maxWait = 50; // 50 * 100ms = 5秒
function waitForChart() {
if (typeof Chart !== 'undefined') {
console.log('[App] Chart.js 已加载,开始初始化');
initCharts();
init();
} else if (waitCount < maxWait) {
waitCount++;
console.log('[App] 等待 Chart.js 加载... (' + waitCount + ')');
setTimeout(waitForChart, 100);
} else {
console.error('[App] Chart.js 加载超时');
document.getElementById('chartsContainer').innerHTML = '<div class="text-center p-4 text-red-500"><i class="fa fa-exclamation-triangle mr-2"></i>图表库加载失败,请检查网络或刷新页面</div>';
// 仍然初始化其他功能
init();
}
}
// 如果已加载,直接初始化;否则等待
if (typeof Chart !== 'undefined') {
initCharts();
init();
} else {
console.log('[App] Chart.js 尚未加载,开始等待...');
setTimeout(waitForChart, 100);
}
}
// TensorBoard 控制
const TB_URL = 'http://10.10.10.177:6006';
function toggleTB() {
const btn = document.getElementById('tbBtn');
btn.innerHTML = '<i class="fa fa-spinner fa-spin mr-1"></i>启动中...';
btn.className = 'bg-gray-500 text-white px-4 py-2 rounded transition-colors text-sm cursor-wait';
// 调用API启动TensorBoard服务
fetch(`${API_BASE}/fine-tune/tensorboard/start`, { method: 'POST' })
.then(res => res.json())
.then(result => {
console.log('TensorBoard启动结果:', result);
if (result.code === 0) {
// 跳转到TensorBoard页面
window.open(TB_URL, '_blank');
btn.innerHTML = '<i class="fa fa-bar-chart mr-1"></i>打开TensorBoard';
btn.className = 'bg-purple-500 text-white px-4 py-2 rounded hover:bg-purple-600 transition-colors text-sm';
} else {
alert('提示: ' + (result.message || '启动失败'));
btn.innerHTML = '<i class="fa fa-bar-chart mr-1"></i>TensorBoard';
btn.className = 'bg-purple-500 text-white px-4 py-2 rounded hover:bg-purple-600 transition-colors text-sm';
}
})
.catch(err => {
console.error('启动TensorBoard失败:', err);
alert('提示: 启动失败 - ' + err.message);
btn.innerHTML = '<i class="fa fa-bar-chart mr-1"></i>TensorBoard';
btn.className = 'bg-purple-500 text-white px-4 py-2 rounded hover:bg-purple-600 transition-colors text-sm';
});
}
// 立即尝试初始化(处理 iframe 情况)
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', startApp);
} else {
startApp();
}
</script>
</body>
</html>