chore: add TokenLens sources and ignore rules

This commit is contained in:
2026-06-12 15:45:58 +08:00
parent 0b48e618d8
commit 887b75b790
37 changed files with 13907 additions and 0 deletions

View File

@@ -0,0 +1,12 @@
import test from "node:test";
import assert from "node:assert/strict";
import { formatTokensZh } from "../src/renderer/displayFormat.js";
test("formatTokensZh uses Chinese magnitude units instead of western suffixes", () => {
assert.equal(formatTokensZh(9999), "9999");
assert.equal(formatTokensZh(10_000), "1万");
assert.equal(formatTokensZh(71_400), "7.1万");
assert.equal(formatTokensZh(6_600_000), "660万");
assert.equal(formatTokensZh(196_700_000), "1.97亿");
assert.equal(formatTokensZh(1_020_000_000), "10.2亿");
});

182
tests/tokenUsage.test.cjs Normal file
View File

@@ -0,0 +1,182 @@
const test = require("node:test");
const assert = require("node:assert/strict");
const fs = require("node:fs/promises");
const os = require("node:os");
const path = require("node:path");
const {
parseClaudeLine,
parseCodexLine,
parseGeminiLine,
summarizeRecords,
scanUsage
} = require("../src/lib/tokenUsage.cjs");
test("parseClaudeLine extracts usage without reading message content", () => {
const line = JSON.stringify({
timestamp: "2026-06-11T02:20:00.000Z",
cwd: "/Users/me/Code/App",
sessionId: "claude-session",
message: {
role: "assistant",
content: "private conversation text",
model: "claude-3-5-sonnet",
usage: {
input_tokens: 120,
output_tokens: 30,
cache_read_input_tokens: 50,
cache_creation_input_tokens: 10
}
}
});
const record = parseClaudeLine(line, "/tmp/claude.jsonl");
assert.equal(record.provider, "Claude");
assert.equal(record.model, "claude-3-5-sonnet");
assert.equal(record.inputTokens, 120);
assert.equal(record.outputTokens, 30);
assert.equal(record.cachedTokens, 60);
assert.equal(record.totalTokens, 210);
assert.equal(record.contentPreview, undefined);
});
test("parseCodexLine uses last_token_usage instead of cumulative total usage", () => {
const line = JSON.stringify({
timestamp: "2026-06-11T03:10:00.000Z",
type: "event_msg",
payload: {
info: {
total_token_usage: {
input_tokens: 1000,
output_tokens: 300,
cached_input_tokens: 200,
reasoning_output_tokens: 40,
total_tokens: 1300
},
last_token_usage: {
input_tokens: 100,
output_tokens: 30,
cached_input_tokens: 20,
reasoning_output_tokens: 4,
total_tokens: 130
}
}
}
});
const record = parseCodexLine(line, "/tmp/rollout.jsonl");
assert.equal(record.provider, "Codex");
assert.equal(record.inputTokens, 100);
assert.equal(record.outputTokens, 30);
assert.equal(record.cachedTokens, 20);
assert.equal(record.reasoningTokens, 4);
assert.equal(record.totalTokens, 154);
});
test("parseGeminiLine extracts tokens from Gemini chat jsonl", () => {
const line = JSON.stringify({
timestamp: "2026-06-11T04:00:00.000Z",
type: "response",
model: "gemini-2.5-pro",
tokens: {
input: 70,
output: 20,
cached: 5,
thoughts: 3,
tool: 2,
total: 100
}
});
const record = parseGeminiLine(line, "/tmp/session.jsonl");
assert.equal(record.provider, "Gemini");
assert.equal(record.model, "gemini-2.5-pro");
assert.equal(record.inputTokens, 70);
assert.equal(record.outputTokens, 20);
assert.equal(record.cachedTokens, 5);
assert.equal(record.reasoningTokens, 3);
assert.equal(record.toolTokens, 2);
assert.equal(record.totalTokens, 100);
});
test("summarizeRecords returns today, month, total, provider, trend and insight data", () => {
const now = new Date("2026-06-11T12:00:00+08:00");
const records = [
record("Claude", "2026-06-11T01:00:00Z", 100, 20, 10),
record("Codex", "2026-06-10T01:00:00Z", 80, 20, 0),
record("Gemini", "2026-05-31T01:00:00Z", 50, 10, 0)
];
const summary = summarizeRecords(records, { now, timeZone: "Asia/Shanghai" });
assert.equal(summary.cards.todayTokens, 130);
assert.equal(summary.cards.monthTokens, 230);
assert.equal(summary.cards.totalTokens, 290);
assert.equal(summary.providerTotals.Claude.totalTokens, 130);
assert.equal(summary.providerTotals.Codex.totalTokens, 100);
assert.equal(summary.providerTotals.Gemini.totalTokens, 60);
assert.ok(summary.dailyTrend.length >= 2);
assert.ok(summary.sourceRows.length >= 3);
assert.ok(summary.insights.length >= 1);
});
test("scanUsage reads only known local usage files and aggregates them", async () => {
const homeDir = await fs.mkdtemp(path.join(os.tmpdir(), "tokenlens-"));
await fs.mkdir(path.join(homeDir, ".claude/projects/proj"), { recursive: true });
await fs.mkdir(path.join(homeDir, ".codex/sessions/2026/06/11"), { recursive: true });
await fs.mkdir(path.join(homeDir, ".gemini/tmp/me/chats"), { recursive: true });
await fs.writeFile(
path.join(homeDir, ".claude/projects/proj/a.jsonl"),
JSON.stringify({
timestamp: "2026-06-11T02:20:00.000Z",
message: { model: "claude", usage: { input_tokens: 10, output_tokens: 5 } }
}) + "\n"
);
await fs.writeFile(
path.join(homeDir, ".codex/sessions/2026/06/11/rollout.jsonl"),
JSON.stringify({
timestamp: "2026-06-11T02:30:00.000Z",
payload: { info: { last_token_usage: { input_tokens: 7, output_tokens: 3 } } }
}) + "\n"
);
await fs.writeFile(
path.join(homeDir, ".gemini/tmp/me/chats/session.jsonl"),
JSON.stringify({
timestamp: "2026-06-11T02:40:00.000Z",
model: "gemini",
tokens: { input: 8, output: 2, total: 10 }
}) + "\n"
);
const summary = await scanUsage({
homeDir,
now: new Date("2026-06-11T12:00:00+08:00"),
timeZone: "Asia/Shanghai"
});
assert.equal(summary.records.length, 3);
assert.equal(summary.cards.totalTokens, 35);
assert.equal(summary.providerTotals.Claude.totalTokens, 15);
assert.equal(summary.providerTotals.Codex.totalTokens, 10);
assert.equal(summary.providerTotals.Gemini.totalTokens, 10);
});
function record(provider, timestamp, inputTokens, outputTokens, cachedTokens) {
return {
provider,
source: `${provider} source`,
sourceFile: `/tmp/${provider}.jsonl`,
timestamp,
model: provider,
inputTokens,
outputTokens,
cachedTokens,
reasoningTokens: 0,
toolTokens: 0,
totalTokens: inputTokens + outputTokens + cachedTokens
};
}