I used Gemini to create this token approximation script.
Save the below code as analyze-context-size.js
I placed it in my scripts/ folder.
node scripts/analyze-context-size.js
Note that it’s not 100% accurate in counting tokens, but it’s close.
import fs from "node:fs/promises";
import path from "node:path";
import ignore from "ignore";
// --- Helper Functions ---
function formatBytes(bytes, decimals = 2) {
if (bytes === 0) return "0 Bytes";
const k = 1024;
const dm = decimals < 0 ? 0 : decimals;
const sizes = ["Bytes", "KB", "MB", "GB"];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + " " + sizes[i];
}
function pad(str, len, char = ' ') {
return String(str).padEnd(len, char);
}
// --- Main Analysis Logic ---
async function analyzeDirectory(baseDir, reportTitle, topN = 20, excludePatterns = [], options = {}) {
const { silent = false } = options;
const startTime = Date.now();
if (!silent) {
console.log(`[Context Analyzer] Analyzing directory: ${reportTitle}`);
}
const defaultIgnores = [
".git", "node_modules", ".next", "out", "public/videos", "*.mp4",
"*.webm", "*.lock", "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp",
"package-lock.json", ".DS_Store", "pnpm-lock.yaml"
];
const ig = ignore().add(defaultIgnores).add(excludePatterns);
const fileDetails = [];
let totalChars = 0;
let totalFiles = 0;
let includedFiles = 0;
async function walk(currentDir) {
try {
const dirents = await fs.readdir(currentDir, { withFileTypes: true });
totalFiles += dirents.length;
for (const dirent of dirents) {
const fullPath = path.join(currentDir, dirent.name);
const relativePath = path.relative(baseDir, fullPath);
if (ig.ignores(relativePath)) {
continue;
}
if (dirent.isDirectory()) {
await walk(fullPath);
} else {
try {
const content = await fs.readFile(fullPath, "utf8");
const charCount = content.length;
totalChars += charCount;
includedFiles++;
fileDetails.push({ path: relativePath, size: content.length });
} catch (readErr) {
// Ignore errors for files that can't be read
}
}
}
} catch (walkErr) {
if (walkErr.code !== 'ENOENT') {
console.error(`[Context Analyzer] Error walking directory ${currentDir}:`, walkErr);
}
}
}
await walk(baseDir);
fileDetails.sort((a, b) => b.size - a.size);
const topFiles = fileDetails.slice(0, topN);
const endTime = Date.now();
if (!silent) {
console.log(`[Context Analyzer] Analysis of "${reportTitle}" complete in ${endTime - startTime}ms.`);
}
return {
reportTitle,
totalFiles: includedFiles,
totalChars,
estimatedTokens: Math.round(totalChars / 4),
topFiles
};
}
// --- Reporting Function ---
function printReport(report, maxTokens = 1048000) {
console.log(`\n--- ${report.reportTitle} Report ---`);
console.log(`Total Files Included: ${report.totalFiles}`);
console.log(`Total Characters: ${report.totalChars.toLocaleString()}`);
console.log(`Estimated Tokens: ${report.estimatedTokens.toLocaleString()} (approx. 4 chars/token)`);
const usagePercent = (report.estimatedTokens / maxTokens) * 100;
let emoji = "🟢";
if (usagePercent > 90) emoji = "🔴";
else if (usagePercent > 75) emoji = "🟡";
console.log(`${emoji} Context Size vs Max (${maxTokens.toLocaleString()} tokens): ${usagePercent.toFixed(2)}%`);
if (report.topFiles.length > 0) {
console.log(`\n--- Top ${report.topFiles.length} Largest Files by Character Count ---`);
const maxPathLen = Math.max(...report.topFiles.map(f => f.path.length));
const maxSizeLen = Math.max(...report.topFiles.map(f => formatBytes(f.size).length));
report.topFiles.forEach((file, index) => {
const rank = pad(`${index + 1}.`, 4);
const filePath = pad(file.path, maxPathLen);
const fileSize = pad(formatBytes(file.size), maxSizeLen);
const tokens = `~${Math.round(file.size / 4).toLocaleString()} tokens`;
console.log(`${rank}${filePath} | ${fileSize} | ${tokens}`);
});
}
}
// --- Main Execution ---
async function main() {
const startTime = Date.now();
console.log("[Context Analyzer] Starting analysis...");
const aiexcludePath = path.join(process.cwd(), '.aiexclude');
let customExcludes = [];
try {
const excludeFileContent = await fs.readFile(aiexcludePath, 'utf8');
customExcludes = excludeFileContent.split('\n').filter(line => line && !line.startsWith('#'));
} catch (err) {
if (err.code === 'ENOENT') {
console.log("[Context Analyzer] No .aiexclude file found, using default ignores.");
} else {
console.error(`[Context Analyzer] Error reading .aiexclude file:`, err);
}
}
// --- Analyze Project Root ---
const projectReport = await analyzeDirectory(process.cwd(), "Project Root", 20, customExcludes, { silent: false });
printReport(projectReport);
// --- Analyze .idx/ai Directory ---
const aiDir = '/home/user/.idx/ai/';
const aiReport = await analyzeDirectory(aiDir, ".idx/ai Directory", 3, [], { silent: true });
if (aiReport.totalFiles > 0) {
printReport(aiReport);
} else {
console.log(`\n[Context Analyzer] Directory /home/user/.idx/ai/ not found or is empty. Skipping its report.`);
}
// --- Combined Report ---
if (aiReport.totalFiles > 0) {
const combinedTotalFiles = projectReport.totalFiles + aiReport.totalFiles;
const combinedTotalChars = projectReport.totalChars + aiReport.totalChars;
const combinedEstimatedTokens = projectReport.estimatedTokens + aiReport.estimatedTokens;
printReport({
reportTitle: "Combined Context",
totalFiles: combinedTotalFiles,
totalChars: combinedTotalChars,
estimatedTokens: combinedEstimatedTokens,
topFiles: [] // No need to show top files for combined report
});
}
const totalEndTime = Date.now();
console.log(`\n[Context Analyzer] Full analysis finished in ${totalEndTime - startTime}ms.`);
}
main().catch(console.error);
