countTokens (agent/TokenCounter.ts)

Module exposing: countTokens, countTokensBatch, TokenLedger, getEnc, fallbackCount

// Source: https://github.com/JaneHax/JanexAgent | License: MIT | Copyright 2024 Janex
// Provided via justclaw.it.com; keep this notice with the code.
import { getEncoding, type Tiktoken } from 'js-tiktoken';

type Encoding = 'cl100k_base' | 'o200k_base';

const encCache = new Map<string, Tiktoken>();

function getEnc(name: Encoding): Tiktoken {
  let enc = encCache.get(name);
  if (!enc) {
    enc = getEncoding(name);
    encCache.set(name, enc);
  }
  return enc;
}

function fallbackCount(text: string): number {
  return Math.ceil(text.length / 4);
}

export function countTokens(text: string, encoding?: Encoding): number {
  try {
    return getEnc(encoding || 'cl100k_base').encode(text).length;
  } catch {
    return fallbackCount(text);
  }
}

export function countTokensBatch(texts: s

... (truncated -- full source via MCP)