goclaw/internal/tokencount/fallback_counter.go

package tokencount

import (
	"cmp"
	"encoding/json"
	"slices"
	"strings"
	"unicode/utf8"

	"github.com/nextlevelbuilder/goclaw/internal/providers"
)

// FallbackCounter is the BEST-EFFORT heuristic counter used when tiktoken-go is
// unavailable or the model is unknown (e.g. 9router brand models that match no
// registry prefix). It is NOT a proven token bound: real tokenizers vary by
// language, and dense scripts like Vietnamese tokenize at ~2 chars/token or
// worse. Because the pre-transport guard treats this count as its ceiling, the
// heuristic deliberately uses a CONSERVATIVE chars-per-token ratio so it errs
// toward over-counting (compact/block early) rather than under-counting (send an
// oversized request). The only way to a provable ceiling for an un-tokenizable
// model is to register it with a real tokenizer or use the provider's own token
// count.
type FallbackCounter struct{}

func NewFallbackCounter() *FallbackCounter { return &FallbackCounter{} }

// fallbackCharsPerToken is the conservative chars-per-token divisor for the
// guard-facing count methods. Lower than a naive ~3-4 chars/token so mixed
// Vietnamese/code content is over-counted rather than under-counted. This is a
// safety heuristic, not an exact tokenization.
const fallbackCharsPerToken = 2

func (c *FallbackCounter) Count(_ string, text string) int {
	return utf8.RuneCountInString(text) / fallbackCharsPerToken
}

func (c *FallbackCounter) CountMessages(_ string, msgs []providers.Message) int {
	total := 0
	for _, m := range msgs {
		total += utf8.RuneCountInString(m.Content)/fallbackCharsPerToken + PerMessageOverhead
		// Match tiktokenCounter.CountMessages coverage so the fallback path
		// applies the same best-effort ceiling (thinking, tool-result id, raw
		// blocks, tool args, media all count toward the wire payload).
		total += utf8.RuneCountInString(m.Thinking) / fallbackCharsPerToken
		total += utf8.RuneCountInString(m.ToolCallID) / fallbackCharsPerToken
		total += utf8.RuneCountInString(string(m.RawAssistantContent)) / fallbackCharsPerToken
		for _, tc := range m.ToolCalls {
			total += utf8.RuneCountInString(tc.ID)/fallbackCharsPerToken + utf8.RuneCountInString(tc.Name)/fallbackCharsPerToken
			if b, err := json.Marshal(tc.Arguments); err == nil {
				total += utf8.RuneCountInString(string(b)) / fallbackCharsPerToken
			}
		}
		total += fallbackMediaTokenCost(m)
	}
	return total
}

// fallbackMediaTokenCost mirrors mediaTokenCost for the heuristic counter.
func fallbackMediaTokenCost(m providers.Message) int {
	const perInlineMediaItem = 1600
	cost := 0
	for _, img := range m.Images {
		if img.Data != "" || img.URL != "" {
			cost += perInlineMediaItem
		}
	}
	for _, vid := range m.Videos {
		if vid.Data != "" || vid.URL != "" {
			cost += perInlineMediaItem
		}
	}
	return cost
}

// CountToolSchemas returns a conservative heuristic count for the JSON-serialised
// tool list. Returns 0 for nil or empty slice.
func (c *FallbackCounter) CountToolSchemas(_ string, tools []providers.ToolDefinition) int {
	if len(tools) == 0 {
		return 0
	}
	blob, _ := json.Marshal(tools)
	return utf8.RuneCountInString(string(blob)) / fallbackCharsPerToken
}

// ModelContextWindow uses longest-prefix-match to avoid ambiguity
// (e.g., "gpt-4o" must match before "gpt-4").
func (c *FallbackCounter) ModelContextWindow(model string) int {
	// Sort prefixes longest-first for correct matching.
	keys := make([]string, 0, len(DefaultRegistry))
	for k := range DefaultRegistry {
		keys = append(keys, k)
	}
	slices.SortFunc(keys, func(a, b string) int { return cmp.Compare(len(b), len(a)) })

	for _, prefix := range keys {
		if strings.HasPrefix(model, prefix) {
			return DefaultRegistry[prefix].ContextWindow
		}
	}
	return 200_000 // conservative default
}