goclaw/internal/consolidation/scoring.go

package consolidation

import (
	"math"
	"time"

	"github.com/nextlevelbuilder/goclaw/internal/store"
)

// Scoring constants. Chosen to match the plan's 4-component formula (plan has
// 6 components but GoClaw lacks per-episode conceptual + consolidation data).
//
//	score = 0.30*frequency + 0.35*relevance + 0.20*recency + 0.15*freshness
//
// halfLifeDays=14 matches typical short-term memory retention windows; tuning
// should be informed by operator feedback once real recall data accumulates.
const (
	recallWeightFrequency = 0.30
	recallWeightRelevance = 0.35
	recallWeightRecency   = 0.20
	recallWeightFreshness = 0.15
	recallHalfLifeDays    = 14.0
	// frequencyCap: recall_count above this yields max frequency component
	// (avoids runaway-frequency bias for hot entries).
	recallFrequencyCap = 10
)

// recallThresholds holds the minimum scoring cutoffs applied to
// ListUnpromotedScored results before LLM synthesis. Chosen well below the
// TS reference (0.75 / 3) because GoClaw boots with zero recall data —
// early-life agents must still reach synthesis via the freshness component.
type recallThresholds struct {
	MinScore       float64
	MinRecallCount int
}

func defaultRecallThresholds() recallThresholds {
	return recallThresholds{MinScore: 0.2, MinRecallCount: 2}
}

// ComputeRecallScore computes the weighted recall score for an episodic entry.
// Pure function — no IO, no randomness — so the dreaming worker can compute
// scores on demand instead of materialising them into the DB on every event.
//
// Returned value is in [0, 1+ε]. In practice results cluster in [0, 0.8];
// thresholds in defaultRecallThresholds are calibrated to that range.
func ComputeRecallScore(entry store.EpisodicSummary, now time.Time) float64 {
	// 1. Frequency (30%): log-normalised recall count. Log1p smooths the
	// curve so 0 → 0, 1 → 0.29, 10 → 1.0; anything above cap stays at 1.
	freq := math.Log1p(float64(entry.RecallCount)) / math.Log1p(recallFrequencyCap)
	if freq > 1.0 {
		freq = 1.0
	}

	// 2. Relevance (35%): running average of memory_search hit scores.
	// Already lives in recall_score column and is clamped server-side to [0,1]
	// by RecordRecall callers. Defensive clamp here for robustness.
	rel := entry.RecallScore
	if rel < 0 {
		rel = 0
	} else if rel > 1 {
		rel = 1
	}

	// Exponential decay shared by recency + freshness components.
	// lambda = ln(2) / halfLife → score halves every `recallHalfLifeDays` days.
	lambda := math.Ln2 / recallHalfLifeDays

	// 3. Recency (20%): decay from LastRecalledAt (falls back to CreatedAt
	// for entries that have never been searched). Entries recalled today
	// score ~1.0; 14 days ago → 0.5; 28 days ago → 0.25.
	lastRef := entry.CreatedAt
	if entry.LastRecalledAt != nil {
		lastRef = *entry.LastRecalledAt
	}
	recency := decayFrom(lastRef, now, lambda)

	// 4. Freshness (15%): decay from CreatedAt regardless of recall history.
	// Keeps brand-new entries eligible even with zero recall — prevents cold
	// start from starving the synthesis pipeline.
	freshness := decayFrom(entry.CreatedAt, now, lambda)

	return recallWeightFrequency*freq +
		recallWeightRelevance*rel +
		recallWeightRecency*recency +
		recallWeightFreshness*freshness
}

// decayFrom returns exp(-lambda * ageDays). Negative ages (clock skew,
// future-dated rows) clamp to 1.0 so the test fixtures are predictable.
func decayFrom(then, now time.Time, lambda float64) float64 {
	if then.IsZero() || !then.Before(now) {
		return 1.0
	}
	ageDays := now.Sub(then).Hours() / 24
	return math.Exp(-lambda * ageDays)
}

// filterByRecallThresholds drops entries whose recall signal is too weak
// to justify LLM synthesis. Pure function so the dreaming worker can
// sort-then-filter in a single pass.
//
// Bootstrap-friendly rules:
//   - Never-recalled entries (RecallCount == 0) bypass BOTH thresholds.
//     Freshness + recency keep them eligible so new agents can synthesise
//     their earliest sessions without any memory_search activity.
//   - Entries that HAVE been recalled must clear both MinRecallCount and
//     MinScore. If an entry has seen 1 weak hit, it's filtered.
func filterByRecallThresholds(entries []store.EpisodicSummary, th recallThresholds, now time.Time) []store.EpisodicSummary {
	if len(entries) == 0 {
		return entries
	}
	kept := entries[:0]
	for _, e := range entries {
		if e.RecallCount == 0 {
			kept = append(kept, e)
			continue
		}
		if e.RecallCount < th.MinRecallCount {
			continue
		}
		if ComputeRecallScore(e, now) < th.MinScore {
			continue
		}
		kept = append(kept, e)
	}
	return kept
}