telegram-exporter/internal/verify/verify.go

// Package verify answers whether a chat is fully archived on a remote.
//
// It replaces verify-export.sh and keeps that script's size judgements, which
// were arrived at by watching real failures rather than by taste.
//
// One judgement is deliberately not carried over. The script also counted files
// sitting in the staging directory as present, because download and upload were
// separate processes and a file could be finished locally but not yet uploaded
// for a whole sync interval. Here a single process owns both legs, so that state
// is not one a verify can meaningfully observe — except after an interrupted
// run, where staging may hold completed files. Phase 5 owns staging and decides
// whether to credit it; until then a verify after an interrupt may report files
// absent that are on local disk, and re-fetch them.
package verify

import (
	"fmt"
	"io"
	"sort"

	"github.com/tiennm99dev/telegram-exporter/internal/naming"
	"github.com/tiennm99dev/telegram-exporter/internal/tgsource"
)

// Index is the part of a remote snapshot verification needs.
//
// An interface rather than *remote.Index so the two questions stay separable:
// presence is answered from a whole name, and the id-keyed lookup is explicitly
// a different method with a name that says it is not an answer. It also lets the
// report be tested without a remote.
type Index interface {
	// Lookup reports the size stored under an exact name.
	Lookup(name string) (size int64, ok bool)
	// StoredUnderOtherNames lists names present for a message id that are not
	// the wanted name. Diagnostics only; never a presence answer.
	StoredUnderOtherNames(messageID int, wanted string) []string
}

// tinyThreshold is the size below which a present file is reported for a human
// to look at but still trusted. Some real media genuinely is this small, so
// treating it as damaged would re-download it forever.
const tinyThreshold = 1024

// Misnamed is a wanted file that is absent, while some other file is stored
// under the same message id.
type Misnamed struct {
	MessageID int
	Wanted    string
	Found     []string
}

// Unsafe is a wanted file whose name cannot be written to a path.
type Unsafe struct {
	MessageID int
	Name      string
	Reason    error
}

// Tiny is a present file small enough to be worth a look.
type Tiny struct {
	MessageID int
	Name      string
	Size      int64
}

// Mismatch is a present file whose stored size is not the size Telegram reports.
type Mismatch struct {
	MessageID int
	Name      string
	Want, Got int64
}

// Report is the outcome of comparing a chat against a remote.
type Report struct {
	Expected int   // media messages in the chat
	Present  int   // present, non-empty
	Bytes    int64 // total size of everything expected

	Absent   []int // not on the remote under the wanted name
	ZeroByte []int // present but empty

	Misnamed   []Misnamed
	Unsafe     []Unsafe
	Tiny       []Tiny
	Mismatched []Mismatch

	// checked records that Check actually ran. Without it a zero Report claims
	// the archive is complete — nothing expected, nothing missing — which is the
	// value a command holds before its Telegram callback has populated it. Any
	// path that returns early therefore reports success on an untouched chat.
	checked bool
}

// Ran reports whether this came from a Check rather than being a zero value.
func (r Report) Ran() bool { return r.checked }

// Todo lists the message ids needing another fetch, in ascending order.
//
// Zero-byte and wrong-size files are included: rclone overwrites a
// size-mismatched destination, so simply fetching again repairs them.
func (r Report) Todo() []int {
	todo := make([]int, 0, len(r.Absent)+len(r.ZeroByte)+len(r.Mismatched))
	todo = append(todo, r.Absent...)
	todo = append(todo, r.ZeroByte...)
	for _, m := range r.Mismatched {
		todo = append(todo, m.MessageID)
	}
	sort.Ints(todo)
	return todo
}

// Fetchable lists the outstanding ids a run could actually retrieve.
//
// It is Todo minus the unsafe names, which are in Todo because they are not
// archived and out of this because no run will ever archive them. The gap
// between the two is what tells "keep going" apart from "this is as far as it
// goes".
func (r Report) Fetchable() []int {
	if len(r.Unsafe) == 0 {
		return r.Todo()
	}
	blocked := make(map[int]struct{}, len(r.Unsafe))
	for _, u := range r.Unsafe {
		blocked[u.MessageID] = struct{}{}
	}
	todo := r.Todo()
	out := todo[:0:0]
	for _, id := range todo {
		if _, skip := blocked[id]; !skip {
			out = append(out, id)
		}
	}
	return out
}

// Complete reports whether every expected file is present and non-empty.
func (r Report) Complete() bool { return r.checked && len(r.Todo()) == 0 }

// Stalled reports that work remains and none of it can ever be done.
//
// This is the state a drive-until-complete loop cannot detect for itself: the
// report is identical on every pass, so a driver retrying on "incomplete" walks
// the whole history and indexes the whole remote forever, achieving nothing.
func (r Report) Stalled() bool {
	return r.checked && len(r.Todo()) > 0 && len(r.Fetchable()) == 0
}

// Check compares the wanted items against an index of the remote.
//
// Matching is on the whole name. A file stored under any other name is not the
// file that was asked for, however close it looks — that is a deliberate policy,
// and the near-misses are collected into Misnamed rather than being quietly
// accepted, because the re-download lands beside them and both copies stay.
func Check(items []tgsource.Item, idx Index) Report {
	r := Report{Expected: len(items), checked: true}

	for _, it := range items {
		r.Bytes += it.Size()

		if err := naming.Safe(it.Name); err != nil {
			// Never counted present: this name cannot be written anywhere safe,
			// so no correctly-behaving run could have archived it.
			r.Unsafe = append(r.Unsafe, Unsafe{MessageID: it.MessageID, Name: it.Name, Reason: err})
			r.Absent = append(r.Absent, it.MessageID)
			continue
		}

		size, ok := idx.Lookup(it.Name)
		switch {
		case !ok:
			r.Absent = append(r.Absent, it.MessageID)
			if others := idx.StoredUnderOtherNames(it.MessageID, it.Name); len(others) > 0 {
				r.Misnamed = append(r.Misnamed, Misnamed{
					MessageID: it.MessageID, Wanted: it.Name, Found: others,
				})
			}
		case size == 0:
			r.ZeroByte = append(r.ZeroByte, it.MessageID)
		case size != it.Size():
			// The remaining way a report could say "complete" when it is not.
			// An upload that died partway leaves a plausible object under the
			// right name, and matching on name and non-zero size alone would
			// count it archived permanently. Telegram's size is known here, so
			// there is no reason not to use it; rclone overwrites a mismatched
			// destination, so fetching again repairs it.
			r.Mismatched = append(r.Mismatched, Mismatch{
				MessageID: it.MessageID, Name: it.Name, Want: it.Size(), Got: size,
			})
		default:
			r.Present++
			if size < tinyThreshold {
				r.Tiny = append(r.Tiny, Tiny{MessageID: it.MessageID, Name: it.Name, Size: size})
			}
		}
	}
	return r
}

// Write renders a report in the shape verify-export.sh printed, so the numbers
// stay comparable across the cutover.
func (r Report) Write(w io.Writer) {
	fmt.Fprintf(w, "media expected     : %d  (%.1f GiB)\n", r.Expected, float64(r.Bytes)/(1<<30))
	fmt.Fprintf(w, "present and intact : %d\n", r.Present)
	fmt.Fprintf(w, "  absent           : %d\n", len(r.Absent))
	fmt.Fprintf(w, "  zero-byte        : %d\n", len(r.ZeroByte))
	if len(r.Mismatched) > 0 {
		fmt.Fprintf(w, "  wrong size       : %d\n", len(r.Mismatched))
		for _, m := range r.Mismatched[:min(5, len(r.Mismatched))] {
			fmt.Fprintf(w, "      id %d  %d B on the remote, expected %d  %q\n",
				m.MessageID, m.Got, m.Want, m.Name)
		}
	}

	if len(r.Tiny) > 0 {
		fmt.Fprintf(w, "  under 1KiB (check, not retried): %d\n", len(r.Tiny))
		for _, t := range r.Tiny[:min(5, len(r.Tiny))] {
			fmt.Fprintf(w, "      id %d  %d B  %q\n", t.MessageID, t.Size, t.Name)
		}
	}

	if len(r.Unsafe) > 0 {
		fmt.Fprintf(w, "\nunarchivable       : %d\n", len(r.Unsafe))
		fmt.Fprintf(w, "  these names cannot be written and will never be fetched:\n")
		for _, u := range r.Unsafe {
			fmt.Fprintf(w, "      id %d  %v\n", u.MessageID, u.Reason)
		}
	}

	if len(r.Misnamed) > 0 {
		fmt.Fprintf(w, "\nstored under a different name : %d\n", len(r.Misnamed))
		fmt.Fprintf(w, "  counted as absent and fetched again; delete the stale copies so the\n")
		fmt.Fprintf(w, "  re-download does not leave two files for the same message:\n")
		for _, m := range r.Misnamed {
			fmt.Fprintf(w, "      id %d\n        wanted: %q\n", m.MessageID, m.Wanted)
			for _, f := range m.Found {
				fmt.Fprintf(w, "        remote: %q\n", f)
			}
		}
	}

	todo := r.Todo()
	switch {
	case r.Stalled():
		fmt.Fprintf(w, "\nSTALLED: %d file(s) remain, none of which can be fetched.\n", len(todo))
	case len(todo) > 0:
		fmt.Fprintf(w, "\nneeds another pass : %d  (ids %d–%d)\n", len(todo), todo[0], todo[len(todo)-1])
	case !r.checked:
		fmt.Fprintf(w, "\nno chat was checked.\n")
	default:
		fmt.Fprintf(w, "\nCOMPLETE: every media message is present and non-empty.\n")
	}
}