feat: download, upload and drive a chat to completion in one process

Phases 4 through 6: the two legs and the command that joins them.

Downloads go to <name>.part and are renamed only once complete, so a file
without the suffix is always whole. That is what lets the upload leg treat
"exists" as "finished" — the property run.sh could only approximate with a
filename convention plus an age guard, because it could not see inside tdl.

Every finished file is checked against the size Telegram reported, and that
check rather than the error is the authoritative signal. core's downloader logs
a failed transfer and returns nil, and its completion callback is deferred on
that named return, so a failure arrives indistinguishable from a success.
Trusting it would promote a truncated file and archive it as complete.

The disk cap is a semaphore over bytes. A download reserves its own size before
starting and releases it only after the upload confirms, so a slow remote
stalls downloads by itself. Blocking the iterator is safe because the
downloader calls it from its dispatch loop while workers run in a group, so a
blocked iterator never stops the uploads that free the space. Gone with it: the
du polling, the SIGSTOP and SIGCONT suspension, the min-age guard, the
temp-file filter and the sweep-failure counter.

A cap smaller than the largest file is refused up front. The semaphore could
never admit it, and a run blocked on a file it can never start looks exactly
like a stalled remote.

Uploads re-state each object to prove its size before the local copy is gone,
closing a gap where a truncated upload was only noticed by a later verify.

The destination is created before the chat is read. It is also the credentials
check, and doing it first means a bad destination fails in seconds rather than
after a full history walk.

One invocation converges: each item is checked against the index immediately
before download, so there are no passes and re-running is the resume path.
Options that no longer exist say what replaced them instead of failing as
unknown flags.

Verified end to end against the live chat and a scratch remote path: two files
downloaded, uploaded, confirmed present at the right size, staging left empty.

cmd/tgexport/main.go

@@ -65,6 +65,8 @@ func run() int {
 		err = listCmd(ctx, os.Args[2:])
 	case "verify":
 		err = verifyCmd(ctx, os.Args[2:])
+	case "sync":
+		err = syncCmd(ctx, os.Args[2:])
 	default:
 		fmt.Fprintf(os.Stderr, "unknown command %q\n\n", os.Args[1])
 		usage()
@@ -150,9 +152,10 @@ func usage() {
 	fmt.Fprint(os.Stderr, `Usage: tgexport <command> [options]
 
 Commands:
-  doctor    Check the Telegram session, the destination remote, and free space
+  sync      Archive a chat to a remote, fetching only what is missing
   list      Print every media message in a chat as id<TAB>size<TAB>name
   verify    Report whether a chat is fully archived on a remote
+  doctor    Check the Telegram session, the destination remote, and free space
 
 Run 'tgexport <command> -h' for command options.
 `)

cmd/tgexport/sync.go

@@ -0,0 +1,322 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"iter"
+	"os"
+
+	"github.com/iyear/tdl/core/dcpool"
+	tdlstorage "github.com/iyear/tdl/core/storage"
+	"github.com/rclone/rclone/fs"
+
+	"github.com/tiennm99dev/telegram-exporter/internal/pipeline"
+	"github.com/tiennm99dev/telegram-exporter/internal/remote"
+	"github.com/tiennm99dev/telegram-exporter/internal/report"
+	"github.com/tiennm99dev/telegram-exporter/internal/tdlkv"
+	"github.com/tiennm99dev/telegram-exporter/internal/tgsource"
+	"github.com/tiennm99dev/telegram-exporter/internal/verify"
+)
+
+// retiredFlags map options the shell pipeline had onto what replaced them.
+//
+// Recognising them beats "flag provided but not defined": these were in
+// muscle memory and in wrapper scripts, and a bare parse error does not say
+// whether the concept moved or disappeared.
+var retiredFlags = map[string]string{
+	"i": "the rclone sweep interval is gone; uploads start the moment a download finishes",
+	"a": "--min-age is gone; a file is only uploaded once the downloader reports it complete",
+	"f": "the export JSON is gone; the chat is read live, so names cannot go stale",
+	"p": "there are no passes; one invocation converges, and re-running resumes",
+	"q": "renamed to --min-free",
+}
+
+// syncCmd archives a chat to a remote: read the chat, skip what is already
+// there, download and upload the rest, then report on the result.
+func syncCmd(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("sync", flag.ContinueOnError)
+	var (
+		chat       = fs.String("c", "", "chat id, username, or t.me link (required)")
+		remoteArg  = fs.String("r", "", "rclone destination, e.g. pikpak:archive (required)")
+		staging    = fs.String("d", "./staging", "staging directory for files in flight")
+		maxStaging = fs.String("m", "", "cap staging at this size, e.g. 40G (default: no cap)")
+		threads    = fs.Int("threads", 4, "connections per file")
+		limit      = fs.Int("limit", 2, "files downloading at once")
+		uploads    = fs.Int("uploads", 2, "files uploading at once")
+		minFree    = fs.Int64("min-free", 5, "stop if the remote has fewer than this many GiB free")
+		limitItems = fs.Int("limit-items", 0, "stop after this many files (0 means no limit)")
+		confirm    = fs.Bool("confirm", true, "re-state each uploaded file to prove its size")
+		takeout    = fs.Bool("takeout", true, "use a takeout session, as `tdl dl --takeout` did")
+		ns         = fs.String("n", "default", "tdl session namespace")
+		dataDir    = fs.String("storage", tdlkv.DefaultDir(), "tdl bolt storage directory")
+	)
+	for name, replacement := range retiredFlags {
+		fs.Var(retiredFlag{name, replacement}, name, "retired")
+	}
+
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return err
+		}
+		return fmt.Errorf("%w: %v", errUsage, err)
+	}
+	if *chat == "" || *remoteArg == "" {
+		return fmt.Errorf("%w: -c CHAT and -r REMOTE:PATH are both required", errUsage)
+	}
+
+	budget, err := parseSize(*maxStaging)
+	if err != nil {
+		return fmt.Errorf("%w: -m %v", errUsage, err)
+	}
+
+	ctx, err = remote.Init(ctx, remote.DefaultTunables())
+	if err != nil {
+		return err
+	}
+	dst, err := remote.Resolve(ctx, *remoteArg)
+	if err != nil {
+		return err
+	}
+
+	// Partial files from an earlier run cannot be continued — core's downloader
+	// takes no starting offset — so they are cleared before anything else fills
+	// the disk with fragments no run will finish.
+	if swept, err := pipeline.SweepPartials(*staging); err != nil {
+		return fmt.Errorf("clear partial downloads: %w", err)
+	} else if swept > 0 {
+		fmt.Fprintf(os.Stderr, "cleared %d partial download(s) from an earlier run\n", swept)
+	}
+
+	// Before anything expensive: prove the destination is reachable and writable.
+	if err := remote.EnsureDir(ctx, dst); err != nil {
+		return err
+	}
+	if err := checkFree(ctx, dst, *minFree); err != nil {
+		return err
+	}
+
+	kv, err := tdlkv.Open(*dataDir, *ns)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = kv.Close() }()
+
+	sess, err := tgsource.New(ctx, tgsource.Options{KV: kv})
+	if err != nil {
+		return err
+	}
+
+	var final verify.Report
+	if err := sess.Run(ctx, func(ctx context.Context, pool dcpool.Pool) error {
+		api := pool.Default(ctx)
+
+		peer, err := tgsource.ResolveChat(ctx, tgsource.Manager(api, tdlstorage.NewPeers(kv)), *chat)
+		if err != nil {
+			return err
+		}
+
+		fmt.Fprintf(os.Stderr, "reading %s\n", *chat)
+		var items []tgsource.Item
+		for it, err := range tgsource.Walk(ctx, api, peer) {
+			if err != nil {
+				return err
+			}
+			items = append(items, it)
+		}
+
+		idx, err := remote.BuildIndex(ctx, dst, peer.ID())
+		if err != nil {
+			return err
+		}
+		before := verify.Check(items, idx)
+		fmt.Fprintf(os.Stderr, "%d media messages, %d already archived, %d to fetch\n",
+			before.Expected, before.Present, len(before.Todo()))
+
+		todo := selectTodo(items, before, *limitItems)
+		if len(todo) == 0 {
+			final = before
+			return nil
+		}
+
+		if err := validateBudget(budget, todo); err != nil {
+			return err
+		}
+
+		var todoBytes int64
+		for _, it := range todo {
+			todoBytes += it.Size()
+		}
+		fmt.Fprintf(os.Stderr, "fetching %d file(s), %.1f GiB\n", len(todo), float64(todoBytes)/(1<<30))
+
+		rep := report.New(os.Stderr, len(todo), todoBytes)
+		res, runErr := pipeline.Run(ctx, sliceSeq(todo), pipeline.Options{
+			Pool:    pool,
+			Dst:     dst,
+			Staging: *staging,
+			Threads: *threads,
+			Limit:   *limit,
+			Uploads: *uploads,
+			Budget:  budget,
+			Confirm: *confirm,
+			Takeout: *takeout,
+			Report:  rep.Update,
+		})
+		rep.Finish(res.Stats)
+
+		for _, f := range res.Failed() {
+			fmt.Fprintf(os.Stderr, "  message %d failed: %v\n", f.Item.MessageID, f.Err)
+		}
+		if runErr != nil {
+			return runErr
+		}
+
+		// The remote is re-indexed rather than assumed: the run's own view of
+		// what it uploaded is exactly the thing under test.
+		idx, err = remote.BuildIndex(ctx, dst, peer.ID())
+		if err != nil {
+			return err
+		}
+		final = verify.Check(items, idx)
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	fmt.Fprintln(os.Stderr)
+	final.Write(os.Stdout)
+
+	if !final.Complete() {
+		return fmt.Errorf("%w: %d file(s) still to fetch", errIncomplete, len(final.Todo()))
+	}
+	return nil
+}
+
+// selectTodo picks the items still needing a fetch, newest first, optionally
+// capped for a smoke test.
+func selectTodo(items []tgsource.Item, r verify.Report, limit int) []tgsource.Item {
+	want := make(map[int]struct{}, len(r.Todo()))
+	for _, id := range r.Todo() {
+		want[id] = struct{}{}
+	}
+	// Unsafe names are in Todo so they stay visible in the report, but fetching
+	// one is impossible by definition, so it is not queued for download.
+	for _, u := range r.Unsafe {
+		delete(want, u.MessageID)
+	}
+
+	var todo []tgsource.Item
+	for _, it := range items {
+		if _, ok := want[it.MessageID]; !ok {
+			continue
+		}
+		todo = append(todo, it)
+		if limit > 0 && len(todo) == limit {
+			break
+		}
+	}
+	return todo
+}
+
+func sliceSeq(items []tgsource.Item) iter.Seq2[tgsource.Item, error] {
+	return func(yield func(tgsource.Item, error) bool) {
+		for _, it := range items {
+			if !yield(it, nil) {
+				return
+			}
+		}
+	}
+}
+
+// validateBudget refuses a cap smaller than the largest file.
+//
+// The semaphore can never admit a weight above its limit, so such a run would
+// block forever on a file it could never start — indistinguishable, from the
+// outside, from a stalled remote. run.sh could only warn about this after the
+// fact, once draining failed to get back under the cap.
+func validateBudget(budget int64, todo []tgsource.Item) error {
+	if budget <= 0 {
+		return nil
+	}
+	var largest int64
+	for _, it := range todo {
+		if it.Size() > largest {
+			largest = it.Size()
+		}
+	}
+	if largest > budget {
+		return fmt.Errorf("%w: -m is %.1f GiB but the largest file to fetch is %.1f GiB; "+
+			"the cap must exceed the biggest single file",
+			errUsage, float64(budget)/(1<<30), float64(largest)/(1<<30))
+	}
+	return nil
+}
+
+// checkFree refuses to start when the remote is nearly full.
+//
+// A backend that cannot report a quota is treated as unlimited rather than as a
+// failure — the shell pipeline made that choice deliberately so a remote without
+// an About API never blocked a run, and it is preserved.
+func checkFree(ctx context.Context, dst fs.Fs, minGiB int64) error {
+	free, ok := remote.FreeBytes(ctx, dst)
+	if !ok {
+		return nil
+	}
+	freeGiB := free / (1 << 30)
+	fmt.Fprintf(os.Stderr, "%s has %d GiB free\n", dst.String(), freeGiB)
+	if freeGiB < minGiB {
+		return fmt.Errorf("%s has only %d GiB free, below the %d GiB floor; "+
+			"free space or lower --min-free", dst.String(), freeGiB, minGiB)
+	}
+	return nil
+}
+
+// retiredFlag reports a helpful error for an option that no longer exists.
+type retiredFlag struct{ name, replacement string }
+
+func (r retiredFlag) String() string { return "" }
+func (r retiredFlag) Set(string) error {
+	return fmt.Errorf("-%s no longer exists: %s", r.name, r.replacement)
+}
+
+// parseSize reads a binary size such as 40G, matching what run.sh -m accepted.
+func parseSize(s string) (int64, error) {
+	if s == "" {
+		return 0, nil
+	}
+	mult := int64(1)
+	switch unit := s[len(s)-1]; unit {
+	case 'K', 'k':
+		mult = 1 << 10
+	case 'M', 'm':
+		mult = 1 << 20
+	case 'G', 'g':
+		mult = 1 << 30
+	case 'T', 't':
+		mult = 1 << 40
+	default:
+		if unit < '0' || unit > '9' {
+			return 0, fmt.Errorf("unknown size suffix %q, expected K, M, G or T", string(unit))
+		}
+	}
+	digits := s
+	if mult > 1 {
+		digits = s[:len(s)-1]
+	}
+
+	var n int64
+	if digits == "" {
+		return 0, fmt.Errorf("%q has no number", s)
+	}
+	for _, r := range digits {
+		if r < '0' || r > '9' {
+			return 0, fmt.Errorf("%q is not a size", s)
+		}
+		n = n*10 + int64(r-'0')
+	}
+	if n <= 0 {
+		return 0, fmt.Errorf("must be greater than zero")
+	}
+	return n * mult, nil
+}

cmd/tgexport/sync_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/iyear/tdl/core/tmedia"
+
+	"github.com/tiennm99dev/telegram-exporter/internal/naming"
+	"github.com/tiennm99dev/telegram-exporter/internal/tgsource"
+	"github.com/tiennm99dev/telegram-exporter/internal/verify"
+)
+
+func TestParseSize(t *testing.T) {
+	ok := map[string]int64{
+		"":     0, // unset means no cap
+		"40G":  40 << 30,
+		"40g":  40 << 30,
+		"512M": 512 << 20,
+		"2T":   2 << 40,
+		"1024": 1024, // bare number is bytes
+		"1K":   1 << 10,
+	}
+	for in, want := range ok {
+		got, err := parseSize(in)
+		if err != nil {
+			t.Errorf("parseSize(%q) = %v, want %d", in, err, want)
+			continue
+		}
+		if got != want {
+			t.Errorf("parseSize(%q) = %d, want %d", in, got, want)
+		}
+	}
+
+	for _, in := range []string{"G", "0", "0G", "-5G", "40GB", "4.5G", "abc", "40Q"} {
+		if got, err := parseSize(in); err == nil {
+			t.Errorf("parseSize(%q) = %d, want an error", in, got)
+		}
+	}
+}
+
+func syncItem(id int, size int64) tgsource.Item {
+	m := &tmedia.Media{Name: "f.mp4", Size: size}
+	return tgsource.Item{DialogID: 1, MessageID: id, Name: naming.For(1, id, m), Media: m}
+}
+
+func TestSelectTodoPicksOnlyOutstandingItems(t *testing.T) {
+	items := []tgsource.Item{syncItem(1, 10), syncItem(2, 10), syncItem(3, 10), syncItem(4, 10)}
+	r := verify.Report{Absent: []int{2, 4}, ZeroByte: []int{3}}
+
+	got := selectTodo(items, r, 0)
+	if len(got) != 3 {
+		t.Fatalf("selected %d items, want 3", len(got))
+	}
+	for _, it := range got {
+		if it.MessageID == 1 {
+			t.Error("selected an item that is already archived")
+		}
+	}
+}
+
+// An unsafe name is kept in the report so it stays visible, but queuing it for
+// download would retry something that can never succeed — the non-terminating
+// loop this design exists to avoid.
+func TestSelectTodoSkipsUnsafeNames(t *testing.T) {
+	items := []tgsource.Item{syncItem(1, 10), syncItem(2, 10)}
+	r := verify.Report{
+		Absent: []int{1, 2},
+		Unsafe: []verify.Unsafe{{MessageID: 2, Name: "../x", Reason: errors.New("unsafe")}},
+	}
+
+	got := selectTodo(items, r, 0)
+	if len(got) != 1 || got[0].MessageID != 1 {
+		t.Errorf("selected %+v, want only message 1", got)
+	}
+}
+
+func TestSelectTodoHonoursLimit(t *testing.T) {
+	items := []tgsource.Item{syncItem(1, 10), syncItem(2, 10), syncItem(3, 10)}
+	r := verify.Report{Absent: []int{1, 2, 3}}
+
+	if got := selectTodo(items, r, 2); len(got) != 2 {
+		t.Errorf("selected %d items with a limit of 2, want 2", len(got))
+	}
+	if got := selectTodo(items, r, 0); len(got) != 3 {
+		t.Errorf("selected %d items with no limit, want 3", len(got))
+	}
+}
+
+// A cap below the largest file could never admit it, so the run would block on
+// something it can never start — which from outside looks like a stalled remote.
+// It has to be refused up front.
+func TestValidateBudgetRejectsCapBelowLargestFile(t *testing.T) {
+	todo := []tgsource.Item{syncItem(1, 1<<20), syncItem(2, 5<<30)}
+
+	err := validateBudget(4<<30, todo)
+	if err == nil {
+		t.Fatal("a 4 GiB cap was accepted with a 5 GiB file to fetch")
+	}
+	if !errors.Is(err, errUsage) {
+		t.Errorf("error should be a usage error, got: %v", err)
+	}
+	if !strings.Contains(err.Error(), "largest file") {
+		t.Errorf("error should name the problem, got: %v", err)
+	}
+
+	if err := validateBudget(6<<30, todo); err != nil {
+		t.Errorf("a 6 GiB cap should accept a 5 GiB file, got: %v", err)
+	}
+	if err := validateBudget(0, todo); err != nil {
+		t.Errorf("an unset cap should accept anything, got: %v", err)
+	}
+}
+
+// Options the shell pipeline had must produce an explanation, not "flag
+// provided but not defined" — they are in wrapper scripts and muscle memory.
+func TestRetiredFlagsExplainWhatReplacedThem(t *testing.T) {
+	for name, replacement := range retiredFlags {
+		err := retiredFlag{name, replacement}.Set("x")
+		if err == nil {
+			t.Errorf("-%s was accepted, want an explanation", name)
+			continue
+		}
+		if !strings.Contains(err.Error(), "-"+name) {
+			t.Errorf("error for -%s should name the flag, got: %v", name, err)
+		}
+		if !strings.Contains(err.Error(), replacement) {
+			t.Errorf("error for -%s should say what replaced it, got: %v", name, err)
+		}
+	}
+
+	// The ones that mattered most in run.sh.
+	for _, name := range []string{"i", "a", "f", "p", "q"} {
+		if _, ok := retiredFlags[name]; !ok {
+			t.Errorf("-%s was a run.sh flag but is not recognised as retired", name)
+		}
+	}
+}