diff --git a/cmd/tgexport/list.go b/cmd/tgexport/list.go new file mode 100644 index 0000000..7afb020 --- /dev/null +++ b/cmd/tgexport/list.go @@ -0,0 +1,85 @@ +package main + +import ( + "bufio" + "context" + "errors" + "flag" + "fmt" + "os" + + "github.com/iyear/tdl/core/dcpool" + tdlstorage "github.com/iyear/tdl/core/storage" + + "github.com/tiennm99dev/telegram-exporter/internal/tdlkv" + "github.com/tiennm99dev/telegram-exporter/internal/tgsource" +) + +// listCmd prints every media message in a chat as `idsizename`. +// +// It is the smallest thing that exercises the whole read path — resolve a chat, +// walk its history, derive a name — so a naming or paging problem shows up here +// rather than halfway through an archive run. The output is tab-separated on +// purpose: filenames contain spaces, commas and quotes, but not tabs. +func listCmd(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("list", flag.ContinueOnError) + var ( + chat = fs.String("c", "", "chat id, username, or t.me link (required)") + ns = fs.String("n", "default", "tdl session namespace") + dataDir = fs.String("storage", tdlkv.DefaultDir(), "tdl bolt storage directory") + ) + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return err + } + return fmt.Errorf("%w: %v", errUsage, err) + } + if *chat == "" { + return fmt.Errorf("%w: -c CHAT is required", errUsage) + } + + kv, err := tdlkv.Open(*dataDir, *ns) + if err != nil { + return err + } + defer func() { _ = kv.Close() }() + + sess, err := tgsource.New(ctx, tgsource.Options{KV: kv}) + if err != nil { + return err + } + + return sess.Run(ctx, func(ctx context.Context, pool dcpool.Pool) error { + api := pool.Default(ctx) + + peer, err := tgsource.ResolveChat(ctx, tgsource.Manager(api, tdlstorage.NewPeers(kv)), *chat) + if err != nil { + return err + } + + // Buffered: one write syscall per item would dominate the runtime on a + // chat with tens of thousands of messages. Flushed explicitly below so a + // write failure — a closed pipe, a full disk — is reported rather than + // swallowed by a deferred call nobody checks. + out := bufio.NewWriter(os.Stdout) + + count := 0 + var total int64 + for it, err := range tgsource.Walk(ctx, api, peer) { + if err != nil { + return err + } + count++ + total += it.Size() + if _, err := fmt.Fprintf(out, "%d\t%d\t%s\n", it.MessageID, it.Size(), it.Name); err != nil { + return err + } + } + + if err := out.Flush(); err != nil { + return err + } + fmt.Fprintf(os.Stderr, "\n%d media messages, %.1f GiB\n", count, float64(total)/(1<<30)) + return nil + }) +} diff --git a/cmd/tgexport/main.go b/cmd/tgexport/main.go index cc9738c..eab509c 100644 --- a/cmd/tgexport/main.go +++ b/cmd/tgexport/main.go @@ -61,6 +61,8 @@ func run() int { switch os.Args[1] { case "doctor": err = doctorCmd(ctx, os.Args[2:]) + case "list": + err = listCmd(ctx, os.Args[2:]) default: fmt.Fprintf(os.Stderr, "unknown command %q\n\n", os.Args[1]) usage() @@ -147,6 +149,7 @@ func usage() { Commands: doctor Check the Telegram session, the destination remote, and free space + list Print every media message in a chat as idsizename Run 'tgexport -h' for command options. `) diff --git a/internal/naming/naming.go b/internal/naming/naming.go new file mode 100644 index 0000000..629ffe5 --- /dev/null +++ b/internal/naming/naming.go @@ -0,0 +1,62 @@ +// Package naming builds the filename a media message is stored under. +// +// This package exists to have exactly one answer to "what is this file called". +// The shell pipeline it replaces had two, and they disagreed. `tdl chat export` +// wrote the raw Telegram filename into a JSON, while `tdl dl` ran that same name +// through its download template — whose default is +// +// {{ .DialogID }}_{{ .MessageID }}_{{ filenamify .FileName }} +// +// (tdl@v0.20.4/cmd/dl.go:50). `filenamify` rewrites characters a filesystem +// rejects and, incidentally, collapses any run of two or more '!' into one. So a +// message whose filename contained "!!" was checked for under one name and +// stored under another; the verifier never found it and re-fetched it on every +// pass, forever. That is not a hypothetical — it cost 966 MB per pass on one +// message in this repo's own archive. +// +// The rule here is therefore not "be careful to keep the two in sync". There is +// one function, called once per message, and the string it returns is used both +// to ask whether the file is already archived and to write it. A divergence +// between those two questions is not made unlikely; it is made unrepresentable. +package naming + +import ( + "strconv" + "strings" + + "github.com/iyear/tdl/core/tmedia" +) + +// Separator between the three fields of a stored filename. +const sep = "_" + +// For returns the archive filename for one media message: +// +// {DialogID}_{MessageID}_{FileName} +// +// The field layout matches tdl's default template, but FileName does not: tdl +// passes it through `filenamify` and this does not. That is a deliberate, +// recorded choice (see the plan's Phase 2 notes) — names stay as Telegram +// reports them rather than being rewritten — and it means a file the old shell +// pipeline stored under a filenamify-altered name will not be recognised here +// and will be fetched again. For this repo's archive that affects exactly one +// message out of 12,000, and it was already removed. +// +// FileName comes from tmedia, the same extractor tdl uses: a document's +// DocumentAttributeFilename, or a generated stable name when it has none +// (`.jpg` for a photo, `` for a document). +// +// The name is taken verbatim and is therefore NOT safe to join onto a path. +// DocumentAttributeFilename is set by whoever uploaded the file, so it can +// contain '/' or '..' and escape a staging directory. Callers that turn a name +// into a path must check containment themselves; doing it here would silently +// rewrite names and reintroduce exactly the two-derivations problem above. +func For(dialogID int64, messageID int, m *tmedia.Media) string { + var b strings.Builder + b.WriteString(strconv.FormatInt(dialogID, 10)) + b.WriteString(sep) + b.WriteString(strconv.Itoa(messageID)) + b.WriteString(sep) + b.WriteString(m.Name) + return b.String() +} diff --git a/internal/naming/naming_test.go b/internal/naming/naming_test.go new file mode 100644 index 0000000..a96f8e4 --- /dev/null +++ b/internal/naming/naming_test.go @@ -0,0 +1,89 @@ +package naming + +import ( + "testing" + + "github.com/iyear/tdl/core/tmedia" +) + +// The format is a compatibility contract, not a style choice: ~15k files are +// already stored under it. Changing it makes every one of them look absent and +// re-downloads the entire archive. +func TestForMatchesStoredLayout(t *testing.T) { + tests := []struct { + name string + dialogID int64 + msgID int + file string + want string + }{ + { + name: "document with a filename attribute", + dialogID: 1234567890, + msgID: 14726, + file: "298.mp4", + want: "1234567890_14726_298.mp4", + }, + { + // The message that started this rewrite. Its name carries a doubled + // '!' which tdl's default template collapsed to one, via filenamify, + // while the export JSON kept both — the two derivations that never + // agreed. This package keeps the name as Telegram reports it, so the + // doubled '!' must survive. + name: "punctuation is preserved verbatim", + dialogID: 1234567890, + msgID: 4242, + file: "Pipe her!! And by her, we mean pipeperr! 1080p.mp4", + want: "1234567890_4242_Pipe her!! And by her, we mean pipeperr! 1080p.mp4", + }, + { + name: "photo gets tmedia's generated name", + dialogID: 1234567890, + msgID: 42, + file: "5901234567890123456.jpg", + want: "1234567890_42_5901234567890123456.jpg", + }, + { + name: "spaces and separators inside the filename are untouched", + dialogID: 1234567890, + msgID: 7, + file: "a_b c-d.e.mp4", + want: "1234567890_7_a_b c-d.e.mp4", + }, + { + name: "non-ascii is untouched", + dialogID: 1234567890, + msgID: 8, + file: "ünïcödé näme 🍓.mp4", + want: "1234567890_8_ünïcödé näme 🍓.mp4", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := For(tt.dialogID, tt.msgID, &tmedia.Media{Name: tt.file}) + if got != tt.want { + t.Errorf("For() = %q, want %q", got, tt.want) + } + }) + } +} + +// Guards the reason the field layout is what it is. If this ever starts failing, +// tdl changed its default template and the compatibility note on For is stale. +func TestForKeepsCharactersTdlWouldRewrite(t *testing.T) { + // filenamify (tdl's default template applies it) collapses runs of '!' and + // replaces reserved characters. None of that may happen here. + for _, file := range []string{ + "double!!bang.mp4", + "a?b:c.mp4", + ".leading-dot.mp4", + "trailing!.mp4", + } { + got := For(1, 2, &tmedia.Media{Name: file}) + want := "1_2_" + file + if got != want { + t.Errorf("For(%q) = %q, want %q — a sanitiser crept in", file, got, want) + } + } +} diff --git a/internal/naming/sole_source_test.go b/internal/naming/sole_source_test.go new file mode 100644 index 0000000..fe7f743 --- /dev/null +++ b/internal/naming/sole_source_test.go @@ -0,0 +1,75 @@ +package naming + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" +) + +// buildsAName matches the ways a stored filename would plausibly be assembled by +// hand: the format verbs ("%d_%d_%s" and relatives) and string concatenation +// around a bare "_" separator. +// +// This is a lint for known shapes, not a proof. A determined reimplementation — +// a strings.Builder copy of For, say — still slips through. It catches the +// realistic accident, which is someone reaching for Sprintf in a new file. +var buildsAName = regexp.MustCompile(`%[ds]_%[ds]|_%[ds]_|\+ *"_" *\+`) + +// The whole point of this package is that it is the only answer to "what is this +// file called". Nothing in the type system prevents a second place from +// formatting the same string, so the common ways of doing so are linted here. +// +// If this test fails, the fix is to call For (or MessageID) rather than to widen +// the pattern. The shell pipeline's bug was two independent name derivations that +// nothing forced to agree; a second one here would reintroduce it exactly. +func TestNamingIsTheSoleSourceOfFilenames(t *testing.T) { + root, err := filepath.Abs("../..") + if err != nil { + t.Fatalf("locate repo root: %v", err) + } + + var offenders []string + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + // Skip VCS and anything vendored; only our own source counts. + switch d.Name() { + case ".git", "vendor", "plans", "staging": + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + // This package is the one place allowed to build the name. + if filepath.Dir(path) == filepath.Join(root, "internal", "naming") { + return nil + } + + src, err := os.ReadFile(path) + if err != nil { + return err + } + for i, line := range strings.Split(string(src), "\n") { + if buildsAName.MatchString(line) { + rel, _ := filepath.Rel(root, path) + offenders = append(offenders, rel+":"+strconv.Itoa(i+1)+": "+strings.TrimSpace(line)) + } + } + return nil + }) + if err != nil { + t.Fatalf("walk source tree: %v", err) + } + + if len(offenders) > 0 { + t.Errorf("filenames must only be built by naming.For; found %d other place(s):\n %s", + len(offenders), strings.Join(offenders, "\n ")) + } +} diff --git a/internal/tgsource/chat.go b/internal/tgsource/chat.go new file mode 100644 index 0000000..7729c84 --- /dev/null +++ b/internal/tgsource/chat.go @@ -0,0 +1,133 @@ +package tgsource + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/gotd/td/telegram/peers" + + "github.com/iyear/tdl/core/util/tutil" +) + +// botAPIID matches a Bot API chat id: the same channel as an MTProto id, but +// with a -100 prefix that MTProto itself does not use. +var botAPIID = regexp.MustCompile(`^-100(\d+)$`) + +// telegramHosts are the hosts Telegram deep links use. gotd accepts all three +// (telegram/deeplink/deeplink.go hasTelegramPrefix), so checking only t.me would +// let a telegram.me or telegram.dog message link through — and gotd's parser +// keeps just the domain and silently drops the message number, which would walk +// an entire chat when the operator asked for one message. +var telegramHosts = []string{"t.me/", "telegram.me/", "telegram.dog/"} + +// isMessageLink reports whether s points at a single message rather than a chat. +// +// This is parsed rather than pattern-matched because the two HTTPS shapes overlap +// in a way a regex gets wrong: a private link is t.me/c// and a public +// one is t.me//, so "t.me/c/1234567890" — a perfectly good private +// *channel* link — looks exactly like a public message link with the username +// "c". The distinction is whether a trailing numeric component follows the chat, +// and where that component sits depends on the "c" marker. +func isMessageLink(s string) bool { + lower := strings.ToLower(s) + + // tg:// links name the message in a query parameter rather than the path. + if strings.HasPrefix(lower, "tg://") { + for _, key := range []string{"post=", "message_id="} { + if strings.Contains(lower, "?"+key) || strings.Contains(lower, "&"+key) { + return true + } + } + return false + } + + rest, ok := afterHost(lower) + if !ok { + return false + } + rest, _, _ = strings.Cut(rest, "?") + rest, _, _ = strings.Cut(rest, "#") + + parts := strings.Split(strings.Trim(rest, "/"), "/") + if len(parts) >= 1 && parts[0] == "c" { + // c/ is the channel; c// is one message in it. + return len(parts) >= 3 && isDigits(parts[2]) + } + // t.me/joinchat/ and t.me/s/ are chats, not messages, and their + // second component is not a bare number — except for a hypothetical all-digit + // invite hash, which is not worth mis-parsing every real link to guard. + if len(parts) >= 1 && (parts[0] == "joinchat" || parts[0] == "s") { + return false + } + return len(parts) >= 2 && isDigits(parts[1]) +} + +// afterHost returns the path following a Telegram host, if s names one. +func afterHost(lower string) (string, bool) { + for _, host := range telegramHosts { + if i := strings.Index(lower, host); i >= 0 { + return lower[i+len(host):], true + } + } + return "", false +} + +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// NormalizeChat converts a chat argument into the form the resolver expects. +// +// The accepted forms are the ones the shell pipeline accepted, because they are +// what an operator already has to hand: a numeric MTProto id as printed by +// `tdl chat ls`, a username with or without '@', or a t.me/tg:// link. Two need +// help. A Bot API id carries a -100 prefix that MTProto does not use, and a +// message link is not a chat — silently treating one as a chat would export the +// wrong thing, so it is refused with an explanation rather than guessed at. +func NormalizeChat(chat string) (string, error) { + chat = strings.TrimSpace(chat) + if chat == "" { + return "", fmt.Errorf("a chat is required") + } + + if isMessageLink(chat) { + return "", fmt.Errorf("%q is a message link, not a chat — "+ + "pass the chat's username or id instead", chat) + } + + if m := botAPIID.FindStringSubmatch(chat); m != nil { + return m[1], nil + } + + // The resolver takes a bare username; '@' is how humans write it. + return strings.TrimPrefix(chat, "@"), nil +} + +// ResolveChat turns a chat argument into a peer. +// +// Numeric arguments are looked up as channel, then user, then chat ids; +// everything else goes through the resolver, which handles usernames and +// t.me/tg:// links. That ordering is tdl's (core/util/tutil.GetInputPeer), kept +// so an id that works in `tdl` works here. +func ResolveChat(ctx context.Context, manager *peers.Manager, chat string) (peers.Peer, error) { + normalized, err := NormalizeChat(chat) + if err != nil { + return nil, err + } + + peer, err := tutil.GetInputPeer(ctx, manager, normalized) + if err != nil { + return nil, fmt.Errorf("cannot resolve chat %q: %w", chat, err) + } + return peer, nil +} diff --git a/internal/tgsource/chat_test.go b/internal/tgsource/chat_test.go new file mode 100644 index 0000000..d13deff --- /dev/null +++ b/internal/tgsource/chat_test.go @@ -0,0 +1,103 @@ +package tgsource + +import ( + "strings" + "testing" +) + +// Every form the shell pipeline accepted must still be accepted, and the one +// form it refused must still be refused. An operator's existing command lines +// are the compatibility surface here. +func TestNormalizeChat(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {"numeric mtproto id", "1234567890", "1234567890"}, + {"username with at", "@mychannel", "mychannel"}, + {"bare username", "mychannel", "mychannel"}, + {"public t.me link", "https://t.me/mychannel", "https://t.me/mychannel"}, + {"tg protocol link", "tg://resolve?domain=mychannel", "tg://resolve?domain=mychannel"}, + {"bot api id loses the -100 prefix", "-1001234567890", "1234567890"}, + {"surrounding whitespace is trimmed", " mychannel\n", "mychannel"}, + {"private channel link without a message", "https://t.me/c/1234567890", "https://t.me/c/1234567890"}, + // Invite and preview links are chats; their second component is not a + // bare message number and must not be read as one. + {"invite link", "https://t.me/+AbCd_1234", "https://t.me/+AbCd_1234"}, + {"joinchat link", "https://t.me/joinchat/AbCd1234", "https://t.me/joinchat/AbCd1234"}, + {"preview link", "https://t.me/s/mychannel", "https://t.me/s/mychannel"}, + {"other telegram host, no message", "https://telegram.dog/mychannel", "https://telegram.dog/mychannel"}, + {"tg link without a post parameter", "tg://resolve?domain=mychannel", "tg://resolve?domain=mychannel"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NormalizeChat(tt.in) + if err != nil { + t.Fatalf("NormalizeChat(%q) returned an error: %v", tt.in, err) + } + if got != tt.want { + t.Errorf("NormalizeChat(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} + +// A message link names one message, not a chat. Guessing the chat from it would +// quietly export something the operator did not ask for, so it is refused. +func TestNormalizeChatRejectsMessageLinks(t *testing.T) { + for _, in := range []string{ + "https://t.me/c/1234567890/4242", + "t.me/c/1234567890/4242", + "https://t.me/mychannel/4242", + // gotd accepts all three Telegram hosts and its parser keeps only the + // domain, silently dropping the message number — so missing one of these + // would walk an entire chat when one message was asked for. + "https://telegram.me/mychannel/4242", + "https://telegram.dog/mychannel/4242", + "https://T.ME/mychannel/4242", + // tg:// names the message in a query parameter, not the path. + "tg://privatepost?channel=1234567890&post=4242", + "tg://resolve?domain=mychannel&post=4242", + "tg://openmessage?user_id=1&message_id=4242", + } { + t.Run(in, func(t *testing.T) { + _, err := NormalizeChat(in) + if err == nil { + t.Fatalf("NormalizeChat(%q) succeeded; a message link is not a chat", in) + } + if !strings.Contains(err.Error(), "message link") { + t.Errorf("error should say it is a message link, got: %v", err) + } + }) + } +} + +func TestNormalizeChatRejectsEmpty(t *testing.T) { + for _, in := range []string{"", " ", "\t\n"} { + if _, err := NormalizeChat(in); err == nil { + t.Errorf("NormalizeChat(%q) succeeded, want an error", in) + } + } +} + +// -100 is stripped only when it prefixes a Bot API id, never from an ordinary +// number that happens to start with those digits. +func TestNormalizeChatOnlyStripsRealBotAPIPrefix(t *testing.T) { + tests := map[string]string{ + "-1001234567890": "1234567890", // Bot API id + "1001234567890": "1001234567890", // no leading '-', not a Bot API id + "-100": "-100", // prefix with no id after it + "-2001234567890": "-2001234567890", // different prefix + } + for in, want := range tests { + got, err := NormalizeChat(in) + if err != nil { + t.Fatalf("NormalizeChat(%q): %v", in, err) + } + if got != want { + t.Errorf("NormalizeChat(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/tgsource/iterate.go b/internal/tgsource/iterate.go new file mode 100644 index 0000000..53d2d66 --- /dev/null +++ b/internal/tgsource/iterate.go @@ -0,0 +1,86 @@ +package tgsource + +import ( + "context" + "fmt" + "iter" + + "github.com/gotd/td/telegram/peers" + "github.com/gotd/td/telegram/query" + "github.com/gotd/td/tg" + + "github.com/iyear/tdl/core/tmedia" + + "github.com/tiennm99dev/telegram-exporter/internal/naming" +) + +// Item is one downloadable media message. +// +// Name is filled here, at the single point where the message is seen, and is the +// same string used to check the remote and to write the file. See the naming +// package for why that matters. Media carries the location, size and DC that the +// downloader needs, so nothing has to be looked up a second time. +type Item struct { + DialogID int64 + MessageID int + Name string + Media *tmedia.Media +} + +// Size reports the media size in bytes. +func (i Item) Size() int64 { return i.Media.Size } + +// Walk yields every media message in a chat, newest first. +// +// Order is Telegram's: GetHistory pages backwards from the most recent message. +// The shell pipeline fetched oldest-first, so an interrupted run leaves a +// different subset archived than the old one would have. +// +// A sequence rather than a callback because the downloader consumes a pull +// iterator (Next/Value/Err), and range-over-func converts either way for free: +// callers that want the callback shape just range over it, while iter.Pull2 +// gives the pull shape without anyone owning a goroutine. Messages are streamed, +// never collected — an 18k-message chat is tens of thousands of descriptors and +// the caller decides what to keep. +// +// Text-only and service messages carry no file and are skipped, the same rule +// the export JSON encoded as an empty "file" field. On error the sequence yields +// a zero Item with that error and stops; cancelling ctx stops it too, so an +// interrupted run does not keep paging. +func Walk(ctx context.Context, api *tg.Client, peer peers.Peer) iter.Seq2[Item, error] { + return func(yield func(Item, error) bool) { + dialogID := peer.ID() + + it := query.NewQuery(api).Messages().GetHistory(peer.InputPeer()).BatchSize(100).Iter() + for it.Next(ctx) { + msg, ok := it.Value().Msg.(*tg.Message) + if !ok { + continue // service messages have no media + } + + media, ok := tmedia.GetMedia(msg) + if !ok { + continue // text-only, or a media kind tmedia cannot download + } + + if !yield(Item{ + DialogID: dialogID, + MessageID: msg.ID, + Name: naming.For(dialogID, msg.ID, media), + Media: media, + }, nil) { + return + } + } + + if err := it.Err(); err != nil { + yield(Item{}, fmt.Errorf("walk chat history: %w", err)) + } + } +} + +// Manager builds a peers manager over the session's peer cache, so resolving the +// same chat twice does not cost a second round trip. +func Manager(api *tg.Client, storage peers.Storage) *peers.Manager { + return peers.Options{Storage: storage}.Build(api) +}