diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 89b5e37..0000000 --- a/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -.venv/ -*.session -*.session-journal -*.session-wal -*.session-shm -spike/ -exports/ -.export-state.json -.export.lock -__pycache__/ -*.egg-info/ -.env -.coverage -.pytest_cache/ -.claude/agent-memory/ diff --git a/README.md b/README.md index 68f057b..69fa95b 100644 --- a/README.md +++ b/README.md @@ -1,281 +1,124 @@ -# telegram-exporter +# telegram-exporter — ⚠️ churned -`tg-export` downloads **all media from a Telegram group** to local disk, one -folder per logical post (albums collapsed), with a `messages.jsonl` sidecar that -links every file back to its message. Resumable, filterable, with a dry-run size -estimate. +**Status: churned as of 2026-08-25. No further development, no maintenance, no +issue support.** This repo now contains only these instructions; the retired +Python implementation lives in git history (see the last section). + +## Why + +[iyear/tdl](https://github.com/iyear/tdl) — an actively maintained Telegram +toolkit in Go, built on [gotd/td](https://github.com/gotd/td) — already covers +this project's purpose: export chat history and download media from DMs, groups +and channels over MTProto with a user account (so neither Bot API limit applies: +it can read history, and there is no 20 MB download cap). Maintaining a parallel +tool is not justified. + +In short: a gotd-based rewrite was researched and judged feasible but not +worth it, since tdl already exists and is proven; the original project's +research and build reports are in git history at commit `3286ee2`. + +## Use tdl instead + +Install: see . ```bash -export TG_API_ID=... TG_API_HASH=... +tdl login # user-account login (phone + code + 2FA) -tg-export --group @mygroup --out ./exports --dry-run # what would it fetch? -tg-export --group @mygroup --out ./exports # fetch it +# 1) export message metadata to JSON +tdl chat export -c @mygroup --all --with-content -o export.json + +# 2) download the media it references +tdl dl -f export.json -d ./exports --takeout --group --skip-same --continue ``` -Kill it at any point and re-run the same command; it resumes without -re-downloading and without gaps. +Flag notes, mapped to what this project used to do: -## Why the Bot API cannot do this +- `--takeout` — takeout session with lower flood-wait limits for bulk export + (an improvement this project never had). +- `--group` — detect albums (`grouped_id`) and download grouped messages together. +- `--skip-same` + `--continue` — resume: skip files matching existing name+size, + continue interrupted downloads without prompting. +- `--all --with-content` — include text-only messages with their content + (equivalent of `--include-text`). +- Filters: time range / message-id range via `-i`, plus expression filters, e.g. + `-f "Media.Size > 5*1024*1024"`. Extension filters via `-i jpg,png` / `-e mp4`. +- tdl defaults to aggressive parallelism (`-t 8 -l 4`). If you hit flood waits + on a large export, lower it (`-t 4 -l 1`). -This is the first question most people ask, so: a bot cannot read a group's -message history at all, and the Bot API caps file downloads at 20 MB. Both limits -are structural, not configuration. Exporting history therefore requires MTProto -with a **user account**, which is what Telethon provides. +What tdl does **not** replicate from this project's contract: per-post album +folders keyed on message id, the `messages.jsonl` append-only sidecar, the +cursor with filter-mismatch refusal, and the dry-run disk estimate. If you need +those guarantees, the old implementation is in this repo's history and `src/`. -## Install +## Special case: exporting to WebDAV -Python 3.12+. +tdl can only write to a local directory — it has no WebDAV (or any remote) +destination. When the group's media is larger than local disk, use a **rolling +pipeline**: tdl downloads into a small staging directory while +[rclone](https://rclone.org/) concurrently moves (upload + delete local) +finished files to WebDAV. Local disk then only needs to hold the files +currently in flight plus one sync interval of throughput — Telegram caps a +single file at 2 GB (4 GB from premium uploaders), so a few dozen GB of staging +covers the worst case regardless of the group's total size. + +One-time rclone remote setup: ```bash -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt -e . -.venv/bin/tg-export --help +rclone config create tg-webdav webdav \ + url=https://dav.example.com/remote.php/dav/files/you \ + vendor=other user=YOU pass=SECRET ``` -`requirements.txt` pins the exact Telethon version the API probes were verified -against. Telethon 2.0 is alpha with a renamed API surface, hence the `<2` bound. - -`cryptg` is **optional and not installed**. It is a native extension sitting in -Telethon's AES path, so it sees the auth key and every plaintext byte. It ships an -aarch64 wheel (no build step), and the reason to trust it *if* you adopt it is -that the Telethon author maintains it — not that it is "CPU-side only". Do not add -a supply-chain root for a throughput problem you have not measured. - -## Credentials - -Create an app at → API development tools. - -| Credential | Source | Why | -|---|---|---| -| `api_id`, `api_hash` | env `TG_API_ID` / `TG_API_HASH` only | needed every run; revocable | -| phone | prompt; `TG_PHONE` optional | PII, not a secret | -| login code | **prompt only** | single-use, 5-minute TTL | -| 2FA password | **`getpass` only** — no env, no file, no flag | once per session lifetime | - -`.env` **may** contain `TG_API_ID`, `TG_API_HASH`, `TG_PHONE`. It **must not** -contain the 2FA password, a login code, or session data. The app does not read -`.env` — your shell already does that: +Pipeline (bash): ```bash -set -a; . ./.env; set +a +tdl dl -f export.json -d ./staging --takeout --group --skip-same --continue & +TDL_PID=$! +while kill -0 "$TDL_PID" 2>/dev/null; do + rclone move ./staging tg-webdav:tg-export --min-age 2m --delete-empty-src-dirs + sleep 60 +done +rclone move ./staging tg-webdav:tg-export --delete-empty-src-dirs # final sweep ``` -### First run and the session file +Pipeline (PowerShell): -The first run prompts for your phone, the login code Telegram sends you, and your -2FA password if you have one. Later runs are non-interactive. - -The session file defaults to -`${XDG_STATE_HOME:-~/.local/state}/tg-export/default.session` — **outside the -repo**, so the safe location needs no opt-in. It and every SQLite sibling are mode -`0600`, enforced by a `umask(0o077)` set before anything can create a file, plus a -post-login assertion. A `chmod` after connecting would be too late: SQLite writes -the auth key *during* login. - -If you point `--session` or `--out` inside a git repo at a path that is not -gitignored, the run refuses with an explanation. Deleting a session file does -**not** revoke it server-side — use Telegram → Settings → Devices for that. - -## Output layout - -``` -exports/g-1001234567890/ <- directory name is the chat id, never the group title - title.txt <- the human-readable name lives here, as data - 1042/ <- one album (shared grouped_id); folder = first member id - 1042_photo.jpg <- filename keyed on message id, not a positional index - 1043_photo.jpg - 1044_video.mp4 - 1047/ <- a standalone message - 1047_document.pdf - messages.jsonl - .export-state.json - .export.lock +```powershell +$tdl = Start-Process tdl -ArgumentList 'dl','-f','export.json','-d','./staging','--takeout','--group','--skip-same','--continue' -PassThru -NoNewWindow +while (-not $tdl.HasExited) { + rclone move ./staging tg-webdav:tg-export --min-age 2m --delete-empty-src-dirs + Start-Sleep -Seconds 60 +} +rclone move ./staging tg-webdav:tg-export --delete-empty-src-dirs ``` -The directory is `g` and not the group title because the title is -server-supplied and editable by any admin. A title of `..` or `a/b` would be a -path-traversal vector that no filename sanitizer could catch, since the escape -happens in a parent component. Naming the directory after an int makes that -structurally impossible — and also means a mid-export group rename cannot orphan -your download. The cost is that `ls exports/` shows numbers. +Why it works: -Filenames are keyed on the **message id** rather than a position in the album, -because a positional index shifts when a member message is deleted between runs, -silently orphaning files already on disk. +- `--min-age 2m` keeps rclone away from files tdl is still writing; the final + sweep after tdl exits catches everything else. +- Both legs are independently resumable: re-run tdl (`--skip-same --continue`) + and re-run the rclone loop; nothing is downloaded or uploaded twice. +- Caveat: `--skip-same` compares name+size against the **staging** dir, which + is empty after files move to WebDAV. Cross-run dedupe therefore rests on + `--continue` (tdl's own completion tracking) — keep the same `export.json` + between runs. If you restart with a fresh export, narrow it to the missing + message-id range (`-T id -i ,`) instead of re-downloading + everything. -## Resume semantics +**Zero-staging streaming (no local disk at all) is not possible with tdl.** It +requires custom code that pipes download chunks straight into a WebDAV `PUT` — +both Telethon (`iter_download`) and gotd (`Stream(ctx, w)`) support that; see +the reports in `plans/reports/` if you ever need to build it. -`.export-state.json` holds one cursor, and it means: +## The retired implementation -> every post with `max_message_id <= cursor_id` has been handled **under these -> filters**. - -Consequences worth knowing: - -- The cursor is written only after a post is completely handled, and never from an - error path. It can never point past in-flight work. -- Filters are stored verbatim. Changing them refuses the run and prints exactly - what changed, because the cursor's meaning depends on them: - - ``` - filter mismatch: - types: ['photo', 'video'] -> ['video'] - max_size: 100000000 -> (none) - re-run with the original filters, or --reset-state to start over - ``` - -- `--reset-state` discards the cursor and rotates `messages.jsonl` to - `messages-.jsonl`. It does **not** delete downloaded files — - dedupe reclaims them on the next run, and deleting 30 GB to change a filter - would be hostile. -- `--limit N` is *not* part of the filter set. It changes when we stop, not what a - post contains, so a `--limit 20` test run is a genuine resumable partial export - that does not poison later full runs. -- `completed_at` is set only when a run exhausts the history cleanly. It is the - only way to answer "did my 20-hour export actually finish?". -- **A file that fails is recorded and skipped, and the cursor moves past it.** The - run continues and the exit summary names the count (`N files failed (see - messages.jsonl)`). Media that was deleted or expired is genuinely gone, but a - file that exhausted its retries is not re-attempted by a later resume — grep - `errors` in `messages.jsonl` to find them, and use `--reset-state` to re-drive - that range if it matters. - -## `messages.jsonl` - -```json -{"message_id":1043,"post_id":1042,"grouped_id":88,"date":"2026-07-04T12:00:00+00:00", - "sender_id":777,"sender_name":"Alice","caption":"beach trip","reply_to":1001, - "files":[{"path":"1042/1043_photo.jpg","size":1234567,"name":"photo.jpg", - "mime":"image/jpeg","kind":"photo","declared_size":1234999}], - "errors":[]} -``` - -**Each record is an append-only event: "the files written for this message in this -run."** So the correct way to read the sidecar is the **union of all records for a -`message_id`**, never last-write-wins — after a filter change, the most recent -record deliberately describes a narrower slice than what is on disk. - -- `size` is bytes on disk; `declared_size` appears only when the server's number - differs (see the photo note below). -- Paths are relative to the export root. -- `sender_name` is filled in only when the sender was already cached. Resolving it - otherwise would mean one extra RPC per message — 100k RPCs and a flood ban on a - large group, in exchange for a display string. -- A partial final line from a host crash is truncated and warned about at startup. -- With `--include-text`, text-only messages get the same record shape with - `"files": []`. Distinguish media from context by `files` being empty, not by a - record type. - -## Flags - -``` -tg-export --group --out DIR - [--dry-run] [--since YYYY-MM-DD] [--until YYYY-MM-DD] - [--types photo,video,document,audio,voice] [--max-size 100MB] - [--include-text] [--limit N] [--session PATH] [--reset-state] - [--min-free 2GiB] [--max-flood-wait SECONDS] [--verbose] -``` - -- `--include-text` is off by default. When set, text-only messages are recorded in - the sidecar with an empty `files[]` so captions and replies keep their - surrounding conversation. They never create a folder and never download - anything. It **is** part of the stored filter set — it changes what a post - contains — so toggling it requires `--reset-state`. -- `--verbose` raises the log level for *this tool only*. The `telethon` logger is - pinned at `WARNING` unconditionally, because its request logging carries - `api_id`, `phone_number` and `phone_code_hash`. There is deliberately no - `--debug-telethon`. -- A link preview is not media: a message containing a URL is not treated as an - attachment, so preview thumbnails are never downloaded. - -## Flood waits have no ceiling by default - -Telegram throttles by telling the client to wait, and those waits escalate from -seconds to hours if you keep pushing. By default `tg-export` **sleeps as long as -Telegram demands**, so an unattended export completes rather than failing fast. - -Every wait is logged at `WARNING` with its duration **and its computed wake -time**: - -``` -2026-08-11 07:42:11 WARNING flood wait 3600s - sleeping until 2026-08-11T08:42:11+00:00 -``` - -A long sleep is a sleep, not a hang. Read the log before assuming otherwise. Pass -`--max-flood-wait SECONDS` to exit cleanly and resumably (code 6) instead. - -## Why downloads are sequential - -Telethon does not parallelize a single transfer, and running several at once -mainly accelerates flood-wait escalation — a 7-second wait becomes a 4-hour -lockout. There is deliberately no `--workers` flag, and an AST test in the suite -fails if `gather`, `create_task`, `to_thread` or a thread pool ever appears in -`src/`. - -## Disk space - -`--dry-run` reports remaining posts, files and bytes by media type against free -space, and exits 2 if it will not fit. - -**That verdict is advisory.** The enforcement is a per-file check before each -download, comparing free space against *this file's* size plus the `--min-free` -reserve. So a `SHORT` verdict does not mean the run is unsafe — it means the run -will stop partway with a clean, resumable cursor. - -Photo totals are labelled approximate because a photo's declared `.size` -describes the size variant Telethon selected. Documents, video and audio carry an -exact size. Media reporting no size at all is downloaded anyway and counted on its -own `unknown size` line rather than as zero, so the estimate never hides an -unbounded error. - -`--dry-run` reads the cursor and measures from where a real run would start, so -`tg-export --dry-run && tg-export` works on a partially completed export. It takes -no export lock, creates no directories, and mutates nothing. It does open the -session file, so running it alongside an export in progress needs its own -`--session` — see [Concurrent runs](#concurrent-runs). - -## Concurrent runs - -One run per export root. A second run against the same `--out` exits 3 and names -the holder's pid, host and start time. `flock` releases on process death, `kill -9` -included, so there is no stale lock to clean up. - -Exporting two *different* groups at the same time works, but each needs its own -`--session` as well as its own `--out`: the session file is a SQLite database and -is locked too, because two Telethon clients sharing one session is a corruption -hazard Telethon itself warns about. - -`--dry-run` takes no *export* lock, so it never contends for an export root. It -does open the session file and therefore takes the session lock: to run it -alongside an export in progress, give it its own `--session`. - -Both locks are `flock`-based. On NFS and some network filesystems `flock` is -advisory-only or silently per-client, so single-instance enforcement is not -guaranteed there — keep the session file and the export root on local storage. - -## Exit codes - -| Code | Meaning | -|---|---| -| 0 | success | -| 1 | unexpected error | -| 2 | `--dry-run` says it will not fit, **or** a bad/missing argument — argparse exits 2 too | -| 3 | disk exhausted, or another run holds the lock | -| 4 | session invalidated — re-login | -| 5 | lost access to the group | -| 6 | flood wait exceeded `--max-flood-wait` (only when you set it) | -| 7 | filter or chat mismatch against the existing state file | -| 130 | interrupted — re-run to resume | - -## Development +A complete, tested (214 offline tests) Python/Telethon exporter with per-post +album folders, a `messages.jsonl` sidecar, resumable cursor semantics and a +dry-run size estimator lives at commit `3286ee2`: ```bash -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt -e ".[dev]" -.venv/bin/python -m pytest # 214 tests, ~3 s +git show 3286ee2:README.md # its documentation +git checkout 3286ee2 -- src tests pyproject.toml requirements.txt # resurrect it ``` -The whole suite runs offline against synthetic message stubs — no credentials, no -network. Note that `messages.jsonl` contains message text and sender names from -other people; the export tree is gitignored and the runtime check refuses an -un-ignored path inside a repo, but check `git status` before any commit. +It is unmaintained and pins Telethon `<2`; expect bit-rot. diff --git a/plans/260811-1411-telegram-group-media-exporter/phase-01-verification-spike.md b/plans/260811-1411-telegram-group-media-exporter/phase-01-verification-spike.md deleted file mode 100644 index 7904a92..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/phase-01-verification-spike.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -phase: 1 -title: "Verification Spike" -status: partial -priority: P1 -dependencies: [] -effort: "~45 min" ---- - -# Phase 1: Verification Spike - -## Overview - -Throwaway script settling the unverified items in the research confidence ledger against the live API, before production code depends on them. The docs page truncated mid-signature; guessing means silent resume bugs discovered after a 40 GB run. - -**Ordering is load-bearing in this phase.** `.gitignore` is committed *first*, before anything can create a credential. The spike uses its own session path so phase 2's "first production login" remains a real, verifiable event. - -## Requirements - -- Functional: answer all six probes; record findings; amend contradicted phases -- Non-functional: throwaway code, not shipped; no credential exists before `.gitignore` is committed; one test download only - -## Probes - -### P1 — `offset_id` direction under `reverse=True` (highest risk) - -Semantics invert between sweep directions. Wrong here means resume silently skips or re-downloads everything. - -```python -ids = [m.id async for m in client.iter_messages(entity, limit=5)] # newest-first -mid = sorted(ids)[2] -fwd = [m.id async for m in client.iter_messages(entity, reverse=True, offset_id=mid, limit=3)] -assert all(i > mid for i in fwd), f"offset_id not exclusive-lower-bound under reverse: {fwd} vs {mid}" -``` - -Record whether `mid` itself is included. **Probe `min_id=mid` the same way** — it is documented as an exclusive lower bound and may be the more robust parameter regardless. The result decides whether the cursor stores "last handled id" or "next id to fetch". - -### P2 — 2FA / login flow - -Record whether `SessionPasswordNeededError` is raised and whether `client.start(phone=..., password=)` handles it. Confirm a second run against the same session needs no interaction. - -2FA password is `getpass`-only — never argv, never env, never a file. (This supersedes any earlier wording permitting env.) - -### P3 — `message.file` attribute surface - -**Open question, not a confirmation.** For one photo, one video, one document, one voice note, one text-only message, and anything unusual in reach (poll, webpage preview, expiring media): - -```python -f = msg.file -print(msg.id, msg.grouped_id, type(msg.media).__name__, - getattr(f, 'name', None), getattr(f, 'ext', None), - getattr(f, 'size', None), getattr(f, 'mime_type', None)) -``` - -Answer: **for which media kinds is `.size` absent?** Do not assume it is always present — phases 3, 5, and 6 each need a rule for `None`. - -### P4 — `download_media` return value and file-object support - -- Download one photo and one document with `file=`; record the exact return type/value. Phase 5 uses this as the source of truth for `os.replace`. -- Test whether `download_media(msg, file=)` works across media branches. Phase 5 needs to own the fd to `fsync` it before rename. If it does not work, the fallback is download-to-path then re-open and fsync. - -### P5 — photo `.size` vs bytes on disk (NEW — ledger item the original probe set missed) - -```python -p = await client.download_media(photo_msg, file=str(tmp)) -print("declared", photo_msg.file.size, "actual", os.path.getsize(p), - "delta", os.path.getsize(p) - photo_msg.file.size) -``` - -Phase 6 documents photo sizes as approximate while phase 5 originally used exact equality as a hard gate — that contradiction was the poison pill this probe closes. The resolved design is answer-independent; the answer only improves report labelling and validator tuning. - -### P6 — album adjacency across a pagination boundary - -The original probe capped `limit ≤ 5`, which cannot cross Telethon's ~100-message page boundary and so could not test the thing it existed to test. **Cap downloads at 1; do not cap history reads.** - -```python -rows = [(m.id, m.grouped_id) async for m in - client.iter_messages(entity, reverse=True, offset_id=start, limit=350)] -assert len(rows) >= 250, "window too small to cross a page boundary — probe is vacuous" -ids = [i for i, _ in rows] -assert ids == sorted(ids), "sweep is not strictly ascending" -runs = [k for k, _ in itertools.groupby(rows, key=lambda r: r[1])] -gids = [g for g in runs if g is not None] -assert len(gids) == len(set(gids)), f"a grouped_id occupies >1 run — albums DO split: {gids}" -``` - -Coverage is probabilistic (3+ pages, several albums). The deterministic guarantee is phase 3's runtime tripwire; this probe is the cheap prior. - -## Related Code Files - -- Create: `.gitignore` (final version, committed first), `requirements.txt` -- Create: `spike/probe_telethon_api.py` (throwaway, gitignored) -- Create: `plans/reports/spike-260811-1411-telethon-api-verification-report.md` - -## Implementation Steps - -1. Write the **final** `.gitignore` and commit it. Nothing else exists yet: - ``` - .venv/ - *.session - *.session-journal - *.session-wal - *.session-shm - spike/ - exports/ - .export-state.json - .export.lock - __pycache__/ - *.egg-info/ - .env - ``` -2. **Gate — must pass before step 3:** `git check-ignore -q .venv spike/ && echo OK` (exit 0) -3. `python3 -m venv .venv && .venv/bin/pip install "telethon>=1.41,<2"`, then `.venv/bin/pip freeze > requirements.txt` — this records the exact version the probes verified -4. `umask 077; mkdir spike` -5. Obtain `api_id`/`api_hash` from my.telegram.org; export as `TG_API_ID`/`TG_API_HASH` (never hardcode) -6. Write and run `spike/probe_telethon_api.py` (P1–P6), session at `spike/probe.session` -7. Write the findings report, marking each ledger item CONFIRMED or CORRECTED, and recording `telethon.__version__` -8. **If any finding contradicts the plan, amend the affected phase file before proceeding** - -## Success Criteria - -- [ ] `.gitignore` is committed before any venv, session, or credential exists — verified by `git log` -- [ ] `git check-ignore -q spike/ .venv` exits 0 before step 3 runs -- [ ] P1: `offset_id` inclusivity/direction under `reverse=True` recorded, and `min_id` compared -- [ ] P2: 2FA flow documented; second run confirmed non-interactive -- [ ] P3: `.size` availability tabulated per media kind, including any kind where it is absent -- [ ] P4: `download_media` return value recorded; file-object support determined -- [ ] P5: photo declared-vs-actual size delta recorded -- [ ] P6: ≥250 messages swept, ascending order asserted, no `grouped_id` occupying two runs -- [ ] `requirements.txt` records the verified Telethon version -- [ ] Findings report written; contradicted phases amended - -## Risk Assessment - -- **Wrong `offset_id` assumption reaches production** → the point of this phase; assertions must run, not be eyeballed. -- **Credential created before `.gitignore`** → step 1 precedes step 3, with an explicit gate between them. -- **Probe capped too small to test what it claims** → P6 requires ≥250 messages and asserts it. History reads are cheap (~4 requests); only *downloads* are meaningfully rate-limited. -- **Spike session left live** → deleting `spike/probe.session` does **not** revoke server-side authorization. Phase 7 cleanup must terminate it in Telegram → Settings → Devices. -- **Credentials leaked into the spike script** → env vars only; `spike/` gitignored from commit one. diff --git a/plans/260811-1411-telegram-group-media-exporter/phase-02-skeleton-and-session-auth.md b/plans/260811-1411-telegram-group-media-exporter/phase-02-skeleton-and-session-auth.md deleted file mode 100644 index 50f97a2..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/phase-02-skeleton-and-session-auth.md +++ /dev/null @@ -1,215 +0,0 @@ ---- -phase: 2 -title: "Skeleton and Session Auth" -status: complete -priority: P1 -dependencies: [1] -effort: "~2h" ---- - -# Phase 2: Skeleton and Session Auth - -## Overview - -Package skeleton, the authenticated Telethon client, the flood-wait primitives every network call routes through, and the process-level safety rails: umask, logging policy, git-safety assertion, and the single-instance lock. - -## Requirements - -- Functional: one-time interactive login; later runs non-interactive; `tg-export --help` after `pip install -e .` -- Non-functional: no file is ever group/world-readable at any instant; no credential in argv, env-where-forbidden, or logs; two concurrent runs cannot corrupt each other - -## Architecture - -### Process rails — `cli.main()`, before anything else - -```python -os.umask(0o077) # FIRST statement. Covers .session and every sqlite sibling, - # export tree, messages.jsonl, .part, state, lock — one line, no per-file chmod. -``` - -Verified empirically: `sqlite3.connect()` on a fresh path yields `0644` under a default umask, and the auth key is written *during* login — so a post-hoc `chmod` is provably too late. With `umask(0o077)` set first, the db and its `-journal` are both `0600`. - -### Session path - -```python -SUFFIX = ".session" - -def prepare_session_path(p: Path) -> Path: - # Telethon appends '.session' when absent — chmod-ing the un-suffixed path protects nothing - p = p if p.name.endswith(SUFFIX) else p.with_name(p.name + SUFFIX) - p.parent.mkdir(parents=True, exist_ok=True) - try: - os.close(os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)) - except FileExistsError: - os.chmod(p, 0o600) - return p - -def assert_session_private(p: Path) -> None: # this is the test, not a checkbox - for f in p.parent.glob(p.name + "*"): - if stat.S_IMODE(f.stat().st_mode) & 0o077: - raise SystemExit(f"insecure mode on {f}: {oct(f.stat().st_mode)}") -``` - -Default path `${XDG_STATE_HOME:-~/.local/state}/tg-export/.session` — **outside the repo**, so the safe location is the default and the git check only bites deliberate choices. - -### Git safety — robust to `--out` / `--session` moving the targets - -Adding `.gitignore` patterns cannot help when the operator can point a flag anywhere. Assert at runtime: - -```python -def assert_not_committable(p: Path) -> None: - """No-op outside a git work tree. rc 0=ignored, 1=NOT ignored, 128=not a repo.""" - try: - rc = subprocess.run(["git", "check-ignore", "-q", str(p)], - cwd=p.parent, capture_output=True).returncode - except FileNotFoundError: - return - if rc == 1: - raise SystemExit(f"refusing: {p} is inside a git repo and is not gitignored.\n" - f"add it to .gitignore, or choose a path outside the repo.") -``` - -Called on the session path and export root at startup. Exit codes verified empirically, including on nonexistent paths. - -### Single-instance lock - -```python -@contextmanager -def exclusive(path: Path): - """flock releases on process death, kill -9 included — no stale-lock cleanup to get wrong.""" - path.parent.mkdir(parents=True, exist_ok=True) - f = open(path, "a+") - try: - fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - f.seek(0) - raise SystemExit(f"busy: another tg-export holds {path} [{f.read().strip() or '?'}]") - f.seek(0); f.truncate() - f.write(f"pid={os.getpid()} host={socket.gethostname()} started={now_iso()}\n"); f.flush() - try: - yield - finally: - f.close() -``` - -Two call sites: `/.export.lock` (covers `.part` files, state, sidecar) and `.lock` (two Telethon clients on one SQLite session is what Telethon itself warns about). Per-root, not global — exporting two different groups concurrently is legitimate. - -Contention: fail fast, exit 3, no wait, no `--force`. The message names pid/host/start-time, so "did it hang?" gets an answer instead of a second process. **Dry-run takes no lock** — it writes nothing. - -### Logging policy - -```python -def configure_logging(verbose: bool) -> None: - logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - logging.getLogger("telegram_exporter").setLevel(logging.DEBUG if verbose else logging.INFO) - logging.getLogger("telethon").setLevel(logging.WARNING) # unconditional floor - logging.getLogger("asyncio").setLevel(logging.WARNING) -``` - -Never touch the root level. `--verbose` means *our* logger only — telethon's request logging carries `api_id`, `phone_number`, and `phone_code_hash`. No `--debug-telethon` flag. - -Note: CPython does not print local variables in tracebacks, so no exception-scrubbing machinery is needed. Just don't install `rich` or `better-exceptions`. - -### Credential policy — one table, referenced by every other doc - -| Credential | Source | Why | -|---|---|---| -| `api_id`, `api_hash` | env `TG_API_ID` / `TG_API_HASH` only | needed every run; revocable at my.telegram.org | -| phone | prompt; `TG_PHONE` optional | PII, not a secret | -| login code | **prompt only** | single-use, 5-minute TTL | -| 2FA password | **`getpass` only** — no env, no file, no flag | once per session lifetime; prompting costs nothing | - -`.env` **may** contain `TG_API_ID`, `TG_API_HASH`, `TG_PHONE`. It **must not** contain the 2FA password, login code, or session data. The app does **not** read `.env` — no `python-dotenv` to read a file the shell already handles (`set -a; . ./.env; set +a`). - -### Flood-wait primitives — two, not one - -The single-value wrapper cannot protect the sweep: `iter_messages` is consumed with `async for`, and Telethon raises `FloodWaitError` on the *next page fetch*, deep inside the loop — so it escapes a wrapper that only guards construction, and re-invoking the factory would restart the sweep from `after_id`. - -```python -def _flood_sleep(e, max_wait_s, where=""): - """max_wait_s is None by default -> sleep however long Telegram demands. - Always log the WAKE TIME, not just the duration: an unattended 4-hour sleep must - read as a sleep, not as a hang. This is the mitigation for the no-ceiling default.""" - if max_wait_s is not None and e.seconds > max_wait_s: - raise Abort(f"flood wait {e.seconds}s exceeds --max-flood-wait {max_wait_s}s", 6) - wake = datetime.now(UTC) + timedelta(seconds=e.seconds) - log.warning("flood wait %ss%s — sleeping until %s", e.seconds, where, wake.isoformat()) - return asyncio.sleep(e.seconds + random.uniform(1.0, 3.0)) - -async def with_flood_retry(make_coro, *, max_wait_s: int | None = None): - """Download path. Never tight-retry — retries during an active wait are fresh violations.""" - while True: - try: - return await make_coro() - except FloodWaitError as e: - await _flood_sleep(e, max_wait_s) - -async def aiter_with_flood_retry(make_agen, *, start_id, max_wait_s: int | None = None): - """Sweep path. Catches mid-iteration waits and rebuilds the generator from the - last id actually yielded, so no message is skipped or replayed.""" - last = start_id - while True: - try: - async for item in make_agen(last): - last = item.id - yield item - return - except FloodWaitError as e: - await _flood_sleep(e, max_wait_s, where=f" mid-sweep after id {last}") -``` - -`max_wait_s=None` is the default per the validation decision: unattended completion is preferred over failing fast. The wake-time log is what keeps that choice diagnosable. - -Also set `client.flood_sleep_threshold = 120` so routine short waits are absorbed inside Telethon. - -## Related Code Files - -- Create: `pyproject.toml`, `README.md` -- Create: `src/telegram_exporter/{__init__,cli,session}.py` -- Create: `tests/{__init__,test_logging,test_lock}.py` -- Modify: `.gitignore` (verify coverage; phase 1 created it) - -## Implementation Steps - -1. `pyproject.toml` — `telethon>=1.41,<2`, `[project.scripts] tg-export = "telegram_exporter.cli:main"`, hatchling backend, `dev = ["pytest>=8"]`. Install with `pip install -r requirements.txt -e .` so the phase-1-verified version is what's used. -2. **Verify** (do not recreate) that `.gitignore` covers the production session path and export root. -3. `cli.py`: `os.umask(0o077)` as the first statement, then `configure_logging`, argparse, `assert_not_committable`, locks, dispatch. -4. `session.py`: credential loading with an actionable error pointing at my.telegram.org; `prepare_session_path`; client construction; login (phone/code prompts, 2FA via `getpass`); `assert_session_private` after connect; async context manager guaranteeing `disconnect()` — SQLite state is not flushed without it. -5. `session.py`: both flood primitives and the `Abort` exception type. -6. Entity resolution for `--group`: numeric id, `@username`, `t.me` link. Use `telethon.utils.get_peer_id(entity)` as the canonical int for the export root. Clear message when the account is not a member. -7. Full flag surface (later phases wire their own): - ``` - tg-export --group --out DIR - [--dry-run] [--since YYYY-MM-DD] [--until YYYY-MM-DD] - [--types photo,video,document,audio,voice] [--max-size 100MB] - [--include-text] [--limit N] [--session PATH] [--reset-state] - [--min-free 2GiB] [--max-flood-wait SECONDS] [--verbose] - ``` - `--max-flood-wait` is **unset by default — the tool sleeps through any flood wait, however long** (validation decision). Passing a number imposes a ceiling above which the run exits cleanly with a resumable cursor (exit 6). - - - -8. Tests: `test_logging.py` (telethon logger stays ≥ WARNING even with `verbose=True`), `test_lock.py` (subprocess holds the lock; second acquire exits 3). - -## Success Criteria - -- [ ] `pip install -r requirements.txt -e ".[dev]"` succeeds on Python 3.12 / aarch64 -- [ ] `tg-export --help` prints the full flag surface -- [ ] Production session path does not exist before first login (asserted) — phase 1's spike session is separate -- [ ] After login: session and every sibling matching `session*` are mode `0600` -- [ ] Second run is non-interactive: `tg-export --group G --limit 0 prev, f"sweep not strictly ascending: {msg.id} after {prev}" - prev = msg.id - if buf and (msg.grouped_id is None or msg.grouped_id != buf[0].grouped_id): - yield _close(buf, closed, filters); buf = [] - if msg.grouped_id is None: - yield _close([msg], closed, filters) - else: - buf.append(msg) - if buf: - yield _close(buf, closed, filters) - -def _close(buf, closed, filters) -> Post: - gid = buf[0].grouped_id - if gid is not None: - if gid in closed: # album-split tripwire - raise AlbumSplitError( - f"grouped_id {gid} reopened at msg {buf[0].id} after being closed — " - f"album split across a non-adjacent boundary. Do not trust this export.") - closed.add(gid) - return Post(post_id=buf[0].id, - messages=[m for m in buf if _keep(m, filters)], - max_message_id=buf[-1].id) -``` - -**Empty posts are yielded, not skipped.** This is the resolved answer to a decision the plan previously deferred. Skipping them stalls the cursor: a `--types video` run over a photo-heavy tail would advance `cursor_id` only at videos, so every resume re-sweeps tens of thousands of messages to download nothing. The "phantom post count" objection is a *counting* bug, fixed with one line in each consumer (`if not post.messages: continue` before counting), not a design objection. - -**Album-split tripwire.** Consecutiveness of `grouped_id` members in a sweep is assumed by the whole design and was never verified by any source. The tripwire converts a silent split — two folders for one album, no exception, no size mismatch, dedupe reporting complete — into a loud failure. Memory is one int per album (~6 MB at 100k albums); the "flat memory" criterion is restated honestly as O(albums) rather than pretending it is O(1). - -Tracking only `last_flushed_gid` was rejected: it catches `A B A` but misses `A B C A`, saving 6 MB and losing most of the coverage. - -### `media_size` — one rule, three consumers (DRY) - -```python -def media_size(msg) -> int | None: - """Single source of truth. Phase 1 P3 determines which kinds return None.""" -``` - -| Consumer | Rule for `None` | -|---|---| -| `--max-size` filter | `size is None or size <= limit` → **include**, WARN once. Silent drops violate the no-silent-gaps ethos; unknown-size media is pathological and near-always small. | -| download validator | accept (nothing to check) | -| estimator | **not** `or 0` — counted in a separate `unknown size: N files (not counted)` line. An unbounded estimate error must be visible. | -| disk guard | treated as 0, absorbed by the reserve | - -Comparing `None <= limit` unguarded raises `TypeError` inside the generator and kills a 20-hour export — that is what this helper exists to prevent. - -### Filters - -| Flag | Predicate | -|---|---| -| `--types` | media kind ∈ set; kind from `msg.media` type + document attributes | -| `--since` / `--until` | `msg.date` in range, UTC-aware on both sides | -| `--max-size` | via `media_size` per the table above | -| `--include-text` | when set, keep text-only messages too (sidecar context); default off | - - - -**`--include-text`** (validation decision). Default off: messages with no media are dropped. When set, text-only messages survive `_keep` and reach the sidecar with an empty `files[]`, so captions and replies have surrounding conversation. They never create a post folder and never reach `download_one`. - -It **is** part of the stored filter set — it changes what a post contains, so running without it and then with it against the same cursor would silently skip every text record. - -`--limit N` counts **posts with files** (counting all posts would let it terminate instantly inside a filtered region, and would now also be skewed by text-only posts) and is *not* a filter — it changes when we stop, not what a post contains, so it stays out of the state file's filter set. - -## Related Code Files - -- Create: `src/telegram_exporter/traversal.py` -- Create: `tests/test_traversal.py` -- Modify: `src/telegram_exporter/cli.py` (wire `--types`, `--since`, `--until`, `--max-size`, `--limit`) - -## Implementation Steps - -1. Define `Post`, `Filters`, `AlbumSplitError`; parse CLI values into `Filters` (size-string and ISO-date parsers) -2. Implement media-kind classification and `media_size` -3. Implement `iter_posts` per the sketch, honoring phase 1 P1's confirmed `offset_id`/`min_id` semantics -4. Consume the sweep through `aiter_with_flood_retry` — not the single-value wrapper -5. Unit test with **synthetic message stubs** (no network): objects with `id`, `grouped_id`, `date`, `file` -6. Test the tripwire explicitly: a synthetic stream where a `grouped_id` reappears must raise - -## Success Criteria - -- [ ] A 3-photo album yields exactly one `Post` with 3 messages -- [ ] Two adjacent albums yield two distinct `Post`s, not one merged -- [ ] A standalone message between two albums yields its own `Post` -- [ ] An album at the very end of history is flushed -- [ ] `post_id` and `max_message_id` are unchanged when a filter removes some members — asserted -- [ ] A fully-filtered group still yields a `Post` (with `messages == []`) so the cursor advances -- [ ] Text-only messages contribute no files, and are dropped entirely unless `--include-text` is set -- [ ] With `--include-text`, a text-only message survives into `Post.messages` but creates no folder and no download -- [ ] `--limit N` counts posts with files, unaffected by `--include-text` -- [ ] A synthetic stream reopening a closed `grouped_id` raises `AlbumSplitError` -- [ ] A non-ascending synthetic stream trips the assertion -- [ ] `media_size` returning `None` does not raise in any of the three consumers -- [ ] Filters compose: `--types video --since 2026-01-01 --max-size 50MB` -- [ ] `--limit 5` stops after 5 posts *with files* -- [ ] Album buffer never exceeds 20 entries (asserted in code) - -## Risk Assessment - -- **Identity computed post-filter** → duplicate folders and re-downloads across filter sets. Invariant 1, plus phase 4's message-id filenames; both explicitly tested. -- **Album split across a pagination boundary** → silent two-folder corruption. Phase 1 P6 is the prior; the tripwire is the guarantee. Phase 3's earlier dismissal of this risk was an unsourced assertion guarded by a test that structurally could not fail. -- **Cursor stalls across filtered regions** → empty posts are yielded. Decided here, not deferred to implementation. -- **`None` size crashes the sweep** → `media_size` plus the stated rule per consumer. -- **Flood wait mid-sweep** → `aiter_with_flood_retry` resumes from the last yielded id. -- **Timezone-naive comparison** → `msg.date` is UTC-aware; parse `--since`/`--until` as UTC-aware or the comparison raises. diff --git a/plans/260811-1411-telegram-group-media-exporter/phase-04-naming-and-path-safety.md b/plans/260811-1411-telegram-group-media-exporter/phase-04-naming-and-path-safety.md deleted file mode 100644 index a487fcb..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/phase-04-naming-and-path-safety.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -phase: 4 -title: "Naming and Path Safety" -status: complete -priority: P1 -dependencies: [2] -effort: "~1h" ---- - -# Phase 4: Naming and Path Safety - -## Overview - -`paths.py` — **all** path construction, from the export root down to the filename leaf. Pure functions, no network, fully unit-testable, which is why it is its own module rather than a helper inside `downloader.py`. - -Renamed from `naming.py`: the module owns directory construction too. That rename is what turns the group-title fix from a patch into a structural property. - -## Requirements - -- Functional: derive the export root, post dir, and filename; sanitize; guarantee uniqueness within a post folder -- Non-functional: pure; deterministic across runs *and across filter sets* (dedupe depends on it) - -## The governing invariant - -> **Exactly one untrusted string ever becomes a path component: the filename leaf.** - -Every other component is an `int`, enforced by a type check. This makes `safe_join`'s "post_dir is trusted" precondition true *by type* rather than by assumption. - -Two untrusted strings were previously in play: - -1. `DocumentAttributeFilename` — attacker-controlled, supplied by arbitrary group members. `../../../.ssh/authorized_keys` is a cheap, real attack. Always was in scope. -2. **The group title** — server-supplied and editable by any group admin, used to build `exports//`. This was *not* in scope, and `safe_join` could never have caught it because the escape happens in the parent component, before the join. - -## Architecture - -```python -def export_root(out_dir: Path, peer_id: int) -> Path: - """peer_id from telethon.utils.get_peer_id(entity). int -> traversal impossible.""" - if not isinstance(peer_id, int): - raise TypeError(peer_id) - return (out_dir / f"g{peer_id}").resolve() # g-1001234567890 - -def post_dir(root: Path, post_id: int) -> Path: - if not isinstance(post_id, int): - raise TypeError(post_id) - return root / str(post_id) - -def derive_filename(msg) -> str: # note: no index parameter - return f"{msg.id}_{sanitize(stem_for(msg), fallback=kind(msg))}{ext_for(msg)}" - -def sanitize(name: str, *, fallback: str, max_bytes: int = 200) -> str: - """Reduce an untrusted string to a safe single path component.""" - -def safe_join(post_dir: Path, filename: str) -> Path: - """Join and assert containment. Raises on escape — never silently repairs.""" -``` - -### Why the directory is the chat id, not the title - -The `g` prefix avoids a leading `-` (a CLI foot-gun, not a security issue). The human-readable name is written once to `/title.txt` and printed in the run header — stored as data, never as a path. - -Rejected alternatives: a sanitized title slug needs a second sanitizer with different rules for directories, a fallback for the empty case, and a collision story — more code and more risk. A `-` hybrid is readable but strands the state file and the downloaded tree in the old directory when the group is renamed mid-export. - -One move fixes traversal, empty-slug collapse into the parent, cross-group collision, *and* rename-orphaning — and it is a net code deletion. The cost is that `ls exports/` shows numbers. - -### Why filenames key on `message_id`, not a positional index - -One Telegram message carries at most one media, so `message_id` cannot collide inside a post folder. Uniqueness becomes structural rather than positional. - -A positional index — even one computed pre-filter — is unstable under *history mutation*: if a member message is deleted between runs the group shrinks and every later index shifts, silently orphaning already-downloaded files. It would also force `Post` to carry the full pre-filter member list purely for naming, reintroducing the coupling Invariant 1 exists to remove. - -Accepted cost: `1042/1042_photo.jpg` is redundant for standalone posts. Worth it — files stay self-identifying when moved out of their folder, and `grep 1043 messages.jsonl` joins sidecar to disk with no index arithmetic. - -### Sanitization rules - -Derivation order: `msg.file.name` when present → `msg.file.ext`/mime with a kind-based stem (`photo`, `video`, `voice` carry no filename) → `.bin`. - -- Take `Path(name).name` only — discards every directory component, including `..` -- Strip NUL and control chars (`< 0x20`); replace path separators and reserved chars -- Strip leading dots and dashes (hidden files, argument-looking names) -- Collapse whitespace runs -- Truncate to `max_bytes` counted in **UTF-8 bytes, not characters** — ext4's limit is 255 bytes and unicode names overflow well before 255 chars. Preserve the extension. -- Empty or fully-stripped result → `fallback` - -Final defense in `safe_join`: - -```python -resolved = (post_dir / filename).resolve() -if not resolved.is_relative_to(post_dir.resolve()): - raise ValueError(f"path escape attempt: {filename!r}") -``` - -Sanitization should make this unreachable; reaching it means sanitization has a hole — hence raise, never repair. - -## Related Code Files - -- Create: `src/telegram_exporter/paths.py` -- Create: `tests/test_paths.py` - -## Implementation Steps - -1. Implement `sanitize` per the rules above -2. Implement `derive_filename` using phase 1 P3's findings on `message.file` availability -3. Implement `export_root` / `post_dir` with `isinstance` guards -4. Implement `safe_join` with the containment assertion -5. Write the table-driven tests below -6. `pytest tests/test_paths.py` must pass before phase 5 consumes this module - -## Test Table - -| Input | Expected | -|---|---| -| `../../../etc/passwd` | `passwd` | -| `a/b/c.jpg` | `c.jpg` | -| `..` / `.` / `""` | fallback | -| `\x00evil.jpg` | `evil.jpg` | -| `....jpg` | leading dots stripped | -| `-rf.jpg` | leading dash stripped | -| 300-char unicode stem | ≤200 UTF-8 bytes, extension intact | -| same msg, two different filter sets | **identical** filename | -| two media in one post | distinct names via distinct message ids | -| `safe_join(dir, "../x")` | raises `ValueError` | -| `export_root(out, "..")` | raises `TypeError` (not an int) | -| `export_root(out, "../../etc")` | raises `TypeError` | -| `post_dir(root, "1042; rm -rf")` | raises `TypeError` | - -## Success Criteria - -- [ ] Every row of the test table passes -- [ ] No sanitized name, joined to a post dir, resolves outside it -- [ ] `export_root` and `post_dir` reject every non-`int` input -- [ ] A group titled `..`, `a/b`, or `""` cannot influence the output path — there is no code path from title to path -- [ ] Truncation is byte-counted; a 300-char unicode name yields a filename ext4 accepts, extension intact -- [ ] `derive_filename` is deterministic across runs **and across filter sets** -- [ ] Tests run offline with no Telethon client - -## Risk Assessment - -- **Path traversal via filename** → sanitize + `safe_join` assertion + table tests. -- **Path traversal via group title** → structurally impossible: no code path exists from title to path component. -- **Filter-dependent filenames** → keyed on `message_id`; asserted by the two-filter-sets test row. -- **Index instability under message deletion** → no positional index exists. -- **Char-based truncation on unicode** → `OSError: File name too long` mid-export. Byte-counted. -- **Non-deterministic fallbacks** (e.g. a timestamp) → dedupe breaks and every re-run re-downloads. Fallbacks derive only from message id and kind. -- **Windows reserved names** (`CON`, `PRN`) → not applicable on Linux; deliberately skipped. diff --git a/plans/260811-1411-telegram-group-media-exporter/phase-05-downloader-state-and-resume.md b/plans/260811-1411-telegram-group-media-exporter/phase-05-downloader-state-and-resume.md deleted file mode 100644 index f90fe3b..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/phase-05-downloader-state-and-resume.md +++ /dev/null @@ -1,268 +0,0 @@ ---- -phase: 5 -title: "Downloader State and Resume" -status: complete -priority: P1 -dependencies: [3, 4] -effort: "~4h" ---- - -# Phase 5: Downloader State and Resume - -## Overview - -The download path: fetch media sequentially, write durably, dedupe, persist the resume cursor and the `messages.jsonl` sidecar. This is where correctness under interruption is won or lost. - -Governing principle: **the filesystem is the state; the state file is a hint; the sidecar is the log.** Everything below follows from it. - -## Requirements - -- Functional: download each `Post`'s media into its folder; skip completed files; resume after any interruption; record every file and every failure -- Non-functional: concurrency 1; no partial file ever presented as complete; the cursor never points past an incomplete post - -## Architecture - -### Durability and dedupe are one decision, not two - -Telethon has no byte-level resume — `download_media` writes from offset 0 every call. So resumability lives at *message* granularity, and completeness is proven by **arrival at the target path**: - -``` -per file: write -> flush -> fsync(fd) -> os.replace(tmp, target) -per post: fsync(post_dir) -> sidecar.append + fsync -> state.commit (tmp -> fsync -> replace -> fsync(root)) -``` - -With `fsync` before `os.replace`, a file at the target path can only have arrived fully downloaded. Therefore: - -```python -if target.exists(): - if target.stat().st_size > 0: - return SKIPPED - target.unlink() # zero-byte junk from ENOSPC; re-fetch -``` - -**Existence, not size equality.** Size equality would only add detection of *external* corruption — a fabricated threat for a personal archive — and it is precisely what created the poison pill: phase 6 documents photo `.size` as the size *variant* Telethon selects, i.e. approximate, while an exact-equality gate would delete every completed photo on resume and then raise. The fsync is what licenses the simpler predicate. - -Stray `.part` files are swept at startup, **after** the lock is acquired — the sweep is the destructive operation, so locking after it locks after the damage. - -### Validator: advisory, never raises - -```python -actual, declared = tmp.stat().st_size, media_size(msg) -if declared is None or actual == declared: accept -elif kind(msg) == "photo": accept, log DEBUG # variant size, expected -else: RETRY; on exhaustion -> FAILED -``` - -Documents, video, and audio carry an exact `document.size`, so a mismatch there is real truncation. Photos are the only approximate case. This design is correct **whether or not** phase 1 P5 finds photo sizes exact. - -### Error taxonomy — the failure paths get the same rigor as the happy path - -Three buckets. `FloodWaitError` never appears here: it is owned by `with_flood_retry`, because retries during an active wait are counted as fresh violations. - -```python -TRANSIENT_ERRNO = {EAGAIN, EINTR, EIO, ETIMEDOUT, ECONNRESET} -FATAL_ERRNO = {ENOSPC, EDQUOT, EROFS, EACCES} -MAX_ATTEMPTS, BACKOFF = 3, lambda n: 5 * 2 ** (n - 1) # 5, 10, 20 s - -async def download_one(client, entity, post_dir, msg) -> Result: - target = safe_join(post_dir, derive_filename(msg)) - if target.exists(): - if target.stat().st_size > 0: return SKIPPED - target.unlink() - check_disk(msg) # before the first byte - tmp, refreshed = target.with_name(target.name + ".part"), False - - for attempt in range(1, MAX_ATTEMPTS + 1): - try: - with open(tmp, "wb") as f: - await with_flood_retry(lambda: client.download_media(msg, file=f)) - f.flush(); os.fsync(f.fileno()) - if validate(tmp, msg) is RETRY: - tmp.unlink(missing_ok=True); await sleep(BACKOFF(attempt)); continue - os.replace(tmp, target) - return DOWNLOADED(target) - - # ---- SKIP: this message is unavailable; the run continues ---- - except FileReferenceExpiredError: - tmp.unlink(missing_ok=True) - if refreshed: return FAILED(msg, "file reference expired") - msg = await client.get_messages(entity, ids=msg.id); refreshed = True - if msg is None or msg.media is None: return FAILED(msg, "message deleted") - continue - except (MediaEmptyError, FileIdInvalidError, FilerefUpgradeNeededError): - tmp.unlink(missing_ok=True); return FAILED(msg, "media unavailable or expired") - - # ---- ABORT: the run cannot continue ---- - except (AuthKeyUnregisteredError, AuthKeyDuplicatedError, - SessionRevokedError, UserDeactivatedBanError): - tmp.unlink(missing_ok=True); raise Abort("session invalidated — re-login", 4) - except (ChannelPrivateError, ChatForbiddenError, ChannelInvalidError): - tmp.unlink(missing_ok=True); raise Abort("lost access to the group", 5) - - except OSError as e: # fatal checked FIRST - tmp.unlink(missing_ok=True) - if e.errno in FATAL_ERRNO: raise Abort(f"filesystem: {e}", 3) - if e.errno in TRANSIENT_ERRNO: await sleep(BACKOFF(attempt)); continue - raise Abort(f"unexpected OS error: {e}", 1) - - except (ConnectionError, asyncio.TimeoutError, asyncio.IncompleteReadError, - rpcerrorlist.TimedOutError, rpcerrorlist.RpcCallFailError): - tmp.unlink(missing_ok=True); await sleep(BACKOFF(attempt)); continue - - return FAILED(msg, f"exhausted {MAX_ATTEMPTS} attempts") -``` - -`FileReferenceExpiredError` handling is **not optional** — file references expire in hours, so a 20-hour run will hit it. One `get_messages` refresh is four lines and the highest-value item in this table. - -If phase 1 P4 finds `download_media` rejects a file object, fall back to download-to-path then re-open and fsync before rename. - -### The sink loop — Invariant 2 by construction - -```python -async def run_download(posts, state, sidecar, root): - try: - async for post in posts: - media, results = [m for m in post.messages if m.media], [] # results must be bound - if media: # text-only members download nothing - results = [await download_one(...) for msg in media] - fsync_dir(post_dir) - if post.messages: # may be text-only under --include-text - sidecar.append(post, results); sidecar.fsync() - if wrote_files or (monotonic() - last_commit) > 5.0: - state.commit(post.max_message_id) # only after a COMPLETE post - state.mark_completed() - except Abort as a: - log.error(a.reason); sys.exit(a.code) # cursor already correct - except KeyboardInterrupt: - log.info("interrupted — re-run the same command to resume"); sys.exit(130) - finally: - await client.disconnect() -``` - -**The cursor is never written from an abort path**, so it cannot point past an in-flight post. "Commit the cursor and exit" is deleted as a concept — it was the mechanism by which the disk guard could have violated Invariant 2. - -The 5-second throttle stops 100k filtered messages from causing 100k state writes. Losing an uncommitted sweep-only advance costs a cheap re-sweep with zero downloads. - -Exit codes: `0` ok · `1` unexpected · `2` dry-run won't fit · `3` disk or lock contention · `4` auth · `5` access · `6` flood ceiling (**only reachable when `--max-flood-wait` is explicitly set** — unset means sleep indefinitely) · `7` filter/chat mismatch · `130` interrupt. - -### Disk guard — one mechanism, per file - -```python -def check_disk(msg): - need = (media_size(msg) or 0) + cfg.min_free # default 2 GiB - if shutil.disk_usage(out_dir).free < need: - raise Abort(f"insufficient disk: need {need}, have {free}", 3) -``` - -Replaces all three previous mechanisms. `shutil.disk_usage` is one `statvfs` — microseconds, unmeasurable against multi-second downloads at concurrency 1. The every-50-files poll optimized a cost that does not exist while opening a ~100 GB blind spot: a single Telegram file reaches 2–4 GB, so up to ~100 GB could be written between two polls on a 41 GB-free host. Comparing against *this file's* size closes that structurally. The 2 GiB reserve is principled — it covers one entire unknown-size file at the non-premium upload cap. - -### State file - -```json -{ - "version": 1, - "chat_id": -1001234567890, - "chat_title": "My Group", - "filters": {"types": ["photo","video"], "since": "2026-01-01T00:00:00Z", - "until": null, "max_size": 104857600, "include_text": false}, - "cursor_id": 1042, - "completed_at": null, - "started_at": "2026-08-11T07:00:00Z", - "updated_at": "2026-08-11T07:42:11Z" -} -``` - -- **One cursor field.** `swept_to_id` vs `last_done_id` is a distinction without a difference here: the sweep is a generator driven by the sink at concurrency 1, so it can never run ahead. Two fields would invent a divergence to manage. `cursor_id` means: *every post with `max_message_id <= cursor_id` has been handled under `filters`*. -- **Filters stored verbatim; the sha256 fingerprint is deleted.** A digest is one-way, and two phases require the tool to *name* what changed. Comparing dicts does that for free, with less code than hashing. -- **`counters` deleted.** Derivable from the sidecar and double-counting on every resume (re-processed albums re-increment), which would make phase 7 chase phantom discrepancies. Each run prints its own in-memory session totals at exit. -- **`completed_at`** — set on clean exhaustion, cleared to `null` at every run start. The only way to answer "did my 20-hour export finish?" without guessing. -- Mismatch on `filters` or `chat_id` → refuse, print the diff, exit 7: - ``` - filter mismatch: - types: ['photo','video'] -> ['video'] - max_size: 100MB -> (none) - re-run with the original filters, or --reset-state to start over - ``` - -`--limit` is **not** in the filter set — it changes when we stop, not what a post contains, so including it would make a limited test run poison every later full run. A `--limit 20` run is a resumable partial export whose cursor genuinely reflects completed work. - -### Sidecar - -```json -{"message_id":1043,"post_id":1042,"grouped_id":88,"date":"2026-07-04T12:00:00Z", - "sender_id":777,"sender_name":"Alice","caption":"beach trip","reply_to":1001, - "files":[{"path":"1042/1043_photo.jpg","size":1234567,"declared_size":1234999, - "name":"photo.jpg","mime":"image/jpeg","kind":"photo"}], - "errors":[]} -``` - -Under `--include-text` a text-only message produces the same record shape with `"files":[]` — no folder is created for it. Consumers distinguish media from context by `files` being empty, not by a separate record type. - - - -`size` is bytes on disk; `declared_size` appears only when it differs — which incidentally accumulates empirical data on photo-variant accuracy. Paths are relative to the export root. `sender_name` comes from `msg.sender` **only if already cached** — never an extra `get_entity` call, which on a 100k-message sweep would be 100k extra RPCs and a flood ban. - -**A record is an append-only event: "the files written for this message in this run."** The correct consumer rule is therefore **union over all records for a `message_id`**, not last-write-wins. Last-write-wins was actively wrong — after a filter change it would report a narrower file list than what is on disk. - -Startup repair: scan back from EOF for the last `\n`; if the trailing segment is non-empty and not valid JSON, truncate to that offset and warn. A host crash mid-`write()` otherwise leaves a partial line that breaks every JSONL consumer. - -`--reset-state` **rotates** `messages.jsonl` → `messages-.jsonl` and starts fresh, so the current sidecar always describes exactly one filter regime. It does **not** delete downloaded files — dedupe reclaims them; deleting 30 GB to change a filter is hostile. - -## Related Code Files - -- Create: `src/telegram_exporter/downloader.py`, `state.py`, `sidecar.py` -- Create: `tests/test_download_decisions.py`, `tests/test_state.py` -- Modify: `src/telegram_exporter/cli.py` (wire the download path, `--reset-state`, `--min-free`, `--max-flood-wait`) - -## Implementation Steps - -1. `state.py`: load/create, filter and chat_id comparison with a named diff, atomic save (tmp → fsync → replace → fsync dir), `commit`, `mark_completed` -2. `sidecar.py`: append-only JSONL writer, relative paths, fsync, startup partial-line repair, rotation on `--reset-state` -3. `downloader.py`: `check_disk`, `validate`, `download_one` with the full taxonomy, startup `.part` sweep (after lock acquisition) -4. `downloader.py`: `run_download` per the sketch — sidecar before cursor, cursor unreachable from abort paths -5. Progress output: per-file line with post id, filename, size, running total; session totals at exit including `N files failed (see messages.jsonl)` -6. Write `/title.txt` once on first run -7. Tests per the table below - -## Test Table — `test_download_decisions.py` (tmpdir + fake downloader, no network) - -| Case | Expected | -|---|---| -| target absent | DOWNLOADED | -| target exists, non-empty | SKIPPED, no download call | -| target exists, zero bytes | unlinked, re-downloaded | -| stray `.part` at startup | swept | -| `.part` present and target valid | target wins, `.part` removed | -| `media_size` is `None` | stated policy, asserted | -| `file.name == "../../../etc/passwd"` | lands inside `post_dir` | -| free space < file size + reserve | `Abort(3)` before any write | - -## Success Criteria - -- [ ] Full run downloads every media file into album-aware folders -- [ ] `kill -9` mid-run then re-run: no re-downloads, no gaps, no orphan `.part` -- [ ] Interrupting mid-album re-downloads that album and leaves no duplicates -- [ ] Every row of the test table passes -- [ ] Re-running with changed filters refuses and prints a per-filter diff (exit 7) -- [ ] A different `chat_id` against an existing state file refuses (exit 7) -- [ ] `messages.jsonl` has one line per media message with correct relative paths; a truncated final line is repaired at startup -- [ ] `--reset-state` rotates the sidecar and preserves downloaded files -- [ ] Media with a dead file reference is refreshed once, then recorded in `errors[]` and **skipped** — the run continues past it -- [ ] With `--include-text`, text-only messages appear in the sidecar with `"files":[]` and create no folder -- [ ] Toggling `--include-text` against an existing cursor triggers the filter-mismatch refusal (exit 7) -- [ ] A flood wait logs its duration **and computed wake time**; with no `--max-flood-wait` set, the run sleeps rather than exiting -- [ ] Simulated low disk aborts before writing, with a resumable cursor -- [ ] `completed_at` is set only on clean exhaustion -- [ ] `test_no_concurrency.py` (AST scan) finds no `gather`/`create_task`/`to_thread`/thread pool in `src/` - -## Risk Assessment - -- **Cursor ahead of reality** → permanent silent gaps; the worst failure here because it looks like success. Invariant 2 is structural: `state.commit` is unreachable from every abort path. -- **Right-sized corrupt file after host crash** → fsync before `os.replace`, plus dir fsync before the cursor claims the post durable. This is what makes existence-based dedupe sound. -- **Poison-pill validator** → validator never raises; photos excused; design independent of phase 1 P5's answer. -- **Unhandled exception kills an unattended 20-hour run** → full taxonomy above; transient errors retry with backoff, terminal ones exit cleanly with a correct cursor. -- **Filter/cursor confusion** → verbatim filters, named diff, hard refusal. -- **Disk exhaustion mid-write** → per-file pre-check against this file's size plus a 2 GiB reserve. -- **Concurrent runs** → flock acquired before the `.part` sweep (phase 2). -- **Sidecar misread after a filter change** → union semantics, documented; `--reset-state` rotates. -- **Temptation to parallelize when it feels slow** → flood-wait escalation to multi-hour lockouts. Enforced by an AST test, not by discipline. diff --git a/plans/260811-1411-telegram-group-media-exporter/phase-06-dry-run-estimator.md b/plans/260811-1411-telegram-group-media-exporter/phase-06-dry-run-estimator.md deleted file mode 100644 index f1821b9..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/phase-06-dry-run-estimator.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -phase: 6 -title: "Dry-Run Estimator" -status: complete -priority: P2 -dependencies: [5] -effort: "~1h" ---- - -# Phase 6: Dry-Run Estimator - -## Overview - -`--dry-run`: sweep, download nothing, report what a real run would fetch **from where it would actually start**. With 41 GB free and an unknown group size, this is the difference between a planned export and a surprise. - -## Requirements - -- Functional: counts and bytes by media type honoring the same filters *and the same cursor*; disk comparison; non-zero exit when it won't fit -- Non-functional: no `download_media` calls; no writes anywhere, including no creation of the export root; no lock taken; no state mutation - -## Architecture - -`run_estimate()` drives the **same** `iter_posts` generator with the same `Filters` as `run_download()`. That shared generator is the anti-drift property — an estimator with its own traversal would report numbers that don't match reality. There is no `Sink` Protocol: both run modes own a plain `async for`, and the duplication is two lines. - -### The cursor fix - -Dry-run **reads** state (never writes) and starts where the real run would: - -```python -state = State.load(out_dir) # read-only -start = state.cursor_id if state and state.filters == filters else 0 -``` - -Without this, `tg-export --dry-run && tg-export` fails **closed** on every resumed run: a 38 GB group with 30 GB already downloaded and 11 GB free would re-measure the whole 38 GB, exit 2, and refuse to fetch the 8 GB that fits comfortably. The user's only escape would be to bypass the guard they were told to trust. - -Deliberately **not** doing a "scan the export dir and subtract bytes on disk" pass: with a cursor-based sweep everything before the cursor is already excluded, and the residue is a handful of failed files and one partial post. A disk-scan reconciliation is unearned complexity. - -### Report - -``` -Group: My Group (id -1001234567890) -Filters: types=video,photo since=2026-01-01 max-size=100MB -Mode: resume from message 1042 # or "full history (no prior state)" - -Remaining: 1,204 posts / 3,891 files - photos 1,902 files 2.1 GiB (approximate — size variant) - videos 1,944 files 35.4 GiB - documents 45 files 0.7 GiB - unknown size 3 files (not counted) - -Total: 38.2 GiB -Free: 11.0 GiB Reserve: 2.0 GiB -Verdict: SHORT by 29.2 GiB exit 2 -``` - -Unknown-size files get their own line rather than being coerced to zero — an unbounded estimate error must be visible. Photo totals are labelled approximate; document/video/audio are exact. - -Posts and files count **posts with files** — neither empty posts (fully filtered groups, which phase 3 yields to keep the cursor advancing) nor text-only posts under `--include-text` are counted, on both sides of any comparison. Text messages carry no bytes, so the estimate is unaffected by the flag. - - - - -### The verdict is advisory; the per-file guard enforces - -Phase 5's per-file `check_disk` is the sole enforcement mechanism. This removes the previously "optional preflight behind a flag" — a flag that appeared in no flag surface — and the double sweep it implied on huge groups. - -Say this plainly in the README: a `SHORT` verdict does not mean the run is unsafe. It means the run will stop partway with a clean, resumable cursor. - -## Related Code Files - -- Create: `src/telegram_exporter/estimate.py` -- Modify: `src/telegram_exporter/cli.py` (`--dry-run` dispatches to `run_estimate`) - -## Implementation Steps - -1. `EstimateSink` accumulating posts-with-files, files, and bytes by media kind, plus an unknown-size tally -2. Human-readable byte formatting (GiB/MiB, thousands separators) -3. `State.load` in read-only mode; derive `start` and the `Mode:` line -4. Free space via `shutil.disk_usage(out_dir)`; compare `total + reserve` against free -5. Report renderer per the layout; label photo totals approximate -6. `cli.py`: `--dry-run` → `run_estimate`, taking no lock and creating no directories -7. Verify dry-run leaves the filesystem untouched - -## Success Criteria - -- [ ] `--dry-run` completes with zero `download_media` calls and zero bytes written -- [ ] Dry-run does not create the export root, `.export-state.json`, `messages.jsonl`, or `.export.lock` -- [ ] Dry-run takes no lock — it can run while a real export is in progress -- [ ] On a resumed export, `Mode:` shows the cursor and totals cover only remaining work -- [ ] `tg-export --dry-run && tg-export` succeeds on a partially completed export that fits -- [ ] Totals broken down by media kind; unknown-size files on their own line, never as zero -- [ ] Exit `2` when total + reserve exceeds free space; `0` otherwise -- [ ] Filters applied identically to a real run via the shared `iter_posts`, not by duplicated logic -- [ ] `--dry-run --limit 20` vs `--limit 20` from the same start: post and file counts **exactly equal**, bytes within **2%** (photo approximation only) - -## Risk Assessment - -- **Estimator drifts from downloader** → the number users trust becomes a lie. Prevented structurally by the shared `iter_posts`; never fork the traversal. -- **Dry-run mutates state** → a later real run resumes from a cursor nothing downloaded. Read-only by construction; asserted. -- **Dry-run fails closed on resume** → fixed by reading the cursor. This was the defect that made the documented safe idiom unusable. -- **Photo approximation misread as exact** → labelled, plus the 2% tolerance is stated rather than left to judgement. -- **Sweeping a very large history triggers flood waits** → history reads are cheap and covered by `aiter_with_flood_retry`. diff --git a/plans/260811-1411-telegram-group-media-exporter/phase-07-end-to-end-validation.md b/plans/260811-1411-telegram-group-media-exporter/phase-07-end-to-end-validation.md deleted file mode 100644 index e7f84b3..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/phase-07-end-to-end-validation.md +++ /dev/null @@ -1,112 +0,0 @@ ---- -phase: 7 -title: "End-to-End Validation" -status: pending -priority: P1 -dependencies: [6] -effort: "~2h + export runtime" ---- - -# Phase 7: End-to-End Validation - -## Overview - -Prove the tool against a real group, with deliberate interruption. The failure modes that matter — cursor ahead of reality, split albums, flood-wait escalation — are invisible in unit tests and surface only against live data. - -Every step below has a checkable outcome. **No step may pass by escape clause.** - -## Requirements - -- Functional: full export completes, resumes correctly after kill, produces a consistent sidecar -- Non-functional: README documents setup and semantics; no secret or third-party data is committable; the spike's server-side session is revoked - -## Validation Script - -1. **Dry run** — `tg-export --group --out ./exports --dry-run`. Record posts, files, total bytes, verdict. Confirm no export root was created. - -2. **Limited real run** — `--limit 20`. Inspect by hand: albums collapsed into one folder, standalone messages in their own, filenames `{message_id}_{name}`, `title.txt` written, directory named `g`. - -3. **Sidecar cross-check** — scripted, not by eye: - ```bash - python3 - <<'EOF' - # 1. every files[].path in messages.jsonl exists on disk - # 2. every file on disk appears in the UNION of records for its message_id - # 3. no two post folders share a grouped_id <- catches album splitting - EOF - ``` - -4. **Idempotency** — `--reset-state`, then re-run `--limit 20`. Expect **zero downloads, all skips**. - *(Re-running `--limit 20` without resetting would correctly continue to posts 21–40 — the cursor advanced because those 20 posts genuinely completed. Resetting is what isolates the dedupe path, which is the thing under test.)* - -5. **Interruption** — start a full run, `kill -9` after ~50 files. Re-run. Confirm: no surviving `.part`, no re-downloads of completed files, no gaps, the in-flight album complete. - -6. **Network loss** — drop the interface or block Telegram with iptables mid-download for ~30s, then restore. Confirm the transient path retries with backoff and the run continues. `kill -9` alone does not exercise exception propagation or reconnect, so it cannot stand in for this. - -7. **Filter guard** — re-run with `--types video` against the existing state. Confirm it refuses, **names the changed filters**, and exits 7. Then `--reset-state` and confirm it proceeds and rotates `messages.jsonl`. - -8. **Concurrency guard** — start a run; while it holds the lock, start a second against the same `--out`. Confirm exit 3 with pid/host/start-time, and that no file was touched. Confirm a run against a *different* group proceeds. - -9. **Path safety** — covered by `tests/test_paths.py` and `tests/test_download_decisions.py`, which include the hostile-filename case offline. *(The previous "otherwise rely on unit tests and note no live sample existed" escape clause is deleted — it made passing the default.)* Optionally self-send a message named `../../../etc/passwd` to a scratch group for live confirmation. - -10. **Full export** — run to completion. Confirm `completed_at` is set. Reconcile against step 1: post and file counts **exactly equal**; bytes within **2%** (photo approximation only). Any file-count delta is a bug to investigate, never accepted — expected differences are only deleted/expired media, which appear in `errors[]` and the exit summary. - -11. **Flood-wait behavior** — grep logs: each wait slept through once, no retry storm, no restart during an active wait, and every entry carries a computed wake time. Separately confirm `--max-flood-wait 1` exits 6 cleanly with a resumable cursor, since that path is unreachable by default. - -11b. **`--include-text`** — run with the flag on a scratch range. Confirm text-only messages appear with `"files":[]`, create no folders, and that toggling the flag against the existing cursor triggers the exit-7 refusal. - -12. **Memory** — `/usr/bin/time -v tg-export --dry-run` → peak RSS under 200 MB. The album tripwire set is O(albums), not O(1); the criterion states the real bound. - -12b. **Sweep cost under a date filter** — measure before deciding whether to optimize it. With `reverse=True` and no `offset_date`, telethon 1.44's `_MessagesIter._init` starts the sweep at message id 1, so `--since` walks the entire history discarding messages in `keep()`. Time `--dry-run --since ` against a large group and record the wall clock, the getHistory round-trip count, and the ratio of messages swept to messages kept. Do the same for `--until`, which currently never terminates the sweep early. - - Both fixes carry an invariant risk, which is why this is a measurement rather than a change: - - server-side `offset_date` can start the sweep *mid-album*, so `_close` would take a `post_id` that is not the album's lowest id — breaking filter-invariant post identity. Needs a mid-album guard before it is safe. - - an early break on `--until` assumes message dates rise monotonically with ids, which is false for imported history. - - Decide with the numbers: if the ratio is small the current sweep is fine, and neither risk is worth taking. - -13. **Revoke the spike session** — Telegram → Settings → Devices → terminate the phase 1 probe session. **Deleting `spike/probe.session` does not revoke server-side authorization**; a live auth key otherwise survives for an account you believe is clean. - -## Related Code Files - -- Modify: `README.md` -- Delete: `spike/` (or leave gitignored) - -## README Must Document - -- Obtaining `api_id`/`api_hash`; the credential policy table (env vs prompt vs getpass; what `.env` may contain) -- First-run login incl. 2FA; session path and why it defaults outside the repo -- **Why the Bot API cannot do this** — the first question any reader will have -- Resume semantics: the cursor means "handled through here *under these filters*", and why changing filters requires `--reset-state` -- Sidecar semantics: records are append-only events; **union over records per `message_id`**, not last-write-wins -- Export directory is `g` with the name in `title.txt`, and why (path safety + surviving group renames) -- Why downloads are sequential — flood-wait escalation, and that `--workers` deliberately does not exist -- `--include-text`: off by default; when set, text-only messages appear in the sidecar with `"files":[]` and no folder. It is part of the stored filter set, so toggling it requires `--reset-state` -- **Flood waits have no ceiling by default** — the tool sleeps as long as Telegram demands, logging the computed wake time each time. A long sleep is a sleep, not a hang; check the log before assuming otherwise. Pass `--max-flood-wait SECONDS` to exit resumably instead - - - -- That the dry-run verdict is **advisory**; the per-file disk check enforces. A `SHORT` verdict means the run stops partway, resumably — not that it is unsafe -- `cryptg` is optional and not installed by default: native code in Telethon's AES path, trusted (if adopted) because the Telethon author maintains it — not because it is "CPU-side only" - -## Success Criteria - -- [ ] Every step 1–13, including 11b, passes with its stated outcome — no step passes by escape clause -- [ ] Full export completes with `completed_at` set; counts reconcile within the stated tolerance -- [ ] Kill-and-resume produces a complete export with no gaps and no duplicates -- [ ] Network-loss interruption recovers without operator intervention -- [ ] Filter guard fires with a named diff and clears with `--reset-state` -- [ ] Concurrency guard exits 3; a different group is unaffected -- [ ] `pytest` green; `pip install -r requirements.txt -e .` from a clean venv yields a working `tg-export` -- [ ] `git status` clean of session files, exports, state, lock, and `.env` -- [ ] Spike session terminated server-side -- [ ] README covers every item listed above - -## Risk Assessment - -- **Group larger than free disk** → step 1 reports it; the per-file guard stops cleanly and resumably. Scope down with `--max-size`/`--types` or export to external storage. -- **Long export interrupted by network loss** → step 6 is the proof; step 5 alone would not be. -- **Estimate/actual discrepancy** → expected sources are deleted/expired media (visible in `errors[]`) and photo approximation. Anything else is a filter or traversal bug — investigate. -- **Album split** → step 3's grouped_id check plus phase 3's runtime tripwire. A split that reaches disk is silent otherwise. -- **Account flood-limited during validation** → keep `--limit` small in steps 2–8; run step 10 once, unattended. -- **Third-party data committed** → the sidecar contains message text and sender names. Export root is gitignored and `assert_not_committable` refuses an un-ignored path inside the repo; verify before any commit. -- **Spike credential left live** → step 13. The most-missed cleanup item, because the local file looks like the whole credential. diff --git a/plans/260811-1411-telegram-group-media-exporter/plan.md b/plans/260811-1411-telegram-group-media-exporter/plan.md deleted file mode 100644 index 70e11f7..0000000 --- a/plans/260811-1411-telegram-group-media-exporter/plan.md +++ /dev/null @@ -1,376 +0,0 @@ ---- -title: "Telegram Group Media Exporter" -description: "CLI tool exporting all media from a Telegram group to local disk, grouped by message/album, with resume, filters, and dry-run size estimation." -status: in-progress -priority: P2 -branch: "main" -tags: [telethon, python, cli, mtproto, export] -blockedBy: [] -blocks: [] -created: "2026-08-11T07:22:18.583Z" -createdBy: "ck:plan" -source: skill ---- - -# Telegram Group Media Exporter - -## Overview - -CLI tool `tg-export` — downloads all media from a Telegram group to local disk, one folder per logical post (albums collapsed), with a `messages.jsonl` sidecar preserving message context. Resumable, filterable, with a dry-run size estimate. - -Greenfield repo. Research: [research report](../reports/research-telethon-mtproto-260811-1411-telegram-group-media-exporter-report.md). Reviewed adversarially — see [Red Team Review](#red-team-review); the design below is post-review. - -## Locked Decisions - -Settled before planning — do not re-litigate without new evidence. - -| Decision | Rationale | -|---|---| -| Telethon + Python 3.12, MTProto **user account** | Bot API is structurally incapable: no history access, 20 MB download cap | -| Sequential downloads, **concurrency 1** | Telethon does not parallelize transfers; parallelism accelerates FloodWait escalation (7s → 4h+) | -| Album-aware folders, id = **lowest message id** in the group | User decision; deterministic across re-runs | -| `SQLiteSession` file (not StringSession) | Persistent local filesystem; StringSession solves a containerization problem we don't have | -| `argparse` (not Typer/Click) | One command, ~12 flags, no subcommands | -| `pyproject.toml` + `[project.scripts]` | Current packaging standard | -| Telethon pinned `>=1.41,<2` + `requirements.txt` | Telethon 2.0 is alpha with a renamed API surface; the pin file records the version phase 1 actually verified | - -## Architecture - -Both run modes drive the **same** `iter_posts` generator with the same `Filters`. That shared generator — not any class hierarchy — is what guarantees the dry-run estimate describes what a real run would download. A forked traversal would drift and lie. - -``` -cli.py ──> session.py ──> traversal.py ──┬──> run_download() ──> downloader.py, state.py, sidecar.py - argparse auth, flood sweep + │ - umask wrapper album group └──> run_estimate() ──> estimate.py - logging + filters - locks paths.py (shared, security-critical) -``` - -``` -src/telegram_exporter/ - cli.py argparse, umask, logging, locks, git-safety asserts, dispatch - session.py credentials, session path prep, client, flood retry, entity resolution - traversal.py Post, Filters, media_size(), iter_posts, album-split tripwire - paths.py export_root, post_dir, sanitize, derive_filename, safe_join (pure) - downloader.py download_one, error taxonomy, .part sweep, run_download() - state.py resume cursor only - sidecar.py messages.jsonl append writer only - estimate.py EstimateSink, byte formatting, run_estimate() -tests/ - test_paths.py sanitize table, safe_join, export_root type guard - test_download_decisions.py skip/re-download/.part logic against a tmpdir - test_traversal.py album grouping + split tripwire fires - test_state.py filter mismatch, atomic save - test_lock.py subprocess contention - test_logging.py telethon logger stays >= WARNING - test_no_concurrency.py AST scan rejects gather/create_task -``` - -**No `Sink` Protocol.** Two implementations with one call site and no shared behavior is indirection describing a `for` loop. `run_download()` and `run_estimate()` each own their own `async for`. The anti-drift property lives in the shared `iter_posts` — restated here so nobody re-adds the Protocol believing it was load-bearing. - -**`state.py` and `sidecar.py` are separate modules on purpose.** The correctness-critical part is the *ordering* between them, and that ordering belongs to the caller. Two objects in two modules makes the sequence visible at one call site and makes a "helpful" `commit_and_append()` — which would destroy Invariant 2 — impossible to write without noticing. - -Output layout: - -``` -exports/g-1001234567890/ <- directory name is the chat id, never the group title - title.txt <- human-readable name lives here, as data - 1042/ <- album (shared grouped_id), folder = first member id - 1042_photo.jpg <- filename keyed on message id, not a positional index - 1043_photo.jpg - 1044_video.mp4 - 1047/ <- standalone message - 1047_document.pdf - messages.jsonl - .export-state.json - .export.lock -``` - -**Exactly one untrusted string ever becomes a path component: the filename leaf.** Every other component is an `int` enforced by a type check. The group title is server-supplied and editable by any admin, so it is stored as data and never as a path — which also stops a group rename from orphaning an in-progress export. - -## Three Invariants - -Violating any of these produces silent corruption that looks like success. - -1. **Identity is filter-invariant.** The folder is the lowest message id in the *full* `grouped_id` group; the filename is keyed on `message_id`. Neither depends on which members a filter dropped, so `--types photo` and `--types video` address the same files. -2. **The cursor commits only after a complete post, and never from an abort path.** `state.commit()` is unreachable from every error exit, so the cursor can never point past an in-flight post. -3. **Filters are stored verbatim in the state file.** The cursor means "handled through here *under these filters*". A mismatch refuses the run and names what changed. - -## Acceptance Criteria - -`[x]` = verified offline by the test suite. `[~]` = implemented and unit-tested, -but the end-to-end proof is phase 7 against a live group. - -- [~] `tg-export --group --out ./exports --dry-run` reports remaining posts/files/bytes by media type and a disk verdict, writing nothing — *"writes nothing" and the per-kind breakdown are tested; the live sweep is phase 7 step 1* -- [~] `tg-export --group --out ./exports` downloads all media into album-aware folders -- [~] Killing the run mid-export and re-running resumes with no re-downloads and no gaps — *dedupe, cursor and `.part` sweep are tested; the `kill -9` proof is phase 7 step 5* -- [x] `messages.jsonl` links every file to its message id, post id, date, sender, and caption -- [x] A message whose filename is `../../../etc/passwd` writes inside the post dir, proven by test -- [x] A group titled `..` or `a/b` cannot affect the output path, proven by test — *structurally: no code path exists from title to path component* -- [x] A run aborts before writing the first byte of any file that would not fit, leaving a resumable cursor -- [x] A second concurrent run against the same export root exits 3 without touching any file — *real subprocess contention* -- [x] `FloodWaitError` is slept through once and logged with its computed wake time; no tight retry; a wait beyond an explicit `--max-flood-wait` cap exits cleanly and resumably -- [x] Media that is deleted, expired, or unavailable is recorded and **skipped**, and the run continues -- [x] `--verbose` leaves the `telethon` logger at `WARNING`; `api_hash` and phone absent from captured output -- [x] Session file and every sibling are mode `0600`; export tree `0700` -- [x] `pytest` green (204 tests, and green under `python -O`); `pip install -r requirements.txt -e .` from a clean venv yields a working `tg-export` - -## Phases - -| Phase | Name | Status | -|-------|------|--------| -| 1 | [Verification Spike](./phase-01-verification-spike.md) | Partial — offline probes settled, live probes await credentials | -| 2 | [Skeleton and Session Auth](./phase-02-skeleton-and-session-auth.md) | Complete | -| 3 | [Traversal Grouping and Filters](./phase-03-traversal-grouping-and-filters.md) | Complete | -| 4 | [Naming and Path Safety](./phase-04-naming-and-path-safety.md) | Complete | -| 5 | [Downloader State and Resume](./phase-05-downloader-state-and-resume.md) | Complete | -| 6 | [Dry-Run Estimator](./phase-06-dry-run-estimator.md) | Complete | -| 7 | [End-to-End Validation](./phase-07-end-to-end-validation.md) | Pending — requires a live group | - -**Dependency chain:** 1 → 2 → 3 → 4 → 5 → 6 → 7. Phase 4 is pure and offline-testable, so it can run parallel to 3. Phase 1 gates everything. - -## Open Questions - -1. **Group scale** — unknown volume. Resolved by phase 6 dry-run. -2. **2FA enabled?** — resolved by phase 1 P2. -3. **Group access type** — resolved by phase 1. -4. ~~**Text-only messages in `messages.jsonl`?**~~ **Resolved (validation):** default media-bearing only, plus an `--include-text` flag that records text messages with an empty `files[]`. The flag is part of the stored filter set, since it changes what a post contains. -5. **Deleted/expiring media** — recorded as an `errors[]` entry on that message's sidecar record and counted at exit; the run continues. -6. **Stray `.part` files** — swept at startup, after the lock is acquired. -7. **Unknown-size media (`file.size is None`)** — **included** and warned, counted separately in the estimate rather than as zero. Invert if disk pressure matters more than completeness. -8. **Export dir named `g`, not the group title** — chosen to make path traversal structurally impossible and to survive group renames. Human name lives in `title.txt`. Reversible if you prefer readable directory names, at the cost of both properties. -9. **Should a FAILED file be permanently abandoned?** Raised by the - post-implementation review, and genuinely unsettled by this plan. Today a - failure advances the cursor and is recorded in `errors[]`, which is right for - deleted or expired media but also covers a file that exhausted its three - retries on transient network trouble — that one is never re-attempted, and - `completed_at` is still set. The exit summary names the count. Options if this - matters: re-drive the range with `--reset-state`, or add a `--retry-failed` - pass that reads the sidecar. **Not changed unilaterally — this is a scope - decision.** -10. **Concurrent exports of two groups need two `--session` files**, not just two - `--out` directories, because the session SQLite database is locked too. The - plan claimed per-root locking made concurrent exports work; it does, but only - with that second flag. Documented in the README and in the exit-3 message. - -11. **What consecutive-failure threshold should stop a run?** `completed_at` is - now withheld when every file failed, but a run against a dead session still - walks the entire history failing each message before returning. A circuit - breaker would stop it early; the threshold is a judgment call, because a dead - session and a legitimately corrupt region of history look identical from - inside `download_one`. **Not invented unilaterally.** -12. **Should a server-declared 0-byte document be exportable?** The zero-byte - guard now refuses `actual == 0` for every kind, which is right for an empty - photo variant and for ENOSPC junk, but makes a genuinely empty uploaded - document unexportable — it costs three attempts and a FAILED record. If - Telegram permits such uploads, `declared == 0 and actual == 0` could be - accepted without weakening the guard. Needs a fact about Telegram that phase 1 - could answer. -## Dependencies - -No cross-plan dependencies. - -External: `telethon>=1.41,<2`, pinned in `requirements.txt` to the version phase 1 verified. `cryptg` is **not** installed in v1: it is a native extension sitting in Telethon's AES path, so it sees the auth key and every plaintext byte. It ships an aarch64 wheel (verified — no build step), and the reason to trust it if adopted is that the Telethon author maintains it, not that it is "CPU-side only". Don't add a supply-chain root for a throughput problem that has not been measured. - -## Red Team Review - -### Session — 2026-08-11 - -**Reviewers:** 4 (Security Adversary, Failure Mode Analyst, Assumption Destroyer, Scope & Complexity Critic) -**Findings:** 39 raw → 20 deduplicated → 15 adjudicated (13 accepted, 2 rejected, 4 corrected as overstated) -**Severity breakdown:** 7 Critical, 8 High - -Evidence standard adapted for a greenfield repo: reviewers cited plan-file and research-report lines, since no source code exists to grep. - -| # | Finding | Severity | Disposition | Applied To | -|---|---------|----------|-------------|------------| -| 1 | Photo `.size` documented approximate but used as an exact-equality gate that raises → poison pill | Critical | Accept | Phase 1, 5 | -| 2 | Filename index computed post-filter → breaks dedupe and determinism | Critical | Accept | Phase 3, 4, 5 | -| 3 | Download loop has no `try`; "skip deleted media" structurally impossible | Critical | Accept | Phase 5 | -| 4 | `` is attacker-controlled and unsanitized | Critical | Accept | Phase 4, plan | -| 5 | Flood-wait wrapper cannot protect the sweep | Critical | Accept (re-reasoned) | Phase 2, 3 | -| 6 | `--limit` advances the cursor; phase 7 idempotency step unrunnable | Critical | Accept (fix relocated) | Phase 7 | -| 7 | Phase 1 creates `.session` before any `.gitignore`; sqlite creates it `0644` before the post-hoc `chmod` | Critical | Accept | Phase 1, 2 | -| 8 | Disk guard fires ≤50 files late; "abort before first byte" unimplemented | High | Accept | Phase 5, 6 | -| 9 | Empty-`Post` contract deferred to the implementer; no completion marker | High | Accept | Phase 3, 5 | -| 10 | Dry-run ignores the cursor → `--dry-run && run` fails closed on resume | High | Accept | Phase 6 | -| 11 | No single-instance lock | High | Accept | Phase 2, 5 | -| 12 | `--reset-state` leaves the sidecar → mixed filter generations | High | Accept | Phase 5 | -| 13 | `msg.file.size is None` handled three incompatible ways | High | Accept | Phase 3, 5, 6 | -| 14 | `--verbose` unmutes telethon loggers; 2FA-env policy contradiction; gitignore coupled to default paths | High | Accept (traceback half rejected) | Phase 2 | -| 15 | Album consecutiveness across pagination unverified and untestable as guarded | High | Accept | Phase 1, 3 | - -**Rejected:** -- *"Fold phase 1 into phase 2."* The ordering bug inside it is real and fixed; the merge argument adds no evidence against the gate itself, which exists to settle facts phases 3/5/6 build on. Fixed the duplication instead. -- *"Drop probe P4 (`download_media` return value) — nothing consumes it."* True as written, but finding 1's fix makes the return value the source of truth for `os.replace`. - -**Corrected as overstated or factually wrong** (verified empirically during resolution): -- `cryptg` "builds from sdist on ARM64, executing at install time" — **false**. It ships a `cp312-manylinux_2_28_aarch64` wheel. The plan's *wording* was still wrong; rationale rewritten. -- `--verbose` "leaks credentials via traceback" — **false**. CPython does not print locals in tracebacks. The logger half is real and fixed. -- Session permission race rated Critical — realistically Medium on a single-user host. Fixed anyway: it is one `umask` line. -- "7 modules for 600 lines" — module count is not a defect metric; cohesion is. `estimate.py` kept. The `Sink` Protocol and the `state.py` bundling were the real findings. - -**Net effect:** the resolved design is *smaller* than the reviewed one. Deleted: the `Sink` Protocol, state `counters`, the sha256 filter fingerprint, the every-50-files disk poll, the optional-preflight flag, the positional filename index, and title-based slugging. - -### Whole-Plan Consistency Sweep -- Files reread: plan.md, phase-01 … phase-07 -- Decision deltas checked: 15 -- Reconciled stale references: filename scheme (ph3/4/5), state schema (ph5/6/7), module names `naming.py`→`paths.py` (plan/ph4/ph5), flag surface (ph2/3/5/6), dedupe predicate (ph5/7), export dir naming (plan/ph4/ph5/ph7), disk guard (plan/ph5/ph6/ph7) -- Unresolved contradictions: 0 - -## Implementation Log - -### Session 1 — 2026-08-12/13 — phases 2–6 - -Phases 2–6 implemented and green: **204 tests**, all offline against synthetic -message stubs. `pip install -r requirements.txt -e .` from a clean venv yields a -working `tg-export` on Telethon **1.44.0** (aarch64 wheel). - -Phase 1 could not be run — no Telegram credentials exist in this environment — -but most of what it gates was settled without them, recorded in the -[spike findings report](../reports/spike-260811-1411-telethon-api-verification-report.md): - -- **P1 CONFIRMED from Telethon's source**, not from a sample: under `reverse=True` - the iterator does `offset_id += 1` before the first request and again per page, - and `min_id` is not consulted in the reverse branch of `_message_in_range`. So - `offset_id` is an exclusive lower bound, the cursor stores the last handled id, - and `iter_posts` carries a defensive `msg.id <= after_id` guard regardless. -- All 15 error classes in phase 5's taxonomy exist in 1.44.0. A missing one would - have been an import-time crash. -- P2, P3, P5 and P6 remain live-only. The design was already answer-independent - for all four, which is why implementation did not wait on them. - -`spike/probe_telethon_api.py` is written and ready to run. - -#### Module deviation - -`media.py` was added, holding `media_kind` / `media_size` / `file_stem` / -`file_ext`. The plan put `media_size` in `traversal.py`, but `paths.py` needs kind -and extension for `derive_filename`, and either direction of import between those -two would have been a cycle or a second copy of the rule. One owner, two -consumers, no cycle. - -One behavioral consequence worth recording: `media_kind` returns `None` when -`msg.web_preview` is set. Telethon's `Message.photo` and `.document` also return -the *web preview's* media, so classifying without that guard would have made every -message containing a URL look like an attachment — inflating counts and -downloading link-preview thumbnails. - -#### Defects found and fixed during implementation - -| # | Defect | Consequence had it shipped | -|---|--------|---------------------------| -| 1 | The 5-second commit throttle never flushed the tail | A run ending on a short stretch of filtered or text-only posts set `completed_at` against a stale cursor; the next run re-swept that tail to download nothing | -| 2 | `git check-ignore` reports `exports` as *not ignored* until the directory exists | The export root would be refused on the first run and accepted on every run after it. Fixed with a directory-form retry, gated to `is_dir=True` so it can never excuse a session *file* | -| 3 | `file_ext` capped extensions at 16 characters, not bytes | A multibyte suffix pushed the filename past ext4's 255-byte limit | - -#### Corrections to the plan's own sketches - -- **`except` ordering.** The plan checked `OSError` before the network clause, but - `ConnectionError` *is* an `OSError`, and asyncio's `ConnectionResetError` can - carry `errno=None` — which the errno test would have routed to "unexpected OS - error" and aborted a healthy run. Network clause first, filesystem second. -- **Lock contention exit code.** `raise SystemExit(f"busy: ...")` exits **1**, not - 3, so it would have failed its own acceptance criterion. Raises `Abort(reason, 3)`. -- **Exit-code ownership.** `run_download` lets `Abort` and `KeyboardInterrupt` - propagate to `cli.main()` rather than calling `sys.exit` mid-loop, and does not - call `disconnect()` — the `connected_client` context manager owns that. -- **Bare asserts.** The ascending-sweep and album-buffer tripwires are now - `SweepOrderError`, not `assert`: `python -O` strips assertions, and these guard - the same silent corruption as the `AlbumSplitError` beside them. - -### Red Team Review — Session 2 — 2026-08-13 (post-implementation) - -Independent `code-reviewer` pass over the implementation against this plan. -**Verdict: DONE_WITH_CONCERNS.** Invariants 1 and 2 verified to hold under trace -and probe; dry-run and real-run agree exactly on posts and files for -`limit ∈ {None, 0, 1, 2}`. All eight declared deviations were assessed sound and -kept. Two reachable defects broke stated guarantees: - -| # | Finding | Severity | Disposition | -|---|---------|----------|-------------| -| H1 | `--types " "` (or `,`) builds an empty kind set that drops every media kind, but `to_state()` serialized it as `None` — i.e. *no filter* | Critical | **Fixed.** Refused in `build_filters`; `to_state` made lossless as a second line of defense | -| H2 | The filename **extension** bypassed `sanitize` entirely | Critical | **Fixed.** Routed through `sanitize_ext`; plus `download_one` now records a path failure as FAILED instead of letting it escape | -| M1 | `validate` excused *any* photo size mismatch, including 0 bytes | High | **Fixed.** Zero bytes is refused for every kind | -| M2 | A FAILED file advances the cursor and is never retried | Medium | **Documented, not changed** — the plan settles this ("recorded and skipped, the run continues"). See open question 9 | -| M3 | `ServerError` (parent of `RpcCallFailError`) and the `telethon.errors.common` set escaped the taxonomy | High | **Fixed.** Both added to the retry tuple | -| M4 | `OSError` from the sidecar or state writes bypassed the exit-code contract | Medium | **Fixed.** `cli.main` maps `FATAL_ERRNO` → 3 | -| M5 | `--max-size 0` printed as `(none)`; a negative `--limit` made the two run modes disagree | Low | **Fixed.** `is not None` throughout; negative `--limit` rejected at the boundary | -| M6 | `Config.max_flood_wait` was dead | Low | **Fixed.** Removed | - -H1 was the worst of these: it is silent *total* data loss, reachable from -`--types "$UNSET_VAR"`, and it defeats Invariant 3 — the very guard meant to catch -it. H2 was a permanent denial of service from one hostile filename, since the -cursor could not advance past the message that raised. - -Also fixed from the review's lower-severity notes: duplicated `_fsync_dir` -(now `state.fsync_dir`, one owner), the session file being created before the git -check rather than after, dead code in `human_bytes` and `EstimateSink.add`, and a -dry-run `Mode:` line that silently measured from zero when the stored filters -differed — it now says a real run would refuse. - -Two criteria the review found untested are now tested: export-tree and file modes -under the process umask, and control characters in the *extension*. - -### Red Team Review — Session 3 — 2026-08-13 (adversarial pass over the fixes) - -The session-2 fixes were unit-tested but never independently reviewed, so a -second pass attacked them specifically. **It found that one of the fixes had -broken the tool outright.** - -| # | Finding | Severity | Disposition | -|---|---------|----------|-------------| -| C1 | Removing `Config.max_flood_wait` left the call site in `cli._real_run` passing it — **every real export died with an uncaught `TypeError`** after taking both locks and writing `title.txt` and the state file, before downloading a byte | **Critical** | **Fixed.** Plus `tests/test_cli_dispatch.py`, which is the real remedy | -| H-A | `AuthKeyNotFound` subclasses plain `Exception` and matched no clause. `MTProtoSender._disconnect` sets it on every in-flight request, so it is *more* likely than the `AuthKeyUnregisteredError` the taxonomy did list | High | **Fixed.** Added to `_AUTH_ERRORS` → exit 4 | -| H-B | `CdnFileTamperedError` (a hash mismatch on CDN-served bytes) was swallowed by `SecurityError` in the network clause and reported as three lines of "network error" | High | **Fixed.** Own clause, own message, not retried | -| H-C | Four of the six classes added for M3 are consumed inside Telethon's receive loop and can never reach an awaited request | Medium | **Fixed.** Dead arms removed; `InvalidBufferError` kept, since it does arrive for a transport-level 429 | -| H-D | A dead session made `run_download` fail every file, return normally, and set `completed_at` over an empty tree | High | **Partly fixed.** `completed_at` is now withheld when everything failed and nothing succeeded. The circuit breaker is open question 11 | -| M-A | `cli.main`'s `OSError` clause reported `ConnectionError`/`TimeoutError` as "filesystem" | Medium | **Fixed.** Network clause first; exit codes were already correct | -| M-B | U+202E and other `Cf`/`Zl`/`Zp` characters survived sanitization — `agpj.exe` renders as `a.exe.jpg` | Medium | **Fixed.** Strips by Unicode category, which also subsumes the old C0 table | -| M-C | A server-declared 0-byte document is now unexportable | Medium | **Open question 12** — a product call, not reversed unilaterally | -| M-D | A state file with an absent `filters` key reported a phantom diff on `include_text` | Low | **Fixed.** Absent is now its own refusal | - -Also fixed: the retry ladder no longer sleeps after its final attempt (20 s per -exhausted file, which on a dead connection was the difference between hours and -days). - -**Why C1 escaped 180 passing tests, which is the more important finding.** Every -test built `Config` by hand with the new signature, and not one exercised -`cli.main()` or `_real_run`. The suite validated each unit and never the wiring -between them, so a removed field and a stale call site could coexist happily. -`tests/test_cli_dispatch.py` now drives `main()` end to end against a fake client -— real run, resume, dry run, `--limit 0`, filter-mismatch exit 7, lock exit 3, -and an unexpected exception still honoring the exit contract. - -Confirmed unchanged by this pass: Invariants 1–3, dry-run/real-run agreement for -`limit ∈ {None, 0, 1, 2, 3, 5}`, and — importantly — that `sanitize_ext` is an -identity transform for every ordinary extension, so no previously downloaded file -is orphaned on resume. - -## Validation Log - -### Session 1 — 2026-08-11 - -Verification pass skipped per the workflow guard: `## Red Team Review` above already carries evidence-backed verification, and zero `[UNVERIFIED]` tags remain across all eight plan files. - -**Questions asked:** 5 (4 + 1 disambiguation) - -| # | Decision point | Answer | Effect | -|---|---|---|---| -| 1 | Export directory naming | `g` + `title.txt` | **Confirms** the red-team fix. No change. | -| 2 | Media with no reported size | Download + warn | **Confirms** the resolved design. No change. | -| 3 | Sidecar scope | Add an `--include-text` flag | **Change** — see below | -| 4 | Disk reserve | 2 GiB | **Confirms.** No change. | -| 5 | Flood-wait ceiling | **No ceiling by default**; `--max-flood-wait N` caps it | **Change** — reverses the plan's prior default | - -**Change 1 — `--include-text`.** Default remains media-bearing only. With the flag, text-only messages are recorded in the sidecar with an empty `files[]`. It joins the stored filter set because it changes what a post contains: running without it and then with it against the same cursor would otherwise silently skip the text records. Text-only messages never create a post folder and never affect `posts_with_files` counting, so the dry-run/real-run reconciliation is unaffected. - -**Change 2 — flood-wait default inverted.** Previously the tool exited resumably after a wait beyond 1 hour. Now it sleeps indefinitely by default, and `--max-flood-wait N` imposes a ceiling. - -The trade-off was presented and the user chose unattended completion over fail-fast: a badly throttled account now looks like a hang rather than an exit. Accepted with one mitigation — every flood wait logs at WARNING with its duration **and computed wake time**, so a long sleep is legible as a sleep. Exit code 6 is retained for the capped case. - -### Whole-Plan Consistency Sweep — Validation Session 1 -- Files reread: plan.md, phase-01 … phase-07 -- Decision deltas checked: 2 -- Reconciled: flag surface (plan/ph2/ph3/ph5), filter set membership of `--include-text` (ph3/ph5), sidecar record shape for text messages (ph5), `posts_with_files` counting (ph3/ph6), flood-wait defaults and exit code 6 (plan/ph2/ph5), README items (ph7) -- Unresolved contradictions: 0 diff --git a/plans/reports/code-review-260822-2259-full-codebase-audit.md b/plans/reports/code-review-260822-2259-full-codebase-audit.md deleted file mode 100644 index 2b4b416..0000000 --- a/plans/reports/code-review-260822-2259-full-codebase-audit.md +++ /dev/null @@ -1,414 +0,0 @@ -# Full-codebase review — telegram-exporter - -Date: 2026-08-22 · Scope: read-only, no source/test/config modified -Reviewed: `src/telegram_exporter/*.py` (~1,900 LOC), `README.md`, plan index, `tests/` (skim) -Method: line-by-line read; Telethon 1.44.0 wheel downloaded to a scratch dir and -`telethon/client/messages.py` read to check the traversal claims. Test suite NOT run -(pytest unavailable here; execution owned by another agent). - -## Verdict - -The design is unusually disciplined and the three invariants hold as written. The -correctness core — sanitizer, `safe_join`, album grouping, cursor ordering, error -taxonomy — survived a deliberate attempt to break it. What is left is a small set -of ordering/edge defects at the *outside* of that core (process startup, sidecar -rotation, sidecar repair, entity resolution) plus two genuinely large throughput -wins that the current sweep leaves on the table. - -Nothing here contradicts an already-adjudicated decision. Where a finding touches -plan open question 9, 11 or 12, that is stated and the finding is narrowed to the -part those questions do not cover. - ---- - -## Must fix (correctness) - -### M1 — the session lock is acquired after the thing it protects (High) - -`cli.py:195` takes `exclusive(session_lock)` inside `async with connected_client(...)` -(`cli.py:157`). By the time the lock is tested, `connected_client` has already run -`prepare_session_path`, `TelegramClient(...)`, `await client.connect()` and `_login()` -(`session.py:246-254`) against the same SQLite file the lock exists to guard. - -Failure scenario (the exact one the README documents as supported, `README.md:242-245`): -two groups, two `--out` dirs, default `--session`. Both processes pass the per-root -lock (different roots), both open and write the same `default.session` SQLite DB and -both register as clients on the same auth key. Only *then* does B hit the session -lock and exit 3. Observed damage window covers Telethon's session writes; the -plausible outcomes are `sqlite3.OperationalError: database is locked` escaping as a -generic exit 1, or the auth-key duplication the README itself calls a corruption -hazard. The exit-3 message ("one SQLite session cannot serve two clients") is -printed *after* the violation it describes. - -Fix: acquire the session lock in `_run` before `connected_client` is entered, i.e. -hoist it above line 157. The root lock can stay where it is (it guards the tree, and -the `.part` sweep is still the first destructive act under it). - -Related, same root cause: `_dry_run` (`cli.py:175`) takes **no** session lock, and -`README.md:247-248` promises "`--dry-run` takes no lock at all, so it can always be -run against an export that is currently in progress." With the default session that -promise puts a second Telethon client on the in-progress run's session file — which -`README.md:244` calls a corruption hazard. The two README paragraphs contradict each -other. Either the dry run must take the session lock (and the README sentence must -gain "…with its own `--session`"), or the sentence must be narrowed. This is the one -finding where the fix depends on product intent, so pick before coding. - -No test covers the session lock at all: `tests/test_lock.py` drives `exclusive()` -directly and `test_different_export_roots_do_not_contend` only proves the *root* -lock is per-root. - -### M2 — `Sidecar.repair()` can discard 1 MiB of valid records (Medium) - -`sidecar.py:58-77`. `cut = tail.rfind(b"\n")` returns `-1` when the last 1 MiB window -contains no newline; that case is not distinguished. `trailing` then becomes the -whole window, `json.loads` fails, and `f.truncate(end - len(trailing))` cuts 1 MiB -off the file — landing mid-line, so the file is *still* unterminated and the repair -did net damage. - -Reachable when the sidecar's tail is one long unterminated stretch: an interrupted -write on a filesystem that padded the tail, or any future record shape larger than -the window. The `window == end` sub-case (small file, no newline anywhere) truncates -to 0, which is correct; only the `end > window` case is wrong. - -Fix: treat `cut == -1` as "no record boundary in the window" and either widen the -window or refuse and tell the operator, rather than truncating blind. - -Untested: `tests/test_sidecar.py` covers partial-line, complete-line-no-newline, -intact and empty — never the no-newline-in-window case. - -### M3 — `--reset-state` can silently clobber the previous sidecar generation, and resets state before rotating (Medium) - -`sidecar.py:79-88` + `cli.py:201-205`. - -1. `rotate()` builds `messages-<%Y%m%dT%H%M%SZ>.jsonl` at one-second granularity and - uses `os.replace`, which overwrites without complaint. Two `--reset-state` runs in - the same wall-clock second (scripted loop, or a fast filter sweep) destroy the - first archive. `rotate()`'s own docstring says the old file "stays inspectable". -2. Ordering: `State.open(..., reset=True)` already wrote the zeroed cursor - (`state.py:95`) *before* `sidecar.rotate()` runs. If rotation fails (EACCES, - read-only dir, ENOSPC), cli exits via the `OSError` handler with the cursor reset - and the old sidecar still in place — exactly the mixed-filter-generation state - rotation exists to prevent. -3. `rotate()` does not `fsync_dir` the rename, unlike every other durability-critical - rename in the codebase (`state.py:133`). - -Fix: rotate first, then `State.open(reset=True)`; pick a non-colliding target name -(suffix a counter when the stamp exists); fsync the directory. - -### M4 — `resolve_entity` has no tests and two escape paths (Medium) - -`session.py:260-299`. - -- `except ValueError as e` at line 275 wraps both `int(target)` and - `client.get_entity(int(target))`. Any non-`ValueError` from that call — a - `TypeError`, or an RPC error such as `ChannelInvalidError`/`ChannelPrivateError` for - a cached-but-lost id — bypasses the username/dialog fallback entirely and lands on - `cli.main`'s generic `except Exception` as exit 1 with a traceback, instead of the - documented exit 5 with the "confirm this account is a member" guidance. -- `_find_in_dialogs` (`session.py:296`) iterates **every** dialog with no flood-wait - wrapper. It is the only network loop in the codebase not routed through - `with_flood_retry`/`aiter_with_flood_retry`, and `session.py:1-2` claims every - network call routes through the two primitives. A `FloodWaitError` on a - many-thousand-dialog account exits 1 instead of sleeping — before a single byte of - a 20-hour export. - -Verification gap: no test in the suite references `resolve_entity` or -`_find_in_dialogs`; `tests/test_cli_dispatch.py:88` monkeypatches the whole function -away. This is the largest untested reachable surface in `src/`. - -### M5 — argparse failures exit 2, colliding with the documented dry-run verdict (Low, but it breaks a published contract) - -`cli.py:227` calls `parse_args()` outside the `try`. Any argparse rejection — -including `_non_negative`'s `ArgumentTypeError` for `--limit -1` (`cli.py:98-105`) and -a non-integer `--limit` — exits **2** (verified against CPython argparse). `README.md:255` -says a bad argument is exit 1, and `README.md:256` reserves 2 for "`--dry-run` says it -will not fit". A wrapper script cannot distinguish "you typed the flag wrong" from -"buy a bigger disk". - -Fix: `parser.exit_on_error = False` / catch `argparse.ArgumentError`, or subclass and -override `error()` to exit 1. Untested either way. - -### M6 — `completed_at` is cleared on disk before any work is done (Low) - -`state.py:82` + `state.py:95`: `State.open` sets `completed_at = None` and immediately -`save()`s. If the run then ends without exhausting the sweep — Ctrl-C (130), exit 6 on -a flood ceiling, exit 3 on disk, an `Abort` from `download_one` — a previously -*completed* export is now permanently recorded as incomplete, and `state.py:116-117` -calls that field "the only way to answer 'did my 20-hour export finish?'". - -Fix: defer the clear until the first `commit()` actually moves the cursor, or keep a -`last_completed_at` alongside. - -### M7 — the "everything failed" guard is defeated by a single SKIPPED file (High; narrows plan open question 11) - -`downloader.py:448`: `if totals.failed and not (totals.downloaded or totals.skipped)`. - -Open question 11 already owns "no circuit breaker; a dead session walks the whole -history". This is a different, narrower hole in the H-D fix that shipped: `skipped` -in that condition means the guard only fires on a run where *nothing at all* -succeeded. Any resumed run that re-processes one post — guaranteed after a hard kill -mid-post, and after any `--reset-state` — produces at least one SKIPPED result. From -that point on, a media DC that fails every remaining file for the rest of the history -still gets `mark_completed()` (`downloader.py:458`) and a cursor at end-of-history. -The operator's export reads as finished over a tree that is missing most of it, and -`README.md:15-16`'s "re-run and it resumes" does nothing because the cursor is at the -end. - -`tests/test_download_decisions.py:457` proves the guard for the fresh-run case only; -no test injects a SKIPPED result alongside universal failure. - -Options (a threshold judgement, so not picked here): -- drop `skipped` from the condition — but then a legitimate resume whose tail is all - SKIPPED plus a couple of genuinely-deleted media stops being marked complete; -- gate on a ratio (`failed > downloaded`) or on consecutive failures, which is - open question 11's circuit breaker arriving anyway. - -### M8 — a refreshed file reference can be thrown away unused (Low) - -`downloader.py:308-319`. The `FileReferenceExpiredError` arm consumes a retry slot. -If expiry lands on attempt 3 of 3, `continue` exits the loop and the message is -reported as `exhausted 3 attempts` — the freshly refreshed reference is never -actually used. The docstring at line 309 says a 20-hour run *will* hit this arm, so -the tail case is not hypothetical. - -Fix: don't count the refresh as an attempt (e.g. `for attempt in ...` over a small -budget that the refresh path extends by one), or retry immediately after refresh -before falling through. - ---- - -## Worth doing (optimization — a long-running bulk exporter) - -### O1 — `--since` still sweeps the history from message id 1 (largest win) - -`traversal.py:193-194` always builds `client.iter_messages(entity, reverse=True, -offset_id=since)` and never passes `offset_date`. Verified against the pinned -Telethon 1.44.0 (`telethon/client/messages.py` `_MessagesIter._init`, lines 52-58): -under `reverse=True`, when `offset_id` is falsy and `offset_date` is unset it forces -`offset_id = 1` — i.e. the beginning of history — and the comment on line 56 states -"offset_id has priority over offset_date". So on a first run with `--since`, every -message before the cutoff is fetched over the wire and dropped locally by -`keep()` (`traversal.py:160-163`). - -Cost: one `messages.getHistory` per 100 messages. `--since` covering the last month -of a 1M-message group is ~9,900 round trips of pure waste — hours of wall clock and -a materially higher chance of the flood-wait escalation the whole design is built to -avoid. - -Caveat that must be handled, not ignored: entering the sweep at a date boundary can -land *inside* an album, and `_close` (`traversal.py:237`) would then take `post_id` -from the wrong member — breaking Invariant 1. Under the current -`offset_id`-from-cursor scheme that is unreachable (the cursor is always a -`max_message_id`, never mid-album), so `offset_date` introduces the risk. Any -implementation needs to either snap back to the album's first member or accept and -document the boundary post. Recommend measuring the win on the real group first -(phase 7) before taking on that complexity. - -### O2 — `--until` never terminates the sweep - -Same call site. Once `msg.date > filters.until`, `keep()` drops every remaining -message (`traversal.py:163`) but `iter_posts` keeps paging to end-of-history, yielding -empty Posts so the cursor advances. Same order-of-magnitude waste as O1, for zero -downloaded bytes. - -An early break is cheap (flush `buf`, `return`) and is *safer* than O1 — the cursor -simply stops at the last in-range post, and re-running with the same `--until` -re-sweeps one page and stops. - -Caveat: it assumes message date is monotonic in message id. That is true for normal -posting but not guaranteed for imported history (`messages.importChatHistory` / -"import from WhatsApp" produce old dates on new ids). Given this project's explicit -no-silent-gaps stance, gate the break on a small tolerance or verify the group has no -imported range before shipping it. Flagging rather than recommending unconditionally. - -### O3 — the cursor commit could be time-gated unconditionally - -`downloader.py:432-435`: `wrote` forces a `state.save()` on every post that -downloaded or failed a file. Each `save()` is a full JSON re-serialize plus two -fsyncs (`state.py:127-133`), and `run_download` adds `fsync_dir(target_dir)` -(`downloader.py:420`) and `sidecar.fsync()` (`downloader.py:426`) — roughly 4 fsyncs -per post. - -The `wrote` clause is not load-bearing for correctness: a cursor that lags is always -safe (Invariant 2 forbids it pointing *ahead*), and the only cost of lagging is -re-processing a few posts whose files are then existence-deduped for free -(`downloader.py:283-285`). Gating all commits on `COMMIT_INTERVAL_S` would cut state -writes by ~2 fsyncs/post at no correctness cost. Meaningful on a group of many small -files or on network/rotational storage; negligible on NVMe with large media, so -measure before bothering. - -Related and *not* actionable: all of this fsync/statvfs/rglob work is synchronous on -the event loop, which serializes it against Telethon's receive loop and keepalive. -The obvious remedy (`asyncio.to_thread`, `run_in_executor`) is explicitly forbidden by -`tests/test_no_concurrency.py:20-24`, and that guard encodes a locked plan decision. -So the actionable lever is "do fewer fsyncs", not "move them off the loop". I checked -whether the stall can itself drop the connection: `telethon/client/updates.py:516-544` -sends keepalive pings fire-and-forget with no response deadline, so a multi-second -fsync stall degrades throughput but does not by itself force a reconnect. - -### O4 — `sweep_part_files` walks the entire export tree at every startup - -`downloader.py:257`: `root.rglob(f"*{PART_SUFFIX}")`. On a mature export (hundreds of -thousands of files across as many post dirs) this is a full recursive walk before the -first byte, seconds-to-minutes on a cold cache or a network filesystem, on every run -including no-op resumes. A `.part` can legitimately be anywhere, so it cannot simply -be scoped — but it can be skipped when the previous run recorded a clean exit -(`completed_at`, or a new "exited cleanly" flag), since a clean exit leaves no `.part` -behind by construction. - -### O5 — cheap redundancies - -- `downloader.py:284` stats the target, then `_result` (`downloader.py:378`) stats it - again. Two `stat()` per already-present file; on a `--reset-state` re-drive of a - 100k-file export that is 100k avoidable syscalls. -- `estimate.py` reads the state file three times for one dry run: `resume_from` - (line 111), `_mode_line` → `stored_filters_differ` (line 94). Also two independent - copies of the same `data.get("filters") != filters.to_state()` comparison - (lines 95 and 114), with a third semantic copy in `state._assert_compatible` - (line 161). - ---- - -## Optional (simplification / DRY / hardening) - -- **Duplicated helper.** `session._nearest_existing_dir` (line 96) and - `estimate._measurable_dir` (line 66) are the same walk-up-to-an-existing-dir - function with different fallbacks (`None` vs `Path.cwd()`). One owner, two callers. -- **Dead code.** `downloader.py:173` `raise AssertionError("unreachable: ...")` is - genuinely unreachable — the loop returns at `"TiB"` because of the `or unit == "TiB"` - guard on line 170. The plan's session-2 log claims dead code in `human_bytes` was - already removed; this line is what remains. -- **`title.txt` is the one untrusted string written unsanitized.** `write_title` - (`downloader.py:475-479`) writes the server-supplied, admin-editable group title - raw. `paths.py:30-42` goes to real trouble to strip `Cc`/`Cf` (ANSI escapes, U+202E - RLO) out of *filenames* for exactly this threat, and `cli.py:163` logs the title - through `%r` so the log is safe — but `cat title.txt` feeds those bytes straight to - the operator's terminal. `paths.sanitize` already exists; reusing it here (or at - least stripping `_STRIPPED_CATEGORIES`) closes the last instance of the class. - Also `write_text` is a truncate-then-write, so a crash leaves an empty or partial - `title.txt` — the only non-atomic write in a codebase that is otherwise scrupulous - about it. -- **`Filters._warned`** (`traversal.py:80`) is a mutable log latch bolted onto an - otherwise pure, frozen, hashable value object that gets serialized into the state - file. It works (the set is excluded from `compare`, so hashing and `to_state` are - unaffected), but it means a `Filters` instance is single-use and couples a logging - concern into the stored filter identity. A module-level latch in `traversal` would - be less surprising. -- **`exclusive` contention message can print `?`.** `cli.py:80-81`: the contender - reads the lock file while the holder may be between `truncate()` and `write()` - (`cli.py:89-90`), yielding the `"?"` holder. Cosmetic, but it degrades the one - message whose whole purpose is to answer "who is holding this?". -- **flock on network filesystems.** `README.md:238-240` presents the lock as - unconditional. `fcntl.flock` semantics on NFS depend on the mount and server; a - bulk media exporter pointed at a NAS is a plausible deployment. Worth one sentence - in the README rather than a code change. -- **`_real_run` mutates the export tree before the state validation.** `cli.py:196-197` - runs `sweep_part_files` and `write_title` before `State.open` can refuse with exit 7. - `cli.py:199` claims "State first … before the sidecar is touched", which is true of - the sidecar only. Harmless in practice (`.part` files are always disposable, the - title is idempotent), but the comment overstates it. -- **`closed: set[int]`** (`traversal.py:190`) accumulates one `grouped_id` for the - whole history. Bounded and small (tens of MB at a million messages), correct as is — - noting it only because it is the codebase's one monotonically growing structure. - ---- - -## Checked and confirmed correct — do not re-flag - -- **`offset_id` as an exclusive lower bound with `reverse=True`.** Verified against - the pinned wheel: `_MessagesIter._init` lines 52-58 do `offset_id += 1`, - `_update_offset` (line 262-265) does it again per page, and `_message_in_range` - (line 249-251) never consults `min_id` in the reverse branch. `traversal.py:198`'s - belt-and-braces `msg.id <= after_id` guard is genuinely redundant, and worth keeping. -- **Flood-wait regeneration across a mid-album pause.** `aiter_with_flood_retry` - (`session.py:181-200`) rebuilds from the last *yielded* id while `iter_posts` keeps - `buf` alive across the pause. No member is skipped or replayed, and `prev` cannot - false-positive after the restart. -- **No exception can be thrown *into* the sweep generator by the consumer.** An - exception in `run_download`'s loop body closes the generator (GeneratorExit) rather - than surfacing at the `yield item` inside `except FloodWaitError`, so the retry arm - cannot swallow an `Abort` and silently restart the sweep. I specifically went - looking for this. -- **No unbounded message accumulation.** `iter_posts` is a true generator; `buf` is - capped at `MAX_ALBUM_BUFFER = 20` (`traversal.py:214-217`); neither run mode - materializes a message list. The "holds all messages in a list" failure mode is - absent. -- **Filename collisions are structurally impossible** inside a post dir, even though - `_truncate_utf8` (`paths.py:152`) is lossy: the `{msg.id}_` prefix - (`paths.py:103`) is unique per message and one message carries at most one media. -- **The sanitizer has no hole I could find.** `..`, `...`, `/`, `foo/`, `C:/x/y`, - `a/../b`, `./..` as an extension, and a NUL-bearing extension all reduce to a leaf - or to the fallback. Worst-case component length is ~229 bytes against ext4's 255. -- **Partial-post recovery is consistent.** When `download_one` raises `Abort` - mid-album, files already `os.replace`d are on disk with no sidecar record and no - cursor advance; the resumed run re-processes the post, reports them SKIPPED, and - writes the record. Nothing is lost or double-counted. -- **`_sender_name` performs no RPC.** `sidecar.py:141` reads the `Message.sender` - property, which returns the cache populated by `_finish_init` - (`telethon/client/messages.py:205`) — not `get_sender()`. The N+1 the docstring - warns about is genuinely avoided. -- **`media_kind`'s `web_preview` guard is correctly ordered** (`media.py:31`): a - message carrying `MessageMediaWebPage` is excluded before `.photo`/`.document` can - misclassify it. -- **A cursor can never land mid-album** under the current `offset_id` scheme, because - every committed value is a `max_message_id` of a closed group. (This stops being - true if O1 is implemented — see that finding.) -- **`assert_not_committable`'s trailing-slash retry** is gated to `is_dir=True` - (`session.py:136`) and cannot excuse a session *file*, exactly as the comment claims. -- **`sanitize_ext` is an identity transform for ordinary extensions**, so no - previously downloaded file is orphaned — re-confirmed, since dedupe depends on it. - -Two lower-confidence notes, called out so they are not mistaken for clean bills: -`assert_session_private` and `assert_not_committable` raise `SystemExit` -(`session.py:93`, `session.py:144`), a `BaseException` that bypasses every handler in -`cli.main` and reaches the interpreter — the message prints and the process exits 1, -which happens to be defensible but is the only place in the codebase that routes -around the `Abort`/exit-code contract. And `_login`'s `sign_in` retry -(`session.py:231-233`) has no handling for a wrong code (`PhoneCodeInvalidError`), -so a typo on first login exits 1 with a traceback rather than a re-prompt. - ---- - -## Verification gaps (behaviour with no test that plausibly regresses) - -Ranked by reachability × blast radius: - -1. `session.resolve_entity` and `_find_in_dialogs` — zero tests; monkeypatched away in - the one end-to-end file. Covers M4. -2. The session lock's existence and placement — `tests/test_lock.py` never constructs - it. Covers M1. -3. `mark_completed` with SKIPPED-plus-universal-failure — covers M7. -4. `Sidecar.repair` with no newline in the tail window — covers M2. -5. argparse rejection exit codes (`--limit -1`, `--limit x`) — covers M5. -6. `Sidecar.rotate` colliding on the same-second stamp — covers M3. -7. `write_title` with a control-character/RLO title, and `title.txt` after a - truncate-then-crash. -8. `completed_at` surviving an interrupted re-run of a finished export — covers M6. -9. `FileReferenceExpiredError` arriving on the final attempt — covers M8. -10. `FloodWaitError` raised from `Fetcher.refresh` (the `with_flood_retry` wrapper on - `get_messages` is unexercised). - ---- - -## Plan status - -Phases 2-6 read as complete and consistent with the code; phase 1 and phase 7 remain -correctly marked pending on live credentials. Open questions 9, 11 and 12 are still -genuinely open and are the right place for the FAILED-retry, circuit-breaker and -0-byte-document decisions — M7 is a *refinement* of 11's scope, not a replacement. -Recommend the lead add M1 (session-lock ordering, with the `--dry-run` README -contradiction) as a blocking item before phase 7, since phase 7 is the first time two -real clients could touch one session file. - -## Unresolved questions - -1. `--dry-run` concurrent with a real run: should it take the session lock (safe, but - breaks the README's "always") or should the README be narrowed to "with its own - `--session`"? Product call. -2. M7 threshold: drop `skipped` from the guard, or move straight to open question 11's - consecutive-failure circuit breaker? Both change when a run is called complete. -3. O1/O2: is a server-side date offset worth the mid-album-entry risk to Invariant 1, - and is date/id monotonicity safe to assume for the target group (any imported - history)? Both are better answered with phase 7's real numbers. -4. Is `title.txt` intended to be terminal-safe? It is the only untrusted string - written raw, and the answer decides whether `write_title` reuses `paths.sanitize`. diff --git a/plans/reports/optimize-260822-2311-review-and-optimization.md b/plans/reports/optimize-260822-2311-review-and-optimization.md deleted file mode 100644 index 08145c6..0000000 --- a/plans/reports/optimize-260822-2311-review-and-optimization.md +++ /dev/null @@ -1,129 +0,0 @@ -# Review + optimization pass — telegram-exporter - -Two agents ran in parallel (full-codebase review, suite verification); findings -cross-checked against source before acting. Suite: **204 → 214 passing**, -coverage held at **93%**. Every fix below has a test that fails without it -(verified by stashing the fix and re-running). - -Companion reports: `code-review-260822-2259-full-codebase-audit.md`, -`tester-260822-2259-suite-verification.md`. - -## Baseline - -Suite was not runnable — no pytest in the environment. Working setup, now in the -README: - -```bash -python3 -m venv .venv -.venv/bin/pip install -r requirements.txt -e ".[dev]" -.venv/bin/python -m pytest -``` - -All 7 safety-critical domains already had real tests (resume, partial files, path -sanitization, album grouping, flood waits, pagination, locking). The correctness -core survived adversarial reading: sanitizer, `safe_join`, album grouping, cursor -ordering, error taxonomy, no unbounded message accumulation, `_sender_name` doing -zero RPC. - -## Fixed — correctness - -**Session lock taken after the resource it guards** (`cli.py`). The lock sat -inside `connected_client`, so `connect()` + `_login()` had already written the -shared SQLite session. Two runs on the default session each passed their own -per-root lock, both opened the same database, and the loser exited 3 *after* -causing the corruption it reported. Lock now wraps `connected_client` in `_run`; -`assert_not_committable` runs first so a refused path leaves no lock file. -`--dry-run` is inside the lock too — it opens the same session file. README's -"dry-run can always run alongside an export" narrowed to "with its own -`--session`", per your call. - -**Completion guard disabled by one skipped file** (`downloader.py`). -`if totals.failed and not (downloaded or skipped)` required that *nothing* had -succeeded, so any resume across a partly-complete export could fail every -remaining file and still set `completed_at` at end-of-history — a broken export -reading as finished. Now refuses when `failed > downloaded + skipped`. A -legitimate all-present tail outnumbers its own stray failures and still -completes. - -**Renewed file reference never fetched** (`downloader.py`). Expiry on the final -attempt consumed the last slot, so the refreshed reference was discarded and the -message reported "exhausted 3 attempts" having been tried twice. The retry loop -no longer counts a refresh as an attempt; the `refreshed` latch still bounds it. -Docstring says a 20-hour run *will* reach this arm. - -**Sidecar repair could eat 1 MiB of valid records** (`sidecar.py`). A partial -record larger than the scan window contains no newline, so `rfind` returned -1 -and the file was truncated to `end - window` — mid-record, still unreadable, one -window shorter. Window now grows until a newline is found. - -**`--reset-state` ordering + rotate collision** (`cli.py`, `sidecar.py`). The -zeroed cursor became durable before the old sidecar rotated, leaving exactly the -mixed-generation state rotation exists to prevent; rotation now precedes -`State.open` (safe only on this path, which has no compatibility check to fail). -Two resets in one second silently clobbered the first archive via `os.replace` — -now serialized, and the rename is `fsync`'d like every other rename here. - -**`title.txt` written raw and non-atomically** (`downloader.py`, `paths.py`). The -one untrusted string reaching disk unfiltered: `cat title.txt` fed ANSI escapes -and U+202E overrides to the operator. Now stripped of the same categories -`paths.sanitize` strips (new `strip_unprintable`, shared set so the two rules -can't drift), and written tmp → fsync → replace → fsync(dir). - -## Fixed — cost - -**Per-post fsync on already-complete posts** (`downloader.py`). The post -directory was fsynced every post, including posts where every file was already -present and nothing was renamed. A resume across a mostly-complete export paid -one fsync per post to re-durably-record a directory entry an earlier run already -had. Now gated on an actual download. - -**Triple `stat` per already-present file** (`downloader.py`). `exists()` + -`stat()` + a third inside `_result`, on the single most common path of any -resume. One `stat` now, reused as the recorded size. - -Note: moving fsync off the loop entirely is **not** available — -`tests/test_no_concurrency.py` forbids `to_thread`/`run_in_executor` by AST scan, -a deliberate decision (flood-wait escalation). Fewer fsyncs is the only lever. - -## Not changed — deferred by decision - -**Sweep starts at message id 1 under `--since`.** Verified in the pinned wheel -(`_MessagesIter._init`): `reverse=True` + no `offset_date` → `offset_id = 1`. -`--since` on a month of a 1M-message group burns ~9,900 discarded round trips. -`--until` never breaks early. Both carry invariant risk — `offset_date` can start -mid-album and break filter-invariant post identity; an `--until` break assumes -date/id monotonicity, false for imported history. Recorded as step **12b** in -phase 7: measure the swept/kept ratio first, decide with numbers. - -## Not changed — reported only - -- `resolve_entity` catches `ValueError` only; a TypeError or RPC error skips the - whole fallback chain → exit 1 + traceback instead of the documented 5. - `_find_in_dialogs` is the only network loop not behind a flood primitive. - Largest untested reachable surface in `src/` (monkeypatched away in tests). -- `SystemExit` in `session.py` is a `BaseException` — bypasses every `cli.main` - handler and lands on interpreter exit 1, routing around the `Abort` contract. -- `_login` has no `PhoneCodeInvalidError` handling: a typo on first login exits 1 - with a traceback instead of re-prompting. -- `completed_at` is cleared at `State.open` before any work, so an interrupted - re-run of a finished export reads incomplete forever. -- `_nearest_existing_dir` ≡ `_measurable_dir`, differing only in fallback. -- Dead `raise AssertionError` in `human_bytes` (loop always returns at TiB). -- `exclusive` can print holder `?` — races the holder's truncate/write. -- `Filters._warned` is a mutable log latch on a frozen, hashable, serialized - value object. Works; surprising. - -## Docs - -README: dry-run/session-lock semantics corrected in both places; `flock`-on-NFS -caveat added; exit code 2 documented honestly (argparse exits 2, colliding with -dry-run SHORT — table previously promised 1); venv creation added to setup. - -## Unresolved questions - -1. `resolve_entity` hardening — widen the `except` and route - `_find_in_dialogs` through a flood primitive? Needs a call on whether dialog - scanning should retry at all. -2. `SystemExit` in `session.py` → convert to `Abort` for one exit-code contract? -3. `completed_at` cleared eagerly — keep (a re-run is genuinely incomplete) or - only clear once work begins? diff --git a/plans/reports/research-telethon-mtproto-260811-1411-telegram-group-media-exporter-report.md b/plans/reports/research-telethon-mtproto-260811-1411-telegram-group-media-exporter-report.md deleted file mode 100644 index ea989ae..0000000 --- a/plans/reports/research-telethon-mtproto-260811-1411-telegram-group-media-exporter-report.md +++ /dev/null @@ -1,325 +0,0 @@ -# Research Report: Telegram Group Media Exporter (Telethon / Python) - -**Conducted:** 2026-08-11 14:11 (Asia/Saigon) -**Target env:** Linux ARM64 (aarch64), headless, Python 3.12.3, ffmpeg 6.1.1, 41 GB free disk -**Stack (confirmed by user, not re-evaluated):** Telethon + Python, album-aware folder layout -**Scope in v1:** resume+dedupe, `messages.jsonl` sidecar, type/date/size filters, dry-run size estimate - ---- - -## Table of Contents - -1. [Executive Summary](#executive-summary) -2. [Methodology](#methodology) -3. [Findings](#findings) - - [1. Auth & Session Persistence](#1-auth--session-persistence) - - [2. History Traversal & Resume](#2-history-traversal--resume) - - [3. Album Grouping via grouped_id](#3-album-grouping-via-grouped_id) - - [4. Media Download & Filenames](#4-media-download--filenames) - - [5. Flood Waits & Concurrency](#5-flood-waits--concurrency) - - [6. Dry-Run Size Estimation](#6-dry-run-size-estimation) - - [7. Packaging & CLI](#7-packaging--cli) -4. [Proposed Architecture](#proposed-architecture) -5. [Common Pitfalls](#common-pitfalls) -6. [Confidence Ledger](#confidence-ledger) -7. [Next Steps](#next-steps) -8. [Unresolved Questions](#unresolved-questions) -9. [References](#references) - ---- - -## Executive Summary - -Three findings drive the whole design. - -**Concurrency is the wrong instinct.** Telethon does not parallelize file downloads internally, and the docs/issues are explicit that parallel downloads only make `FloodWaitError` arrive *sooner*. Worse, retrying during an active flood wait is itself counted as a violation — a 7s wait can escalate past 4 hours within an hour of retry-looping. **Default concurrency must be 1**, with explicit flood-wait sleeping. This kills any worker-pool design before it is written. - -**Telethon has no byte-level download resume.** `download_media` writes from offset 0 every call. Resumability must therefore be built at *message granularity*: download to a `.part` file, `os.replace()` atomically on completion, and treat any surviving `.part` as incomplete garbage to be re-fetched. Combined with an oldest→newest sweep (`reverse=True`) and a persisted last-completed message id, this gives a monotonic, crash-safe cursor. - -**Albums need buffering, not lookups.** An album is N separate messages sharing a `grouped_id`; there is no "fetch the album" API. The common StackOverflow pattern (scan ±10 message ids around a post) is for the single-post case and is wasteful here. Since we sweep every message anyway, group by buffering consecutive same-`grouped_id` messages and flushing when the id changes. - ---- - -## Methodology - -- Sources consulted: 5 research passes (1 doc fetch, 4 web searches) → ~20 distinct sources -- Gemini CLI: **absent on this host**, fell back to WebSearch per skill config -- Primary authority: `docs.telethon.dev` (1.44.0 stable) + LonamiWebs/Telethon GitHub issues -- Key terms: `grouped_id album`, `FloodWaitError bulk download`, `StringSession vs SQLiteSession`, `iter_messages reverse offset_id`, `pyproject.toml CLI 2026` -- **Limitation:** `docs.telethon.dev/en/stable/modules/client.html` is a single very large page; the fetch truncated before the full `iter_messages` and `download_media` signatures. Items depending on it are flagged in the [Confidence Ledger](#confidence-ledger). - ---- - -## Findings - -### 1. Auth & Session Persistence - -Bot API is structurally disqualified (no history access; 20 MB download cap) — MTProto user account is the only path. `api_id`/`api_hash` from my.telegram.org. - -| | SQLiteSession (default) | StringSession | -|---|---|---| -| Storage | on-disk `.session` SQLite DB | in-memory, serializable to base64 string | -| Best for | long-lived local tool | serverless / env-var-injected secrets | -| Gotcha | **must call `disconnect()` before exit** or state is not flushed | must persist the string yourself | - -**Recommendation: SQLiteSession file**, `chmod 600`, path configurable, `.gitignore`d. This is a localhost tool with a persistent filesystem — StringSession solves a containerization problem we do not have (YAGNI). One-time interactive login (phone + code), every later run unattended. - -Security, verbatim from the docs: *"Never give the saved session file to anyone, since they would gain instant access to all your messages and contacts."* Treat `.session` as a credential — never commit, never log its path contents. - -**2FA is unresolved by the sources.** The searched docs did not cover the 2FA password flow. Telethon's `client.start(phone=..., password=...)` and `SessionPasswordNeededError` are the expected mechanism — **verify at implementation time**, and never accept the 2FA password as a CLI argument (it lands in shell history); prompt interactively or read from env. - -### 2. History Traversal & Resume - -`iter_messages(entity, ...)` with parameters `limit`, `offset_id`, `min_id`, `max_id`, `offset_date`, `filter`, `reverse` — all confirmed present, full semantics truncated in the fetch. - -**Design: sweep oldest→newest with `reverse=True`.** Rationale: message ids increase monotonically over time, so "highest fully-completed id" is a single-integer resume cursor that is always correct, even if the run is killed mid-album. A newest-first sweep would need an interval set instead. - -```python -# resume: continue after the last fully-completed message -async for msg in client.iter_messages(entity, reverse=True, offset_id=state.last_done_id): - ... -``` - -**Footgun to verify:** `offset_id` semantics *invert* with `reverse`. Newest-first, it is an upper bound (start below this id); with `reverse=True` it acts as a lower bound (start above this id). Confirm empirically with a 3-message probe before trusting the resume path — an off-by-one here silently re-downloads or silently skips. - -Pagination is handled internally (~100 messages/request); no manual paging needed. - -### 3. Album Grouping via `grouped_id` - -Confirmed: *"Albums are a way to send multiple photos or videos as separate messages with the same grouped identifier. It is possible to know that it's a grouped message thanks to `grouped_id`."* - -Album members arrive as **consecutive** messages in a chronological sweep. Grouping algorithm: - -``` -buffer = [] -for msg in sweep(reverse=True): - if msg.grouped_id is None: - flush(buffer); emit_single(msg) - elif buffer and msg.grouped_id != buffer[0].grouped_id: - flush(buffer); buffer = [msg] - else: - buffer.append(msg) -flush(buffer) # end of history -``` - -**Folder id = the lowest message id in the group** (first member, since we sweep ascending). Deterministic and resume-stable: a re-run recomputes the same folder name. - -**Resume interacts with albums.** Only advance `last_done_id` once *every* member of a group has landed — otherwise a crash mid-album leaves a half-folder that resume skips past. Simplest correct rule: commit the cursor at group-flush boundaries, not per file. - -Do **not** use the "iterate ±N ids around the post" pattern from issue #3788 — that solves the single-known-post case, and duplicates work in a full sweep. - -### 4. Media Download & Filenames - -Confirmed chunked download primitive: - -```python -async download_file( - input_location: hints.FileLike, - file: hints.OutFileLike = None, - *, - part_size_kb: float = None, - file_size: int = None, - progress_callback: hints.ProgressCallback = None, - dc_id: int = None, - key: bytes = None, - iv: bytes = None -) -> bytes | None -``` - -For our purposes the higher-level `download_media(message, file=path, progress_callback=...)` is correct — it wraps chunking and DC routing. `part_size_kb` tuning is a micro-optimization; skip it (YAGNI). - -**Filename/extension derivation:** `message.file` exposes `.name`, `.ext`, `.mime_type`, `.size`. Preference order: -1. `DocumentAttributeFilename` via `message.file.name` — real filename for documents -2. `message.file.ext` + mime type — for photos/voice/round video, which carry no filename -3. Fallback `bin` - -**Sanitize aggressively.** Filenames are attacker-controlled strings from arbitrary group members: strip `/`, `..`, NUL, leading dots, control chars, and cap length. Always join under the target dir and assert the resolved path stays inside it. A crafted `../../../.ssh/authorized_keys` is a real, cheap attack on a naive exporter. - -**No byte-level resume** (see Executive Summary). Pattern: - -``` -tmp = folder / f"{idx}_{name}.part" -download_media(msg, file=tmp) -os.replace(tmp, folder / f"{idx}_{name}") # atomic on same filesystem -``` - -**Dedupe check:** skip when the final file exists *and* its size equals `message.file.size`. Size-match is cheap and catches truncation; hashing every file is wasted I/O at multi-GB scale. - -### 5. Flood Waits & Concurrency - -The single most decision-relevant finding. - -- `client.flood_sleep_threshold` (default **60s**): Telethon auto-sleeps on flood waits ≤ threshold, raises `FloodWaitError` above it. -- *"The library does not download or upload files in parallel"* — no internal parallelism to inherit. -- *"The limiting factor in the long run are FloodWaitError, and using parallel download or uploads only makes them occur sooner."* -- **Escalation is real:** *"a wait that starts at 7 seconds can escalate past four hours within less than an hour of repeated restarts, because every retry during an active FloodWait is itself treated as another violation."* - -**Design rules:** -1. Sequential downloads. Default concurrency **1**. Do not ship a `--workers` flag in v1 — it is a footgun that trades a small speedup for hour-long lockouts. -2. Raise `flood_sleep_threshold` (e.g. 120s) so routine small waits are absorbed silently. -3. Catch `FloodWaitError` explicitly, `await asyncio.sleep(e.seconds + jitter)`, then continue. **Never** tight-retry, never restart the process to "get around" a wait. -4. Log every flood wait with its duration — it is the primary signal for whether the run is healthy. - -### 6. Dry-Run Size Estimation - -`message.file.size` gives bytes without fetching content. Dry-run = full `iter_messages` sweep with zero `download_media` calls: cheap (~100 messages/request, no file-DC traffic), and the flood-wait risk is low because history reads are far cheaper than file reads. - -Output: total media count, total bytes, breakdown by type, and **free-disk delta**. With 41 GB free on `/config`, an export that does not fit should fail loudly *before* the first byte, not at 97%. - -Caveat: for photos, `.size` reflects the size variant Telethon selects (largest by default); treat photo totals as a close estimate, document sizes as exact. - -### 7. Packaging & CLI - -Consensus from the packaging guides: `argparse` for simple tools with no dependencies, Click for complex multi-command CLIs, Typer for type-hint ergonomics (built on Click). - -**Recommendation: `argparse`.** One command, ~8 flags, no subcommands. Adding Typer means +2 transitive deps to save perhaps 25 lines. YAGNI/KISS. If subcommands ever appear, Typer is a contained swap. - -Packaging: `pyproject.toml` with `[project.scripts]` entry point (`tg-export = "telegram_exporter.cli:main"`) — the current standard, declarative and static. - -**`uv` is not installed on this host.** Options: `pip install -e .` inside a venv (works today, zero setup), or install `uv` first. The venv path is fine; nothing here needs uv. - ---- - -## Proposed Architecture - -``` - ┌──────────────┐ - │ cli.py │ argparse: --group --out --dry-run - │ │ --since/--until --types --max-size - └──────┬───────┘ - │ - ┌──────▼───────┐ - │ client.py │ auth, SQLiteSession, flood-wait wrapper - └──────┬───────┘ - │ - ┌───────────────────▼────────────────────┐ - │ sweep.py │ - │ iter_messages(reverse=True, │ - │ offset_id=cursor) │ - └───────────────────┬────────────────────┘ - │ messages, ascending - ┌───────────────────▼────────────────────┐ - │ grouper.py buffer by grouped_id │ - └───────────────────┬────────────────────┘ - │ logical posts - ┌─────────────┴─────────────┐ - │ │ - ┌────────▼────────┐ ┌────────▼────────┐ - │ estimator.py │ │ downloader.py │ - │ sum .file.size │ │ .part → rename │ - │ (dry-run only) │ │ size-match skip │ - └─────────────────┘ └────────┬────────┘ - │ - ┌─────────────▼─────────────┐ - │ state.py cursor + jsonl │ - │ commit at group boundary │ - └───────────────────────────┘ -``` - -Output layout (album-aware, per user decision): - -``` -exports// - 1042/ <- album, grouped_id shared, folder = first msg id - 0_photo.jpg - 1_photo.jpg - 2_video.mp4 - 1047/ <- standalone message - 0_document.pdf - messages.jsonl <- id, date, sender, caption, grouped_id, reply_to, files[] - .export-state.json <- last_done_id, counters -``` - ---- - -## Common Pitfalls - -| Pitfall | Consequence | Mitigation | -|---|---|---| -| Parallel downloads to "go faster" | Flood waits arrive sooner, escalate to hours | Sequential, concurrency 1 | -| Retrying during an active flood wait | 7s wait → 4h+ lockout | Sleep the full `e.seconds`, never restart to bypass | -| Forgetting `disconnect()` | SQLite session not flushed, re-login next run | `try/finally` or async context manager | -| Unsanitized `DocumentAttributeFilename` | Path traversal outside the export dir | Sanitize + assert resolved path is inside target | -| Advancing the resume cursor per file | Crash mid-album → permanently half-downloaded folder | Commit cursor only at group flush | -| Assuming `download_media` resumes | Silent truncated files | `.part` + atomic rename; treat stray `.part` as garbage | -| `offset_id` semantics under `reverse=True` | Silent skip or full re-download | Verify empirically with a 3-message probe | -| Committing the `.session` file | Full account takeover | `.gitignore`, `chmod 600` | -| No pre-flight size check | Fills 41 GB disk mid-run | Dry-run + free-space assertion | - ---- - -## Confidence Ledger - -Honest separation of what the sources confirmed from what needs checking in code. - -**Verified by sources:** -- `flood_sleep_threshold` default 60s, auto-sleep ≤ threshold -- Telethon does not parallelize file transfers; parallelism accelerates flood waits -- Flood-wait escalation from retries during an active wait -- `download_file` full signature incl. `part_size_kb`, `progress_callback`, `dc_id` -- `grouped_id` identifies album membership; albums are separate messages -- SQLiteSession vs StringSession trade-offs; `disconnect()` flush requirement -- `iter_messages` accepts `reverse`, `offset_id`, `min_id`, `max_id`, `offset_date`, `filter`, `limit` -- `[project.scripts]` in `pyproject.toml` as the entry-point standard - -**Not confirmed — verify during implementation:** -- Exact `offset_id` direction under `reverse=True` (**highest-risk item**; breaks resume silently) -- 2FA / `SessionPasswordNeededError` flow — sources did not cover it -- `message.file` attribute surface (`.name`/`.ext`/`.size`/`.mime_type`) — strongly indicated by the custom-types docs, not read verbatim -- Whether `download_media` returns the final path string in all media branches -- Photo `.size` accuracy vs the actually-downloaded variant - ---- - -## Next Steps - -1. **Spike first, build second** (~30 min): a throwaway script that logs in, iterates 20 messages of a real group, prints `id / grouped_id / file.name / file.size`, and downloads one album. Settles all four unverified items above at once. -2. Run `/ck:plan` to phase the implementation against the architecture sketch. -3. Phase order: auth+session → sweep+resume → grouper → downloader → jsonl sidecar → filters → dry-run estimator. -4. Add `.session`, `exports/`, `.export-state.json` to `.gitignore` **before** the first login. - ---- - -## Unresolved Questions - -1. **Group scale?** Unknown message/media count. If the group is >40 GB the 41 GB free disk is the binding constraint and dry-run becomes mandatory before any real run. -2. **2FA enabled on the account?** Changes the login flow and whether an interactive prompt is required on first run. -3. **Group access type** — public/private, and is the account already a member? Private groups need the account joined; `get_entity` resolution differs for public username vs invite-link groups. -4. **Should non-media messages appear in `messages.jsonl`?** Text-only messages give context to captions and replies, but inflate the sidecar. Default assumption: media-bearing messages only. -5. **Deleted/expiring media and service messages** — skip silently or record as gaps in the sidecar? -6. **Retention of `.part` files** on a failed run — auto-clean at startup, or leave for inspection? - ---- - -## References - -### Official Documentation -- [Client class — Telethon 1.44.0](https://docs.telethon.dev/en/stable/modules/client.html) — `download_file` signature, `iter_messages` params -- [RPC Errors — Telethon 1.44.0](https://docs.telethon.dev/en/stable/concepts/errors.html) — FloodWaitError semantics -- [FAQ — Telethon 1.44.0](https://docs.telethon.dev/en/stable/quick-references/faq.html) — no-parallel-transfer statement -- [Session Files — Telethon 1.44.0](https://docs.telethon.dev/en/stable/concepts/sessions.html) — SQLite vs String sessions -- [Sessions module — Telethon 1.44.0](https://docs.telethon.dev/en/stable/modules/sessions.html) -- [Custom package (types) — Telethon 1.44.0](https://docs.telethon.dev/en/stable/modules/custom.html) — `Message.file` wrapper -- [Telethon 1.44.0 full PDF](https://media.readthedocs.org/pdf/telethon/stable/telethon.pdf) — offline reference for the truncated pages -- [Creating and packaging command-line tools — PyPA](https://packaging.python.org/en/latest/guides/creating-command-line-tools/) -- [Writing your pyproject.toml — PyPA](https://packaging.python.org/en/latest/guides/writing-pyproject-toml/) - -### Issues & Community -- [Download all media in a post by grouped_id — Telethon #3788](https://github.com/LonamiWebs/Telethon/issues/3788) -- [Detect a grouped message (album) — Telethon #1216](https://github.com/LonamiWebs/Telethon/issues/1216) -- [FloodWaitError in client.download_media — Telethon #1426](https://github.com/LonamiWebs/Telethon/issues/1426) -- [FloodWaitError during document download — Telethon #206](https://github.com/LonamiWebs/Telethon/issues/206) -- [Converting StringSession to SQLiteSession — Telethon #3660](https://github.com/LonamiWebs/Telethon/issues/3660) -- [Telethon — DeepWiki](https://deepwiki.com/LonamiWebs/Telethon) - -### Further Reading -- [Fix Telegram FloodWait Error Fast](https://membertel.com/blog/how-to-fix-telegram-floodwait-error-fast/) — flood-wait escalation behavior -- [telebackup — multi-connection Telegram downloader](https://github.com/xwc9527/telebackup) — prior art for parallel/checkpoint downloading; **contrary to our sequential decision**, review only if throughput becomes a proven problem -- [Packaging Python CLI apps the modern way with uv](https://thisdavej.com/packaging-python-command-line-apps-the-modern-way-with-uv/) -- [Building Python CLI tools: Click, Typer, argparse](https://inventivehq.com/blog/python-cli-tools-guide) - -### Glossary -- **MTProto** — Telegram's native protocol; full user-account API access (vs the restricted Bot API) -- **DC** — Telegram data center; large files may live on a DC other than the session's home DC -- **`grouped_id`** — shared identifier marking messages that belong to one album -- **FloodWait** — server-imposed cooldown after too-rapid requests; escalates on retry diff --git a/plans/reports/spike-260811-1411-telethon-api-verification-report.md b/plans/reports/spike-260811-1411-telethon-api-verification-report.md deleted file mode 100644 index b883beb..0000000 --- a/plans/reports/spike-260811-1411-telethon-api-verification-report.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: "Phase 1 — Telethon API Verification Findings" -phase: 1 -status: partial -telethon_version: "1.44.0" -created: "2026-08-12" ---- - -# Phase 1 — Telethon API Verification Findings - -Two sources of evidence appear below and are labelled separately: - -- **SOURCE** — settled offline by reading the installed `telethon` 1.44.0 package. - Deterministic, and stronger than a single live observation for questions about - parameter semantics, because it shows the code path rather than one sample of - its behavior. -- **LIVE** — requires `TG_API_ID`/`TG_API_HASH`, an interactive login, and a real - group. **Not yet run.** Run `spike/probe_telethon_api.py` and paste its output - into the corresponding section. - -Environment: Python 3.12.3, aarch64, Telethon **1.44.0** (wheel, no build step), -pinned in `requirements.txt`. - -## Confidence ledger - -| Probe | Question | Status | Answer | -|---|---|---|---| -| P1 | `offset_id` direction/inclusivity under `reverse=True` | **CONFIRMED (SOURCE)** | Exclusive lower bound: results satisfy `id > offset_id` | -| P2 | 2FA / login flow | **PENDING (LIVE)** | — | -| P3 | `message.file` surface; which kinds lack `.size` | **PENDING (LIVE)** | — | -| P4 | `download_media` return value; file-object support | **PARTIAL (SOURCE)** | File objects are accepted; return value pending live confirmation | -| P5 | photo declared `.size` vs bytes on disk | **PENDING (LIVE)** | — | -| P6 | album adjacency across a pagination boundary | **PENDING (LIVE)** | Runtime tripwire implemented regardless | -| — | error classes in phase 5's taxonomy exist | **CONFIRMED (SOURCE)** | All 15 present | - -## P1 — `offset_id` under `reverse=True` — CONFIRMED (SOURCE) - -`telethon/client/messages.py`, `_MessagesIter._init`: - -```python -if self.reverse: - if offset_id: - offset_id += 1 - elif not offset_date: - offset_id = 1 -``` - -and `_update_offset`, run after every page: - -```python -self.request.offset_id = last_message.id -if self.reverse: - self.request.offset_id += 1 # "We want to skip the one we already have" -``` - -**`offset_id` is an exclusive lower bound under `reverse=True`.** `offset_id=N` -yields messages with `id > N`; `N` itself is excluded. `offset_id=0` starts from -the beginning of history. - -`min_id` was compared as the plan asked: `_message_in_range`'s `reverse` branch -tests `message.id <= self.last_id or message.id >= self.max_id` and **does not -consult `min_id` at all**, so `offset_id` is the correct parameter for a -chronological sweep, not `min_id`. - -**Consequence, implemented:** the cursor stores the **last handled id** and is -passed as `offset_id`. `iter_posts` additionally carries an explicit -`if msg.id <= after_id: continue` guard, so the sweep stays correct even if this -internal detail changes in a future Telethon release — -`src/telegram_exporter/traversal.py`. - -## Error-class inventory — CONFIRMED (SOURCE) - -Every exception phase 5's taxonomy catches exists in 1.44.0 (checked against -`telethon.errors` and `telethon.errors.rpcerrorlist`). A missing name would have -been an import-time crash on the first run: - -`FloodWaitError`, `FileReferenceExpiredError`, `MediaEmptyError`, -`FileIdInvalidError`, `FilerefUpgradeNeededError`, `AuthKeyUnregisteredError`, -`AuthKeyDuplicatedError`, `SessionRevokedError`, `UserDeactivatedBanError`, -`ChannelPrivateError`, `ChatForbiddenError`, `ChannelInvalidError`, -`TimedOutError`, `RpcCallFailError`, `SessionPasswordNeededError`. - -Constructor signature for all of them is `(request)`, with `FloodWaitError` -taking `(request, capture=0)` where `capture` becomes `.seconds`. The test suite -constructs them this way. - -## P4 — `download_media` file-object support — PARTIAL (SOURCE) - -`Message.file` wraps `photo or document` and is `None` for other media types -(polls, games, none), which is why `media.media_kind()` classifies from the media -type rather than from `.file`. - -Telethon accepts a stream for `file=`, so phase 5 holds the fd and `fsync`s it -before `os.replace`. The plan's fallback — download to a path, then re-open and -fsync — is not needed unless the live probe contradicts this. - -**Still pending (LIVE):** the exact return type/value of -`download_media(msg, file=)`. - -## P3 / P5 note — why the implementation does not wait on these - -Both remaining questions are about *sizes*, and the resolved design is -answer-independent by construction: - -- `media.media_size()` returns `int | None`, and each of the four consumers has a - stated rule for `None` (include+warn / accept / separate tally / treat as 0). - Tested in `tests/test_traversal.py`. -- The download validator never raises: a photo mismatch is accepted as a size - variant, and a document/video/audio mismatch retries then reports FAILED. - Tested in `tests/test_download_decisions.py`. - -P3's answer improves the `--max-size` warning's accuracy; P5's improves the -dry-run label and phase 7's reconciliation tolerance. Neither can invalidate code -that already treats `None` as a first-class case. - -## P2 / P6 — PENDING (LIVE) - -- **P2** — record whether `SessionPasswordNeededError` is raised, and confirm a - second run is non-interactive. The 2FA password is `getpass`-only, enforced by - `tests/test_logging.py`. -- **P6** — the probe requires ≥250 messages swept and asserts it rather than - passing vacuously. Coverage is probabilistic either way; the deterministic - guarantee is the runtime tripwire in `traversal._close`, which raises - `AlbumSplitError` when a `grouped_id` reopens after its run closed. Tested for - both `A B A` and `A B C A`. - -## Cleanup owed after running the spike - -Deleting `spike/probe.session` does **not** revoke server-side authorization. -Terminate the probe session in Telegram → Settings → Devices. This is phase 7 -step 13 and the most-missed item, because the local file looks like the whole -credential. - -## Unresolved - -1. P2, P3, P5 and P6 need a live run — no Telegram credentials exist in this - environment. -2. If P3 finds a media kind whose `.size` is absent *and* commonly large, revisit - only the estimate's `unknown size` line; no code change is implied. diff --git a/plans/reports/tester-260822-2259-suite-verification.md b/plans/reports/tester-260822-2259-suite-verification.md deleted file mode 100644 index 614f619..0000000 --- a/plans/reports/tester-260822-2259-suite-verification.md +++ /dev/null @@ -1,416 +0,0 @@ -# Test Suite Verification Report -**Date:** 2026-08-22 | **Python:** 3.12.3 | **Telethon:** 1.44.0 | **pytest:** 9.1.1 - -## SETUP COMMANDS - -**Commands that work (tested on headless Linux ARM64):** - -```bash -# Create venv -python3 -m venv .venv - -# Install dependencies + dev + coverage -./.venv/bin/pip install -e '.[dev]' -r requirements.txt pytest-cov - -# Run full suite -./.venv/bin/python -m pytest tests/ -v - -# Run with coverage report -./.venv/bin/python -m pytest tests/ --cov=src/telegram_exporter --cov-report=term-missing:skip-covered -``` - -**.gitignore status:** VERIFIED — `.venv/` already covered; no action needed. - ---- - -## TEST SUITE RESULTS - -**Overall Status:** ✓ ALL PASS (204/204 tests) - -| Metric | Value | -|--------|-------| -| **Tests Passed** | 204 | -| **Tests Failed** | 0 | -| **Tests Skipped** | 0 | -| **Execution Time** | 3.01s (--v) / 4.27s (--cov) | -| **Exit Code** | 0 (success) | - -### By Test File - -| File | Count | Result | -|------|-------|--------| -| test_cli_dispatch.py | 11 | ✓ PASS | -| test_download_decisions.py | 34 | ✓ PASS | -| test_estimate.py | 13 | ✓ PASS | -| test_lock.py | 5 | ✓ PASS | -| test_logging.py | 7 | ✓ PASS | -| test_no_concurrency.py | 12 | ✓ PASS | -| test_paths.py | 46 | ✓ PASS | -| test_session.py | 20 | ✓ PASS | -| test_sidecar.py | 9 | ✓ PASS | -| test_state.py | 14 | ✓ PASS | -| test_traversal.py | 38 | ✓ PASS | - ---- - -## COVERAGE ANALYSIS - -**Overall Coverage:** 93% (880/943 statements covered) - -### Per-Module Breakdown - -| Module | Stmts | Miss | Cover | Status | -|--------|-------|------|-------|--------| -| sidecar.py | 81 | 1 | 99% | ✓ Nearly complete | -| traversal.py | 100 | 2 | 98% | ✓ Nearly complete | -| downloader.py | 220 | 9 | 96% | ✓ Strong | -| estimate.py | 94 | 4 | 96% | ✓ Strong | -| media.py | 43 | 2 | 95% | ✓ Strong | -| cli.py | 128 | 12 | 91% | ⚠ Gaps | -| session.py | 140 | 34 | 76% | ⚠ Notable gaps | - -### Files with Complete Coverage (100%) -- paths.py (100% — all 141 statements covered) -- Other complete files exist but skipped in report - ---- - -## COVERAGE GAPS BY DOMAIN - -### 1. **Crash-Mid-Download Resume** — ✓ COVERED - -**Test Cases:** -- `test_a_second_run_downloads_nothing` — resume from cursor skips already-downloaded files -- `test_a_second_concurrent_run_exits_three` — locking prevents concurrent runs -- `test_commit_persists_and_leaves_no_temp_file` — state persistence via atomic ops -- `test_an_album_at_the_end_of_history_is_flushed` — album buffer flush on end - -**Implementation:** `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/downloader.py:run_download` (loop resumption from state.cursor); `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/state.py:commit` (atomic swap with fsync) - -**Gap Status:** None — resume logic from cursor is tested end-to-end via `wired` fixture + state file round-trips. - ---- - -### 2. **Partial-File Handling** — ✓ COVERED - -**Test Cases:** -- `test_a_zero_byte_target_is_unlinked_and_refetched` — zero-byte files are refetched -- `test_stray_part_files_are_swept` — `.part` files are cleaned before run -- `test_a_valid_target_wins_over_a_leftover_part_file` — target beats `.part` even if both exist -- `test_an_unsafe_filename_skips_one_file_instead_of_ending_the_run` — partial write errors don't crash -- `test_a_zero_byte_download_is_never_accepted` — zero-byte downloads rejected - -**Implementation:** `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/downloader.py:download_one` (dedupe check at line 254-256); `sweep_part_files` (line 290-295); write-to-temp + fsync + `os.replace` (line 330-342) - -**Gap Status:** None — write durability, temp-file cleanup, and dedup logic all exercise; fsync before replace is tested. - ---- - -### 3. **Path Sanitization Edge Cases** — ✓ COVERED - -**Test File:** `test_paths.py` (46 tests — highest density) - -**Unicode & Reserved Names:** -- `test_sanitize_table` with: - - Traversal: `../../../etc/passwd` → `passwd` ✓ - - Null bytes: `\x00evil.jpg` → `evil.jpg` ✓ - - Bidi spoofing: `‮gpj.exe` (RIGHT-TO-LEFT OVERRIDE) → stripped ✓ - - Control chars: `\x1b[31m` (ANSI escape) → stripped ✓ - - Reserved chars on Windows: `:*?<>|"` → `_` ✓ - - Tab/newline: `\t\n` → stripped ✓ -- `test_no_control_character_survives_anywhere_in_a_filename` — comprehensive control-char sweep ✓ - -**Long Names:** -- `test_a_whole_filename_stays_inside_ext4s_255_byte_component_limit` — UTF-8 byte counting, not char counting ✓ -- `test_truncation_is_byte_counted_and_keeps_the_extension` — extension preserved across truncation ✓ -- `test_truncation_of_a_pathological_extension_still_yields_a_usable_name` — long extensions handled ✓ - -**Traversal:** -- `test_no_sanitized_name_escapes_its_post_dir` — chroot guard via `safe_join` ✓ -- `test_safe_join_raises_rather_than_repairing` — `.. / ../../ / ..` paths refuse at init ✓ -- `test_hostile_declared_filename_becomes_a_leaf` — `a/b/c.jpg` → `c.jpg` (leaf only) ✓ - -**Gap Status:** None — all critical sanitization paths exercised. - ---- - -### 4. **Album/Grouped-Message Grouping** — ✓ COVERED - -**Test File:** `test_traversal.py` (38 tests) - -**Tests:** -- `test_a_three_photo_album_is_one_post` — grouped media yields one post ✓ -- `test_two_adjacent_albums_do_not_merge` — album boundaries are strict ✓ -- `test_a_standalone_message_between_albums_gets_its_own_post` — album isolation ✓ -- `test_an_album_at_the_end_of_history_is_flushed` — buffer flush when sweep ends ✓ -- `test_a_reopened_grouped_id_raises_album_split_error` — invariant: grouped_id cannot reopen ✓ -- `test_the_tripwire_catches_an_immediately_reopened_album` — race condition guard ✓ -- `test_the_album_buffer_is_bounded` — bounded buffer prevents memory leak ✓ - -**Implementation:** `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/traversal.py:grouped_sweep` (album accumulation + flush logic, lines 143-220) - -**Gap Status:** None — album assembly and edge cases are well-covered. - ---- - -### 5. **Flood-Wait Retry** — ✓ COVERED - -**Test File:** `test_session.py` (20 tests) - -**Tests:** -- `test_a_flood_wait_is_slept_through_once_by_default` — FloodWaitError triggers sleep ✓ -- `test_a_wait_beyond_the_explicit_ceiling_exits_six` — `--max-flood-wait` enforced, exit 6 ✓ -- `test_with_no_ceiling_even_a_four_hour_wait_is_slept` — no ceiling = wait arbitrarily long ✓ -- `test_the_flood_log_names_the_computed_wake_time` — logging shows wake time (UTC) ✓ -- `test_a_mid_sweep_flood_wait_resumes_from_the_last_yielded_id` — sweep resumes from last yielded ID, no gaps ✓ - -**Implementation:** - - `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/session.py:_flood_sleep` (lines 153-167) - - `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/session.py:aiter_with_flood_retry` (lines 181-200) - -**Coverage Status:** Lines 153-200 are executed (except line 106 which is unreachable code). - -**Gap Status:** None — flood-wait exit codes, sleep duration, and cursor resume all tested. - ---- - -### 6. **Pagination/Offset Correctness** — ✓ COVERED - -**Test File:** `test_traversal.py` - -**Tests:** -- `test_after_id_is_an_exclusive_lower_bound` — `offset_id` is exclusive (resume doesn't replay) ✓ -- `test_identity_survives_a_filter_dropping_members` — filter changes don't affect offset logic ✓ -- `test_a_fully_filtered_group_still_yields_a_post_so_the_cursor_advances` — cursor advances even when all media filtered ✓ -- `test_a_non_ascending_stream_is_refused` — descending IDs raise error ✓ - -**Implementation:** `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/traversal.py:grouped_sweep` (offset_id at lines 198-199) - -**Gap Status:** None — offset semantics are exercised via end-to-end sweeps. - ---- - -### 7. **Concurrent-Run Locking** — ✓ COVERED - -**Test File:** `test_lock.py` (5 tests) - -**Tests:** -- `test_a_second_acquire_exits_three` — second hold attempt exits 3 ✓ -- `test_contention_names_the_holder` — lock names the process holding it ✓ -- `test_contention_touches_no_file` — contention produces no side files ✓ -- `test_the_lock_is_released_when_the_holder_dies` — lock file cleaned after exit ✓ -- `test_different_export_roots_do_not_contend` — different export dirs use different locks ✓ - -**Implementation:** `/config/workspace/tiennm99dev/telegram-exporter/src/telegram_exporter/cli.py:Holder` (lines 38-110) - -**Coverage Status:** No parallelism primitives used (verified by test_no_concurrency.py). - -**Gap Status:** None — lock contention, naming, and cleanup all tested. - ---- - -## UNCOVERED CODE ANALYSIS - -### session.py (76% coverage — 34 lines missed) - -**Missed Lines (by category):** - -1. **Lines 106, 115-116, 134** — git availability edge cases - - Line 106: `return None` in `_nearest_existing_dir` — all parent dirs missing (unrealistic) - - Lines 115-116: `NotADirectoryError` in `_check_ignore` — filesystem race (git moved, dir became file) - - Line 134: early `return` when `cwd is None` — same as line 106 - - **Why untested:** These require a nonexistent path with no existing ancestors, which contradicts the path-exists-on-init contract - - **Risk:** Low — these are guards against transient filesystem races and complete absence of a filesystem, both exceedingly rare - -2. **Lines 224-233** — interactive login prompts - - Line 224: `phone = os.environ.get("TG_PHONE") or input(...)` - - Lines 228, 232-233: `input()` and `getpass()` calls during 2FA - - **Why untested:** Tests supply `TG_PHONE` via env and do not trigger 2FA (fixture uses existing session or mocks) - - **Risk:** Low — login is exercised end-to-end in e2e tests; the `input()` path exists but is terminal-only - -3. **Lines 266-287** — private invite link rejection - - Conditions: `if "/+" in spec or "joinchat/" in spec:` - - **Why untested:** Tests pass numeric IDs or @usernames; no test for invite links - - **Risk:** Medium — this is a documented rejection path, but no test verifies the error message or exit code - - **Recommendation:** Add test: `test_private_invite_link_rejected_with_exit_five` - -4. **Lines 292-299** — `_find_in_dialogs` fallback (numeric ID not in cache) - - **Why untested:** Tests use wired fixture with cached entity; rare case of numeric ID needing dialog scan - - **Risk:** Low — fallback is defensive; normal path via cache is tested - - **Recommendation:** Low priority; would require explicit eviction from session cache - -**Verdict:** session.py gaps are mostly edge cases (filesystem races, interactive prompts). Only the private-invite-link path is a documented feature with no test. - -### cli.py (91% coverage — 12 lines missed) - -**Missed Lines:** - -- Line 104: `except KeyboardInterrupt` — Ctrl-C handling -- Line 205: Error within `__aexit__` logging -- Lines 237-238, 243-244, 249-251, 256-257, 268: Mostly error/logging paths in `main()` -- **Why untested:** Error paths are triggered only by production exceptions or user signals; hard to inject into tests -- **Risk:** Medium — logging paths are code, but their absence won't prevent runs; the visible behavior (exit code, state) is tested - -**Verdict:** Mostly error-handling logging. The exit-code contracts are verified; the logging details are not. - -### downloader.py (96% coverage — 9 lines missed) - -**Missed Lines:** - -- Line 142: `if self.declared_size is not None and self.declared_size != self.size:` → photo variant size mismatch -- Line 162: `if self.failed:` → run completed with failed files -- Line 173: `raise AssertionError("unreachable...")` → unreachable code guard in `human_bytes` -- Line 194: `return await with_flood_retry(...)` — refresh call path -- Line 313: `error="file reference expired twice"` — rare double-expiry -- Lines 363-366: `if e.errno in TRANSIENT_ERRNO:` and subsequent error handling -- **Why untested:** Some are rare conditions (photo size mismatch, double expiry); others require injected errno values -- **Risk:** Low to medium — most are retry/error cases where the loop continues; the terminal errors (line 366) are less tested - -**Verdict:** Strong coverage; gaps are in error paths and rare photo-variant cases. - -### estimate.py (96% coverage — 4 lines missed) - -- Lines 72, 84, 128, 130: Mostly conditional logging and rare format branches -- **Risk:** Low — estimate logic is exercised; missing lines are output-only - -### media.py (95% coverage — 2 lines missed) - -- Lines 42, 106: Media-type branches -- **Risk:** Low — basic media kinds are tested - ---- - -## CRITICAL FINDINGS & RISK ASSESSMENT - -### ✓ NO BLOCKERS - -All tests pass. The codebase has no syntax errors, import errors, or runtime failures in the test suite. - -### ⚠ IDENTIFIED COVERAGE GAPS (Ranked by Risk) - -1. **[MEDIUM] Private invite link handling untested** - - Location: `src/telegram_exporter/session.py:resolve_entity`, lines 266-270 - - Impact: User passes a Telegram group invite link; code must reject with exit code 5 and an actionable message - - Evidence: Test file `test_session.py` has no test matching `"invite\|joinchat\|/+"` - - Current state: Code exists, but no test verifies the error path - - Recommendation: Add test `test_resolve_entity_rejects_private_invite_link_with_exit_five` - -2. **[LOW] Double-expiry file reference handling** - - Location: `src/telegram_exporter/downloader.py:download_one`, line 313 - - Impact: If a file reference expires twice in a single message (rare but possible), the error is recorded - - Evidence: Code path exists (line 310-313) but no test injects two refresh calls - - Current state: Retry logic is tested; double-expiry is not explicitly exercised - - Recommendation: Low priority; defensive code that would only trigger under network instability - -3. **[LOW] Transient OS errors during write** - - Location: `src/telegram_exporter/downloader.py`, lines 363-366 - - Impact: If a write fails with EAGAIN, EINTR, EIO, etc., the retry loop continues; unexpected errors abort - - Evidence: Test mocks the filesystem; does not inject errno values - - Current state: Retry ladder is tested; errno branches are not - - Recommendation: Low priority; would require ctypes or os.errno injection - -4. **[LOW] Photo size mismatch declared_size reporting** - - Location: `src/telegram_exporter/downloader.py`, line 142 - - Impact: When a photo is fetched in a variant size, the sidecar records both sizes - - Evidence: Telethon returns a size that differs from declared; no test sets up this scenario - - Current state: Sidecar recording is tested; the conditional `declared_size != size` branch is not - - Recommendation: Would need to mock a photo fetch with size mismatch - -5. **[LOW] Interactive login prompts** - - Location: `src/telegram_exporter/session.py`, lines 224-233 - - Impact: User is prompted for phone and 2FA during first login - - Evidence: Tests supply credentials via env or use existing sessions - - Current state: End-to-end login tested; interactive `input()` / `getpass()` paths are not - - Recommendation: Terminal-only; covered by manual testing and e2e fixture - ---- - -## DOMAIN COVERAGE SUMMARY - -| Domain | Coverage | Gaps | Risk | -|--------|----------|------|------| -| Crash-mid-download resume | ✓ Full | None | None | -| Partial-file handling | ✓ Full | None | None | -| Path sanitization (unicode, long, reserved, traversal) | ✓ Full | None | None | -| Album/grouped-message grouping | ✓ Full | None | None | -| Flood-wait retry | ✓ Full | None | None | -| Pagination/offset correctness | ✓ Full | None | None | -| Concurrent-run locking | ✓ Full | None | None | - ---- - -## DIAGNOSIS: ROOT CAUSE ANALYSIS - -**Q: Are there any failing tests?** -A: No. All 204 tests pass. Exit code 0. - -**Q: Are there any errors in the test environment?** -A: No. venv, pip install, pytest all succeeded without warnings. Python 3.12.3 is compatible. - -**Q: Is coverage adequate for a resumable media exporter?** -A: Yes. 93% overall coverage, with critical paths at 96-99%. Resume, locking, sanitization, and album grouping are all well-tested. Session.py has lower coverage (76%) due to edge cases (git race conditions, interactive prompts) that are not realistic in test harnesses. - ---- - -## RECOMMENDATIONS & NEXT STEPS - -### Immediate (if high rigor required) -1. Add test for private invite link rejection (`test_session.py::test_resolve_entity_rejects_private_invite_link_with_exit_five`) - - File: `tests/test_session.py` - - Test: Create entity with spec containing `joinchat/` and assert Abort with exit 5 - -### Short-term (nice-to-have) -2. Add photo size mismatch test to `test_download_decisions.py` - - Simulate Telethon returning variant size; verify sidecar has both sizes -3. Add errno injection test for transient write errors - - Test retry on EAGAIN, EINTR, EIO - -### Deferred (low value) -4. Interactive login tests would require `pexpect` or terminal automation - - Current e2e coverage sufficient; low defect risk for this path -5. Filesystem race conditions (no existing parent dirs) are unrealistic - - Already have guards; low defect risk - ---- - -## BUILD & CI/CD VALIDATION - -**Build Status:** ✓ PASS - -- Python 3.12.3 / pytest 9.1.1 / Telethon 1.44.0 -- No syntax errors, no import errors, no missing dependencies -- No build warnings or deprecation notices -- .venv is properly gitignored - -**Recommended CI/CD Step:** - -```yaml -test: - script: - - ./.venv/bin/python -m pytest tests/ -v --tb=short - - ./.venv/bin/python -m pytest tests/ --cov=src/telegram_exporter --cov-report=term-missing --cov-fail-under=90 -``` - ---- - -## SETUP VERIFICATION CHECKLIST - -- [x] venv created at `./.venv` -- [x] Dependencies installed: pyaes, pyasn1, rsa, Telethon==1.44.0, pytest, pytest-cov -- [x] Project installed in dev mode: `-e '.[dev]'` -- [x] All 204 tests pass -- [x] Coverage report generated: 93% overall -- [x] No test modifications made -- [x] .gitignore verified (`.venv/` already covered) - ---- - -## UNRESOLVED QUESTIONS - -1. **Should the private-invite-link test be added before the next release?** (Depends on SLA for untested error paths) -2. **Is 93% coverage the project target, or should it reach 95%+?** (Affects prioritization of gap closure) -3. **Are e2e tests (the `wired` fixture with real Telegram session) run separately in CI?** (If yes, interactive paths and rare errors may be covered there) - ---- - -**Report Status:** Complete | **Recommendation:** All systems nominal; proceed with deployment. Optional: add one test for private-invite-link handling if compliance requires 100% of error paths. diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 2417a13..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,25 +0,0 @@ -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[project] -name = "telegram-exporter" -version = "0.1.0" -description = "Export all media from a Telegram group to local disk, grouped by post, resumably." -requires-python = ">=3.12" -license = { file = "LICENSE" } -# Telethon 2.0 is alpha with a renamed API surface. requirements.txt pins the -# exact version the API probes were verified against. -dependencies = ["telethon>=1.41,<2"] - -[project.optional-dependencies] -dev = ["pytest>=8"] - -[project.scripts] -tg-export = "telegram_exporter.cli:main" - -[tool.hatch.build.targets.wheel] -packages = ["src/telegram_exporter"] - -[tool.pytest.ini_options] -testpaths = ["tests"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 41fe3c6..0000000 --- a/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Pinned to the Telethon version the API probes were verified against. -# Regenerate with `pip freeze` only after re-running spike/probe_telethon_api.py. -pyaes==1.6.1 -pyasn1==0.6.4 -rsa==4.9.1 -Telethon==1.44.0 diff --git a/src/telegram_exporter/__init__.py b/src/telegram_exporter/__init__.py deleted file mode 100644 index bc5721d..0000000 --- a/src/telegram_exporter/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Export all media from a Telegram group to local disk, grouped by post.""" - -__version__ = "0.1.0" diff --git a/src/telegram_exporter/cli.py b/src/telegram_exporter/cli.py deleted file mode 100644 index c80a6a6..0000000 --- a/src/telegram_exporter/cli.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Command line entry point: process rails, argument parsing, dispatch. - -The rails come first and in this order for reasons that are not stylistic: -umask before anything can create a file, logging policy before anything can log a -credential, the git-safety assertion before anything writes into a repo, and the -lock before the destructive .part sweep. -""" - -from __future__ import annotations - -import argparse -import asyncio -import fcntl -import logging -import os -import socket -import sys -from contextlib import contextmanager -from pathlib import Path - -from . import paths -from .downloader import ( - FATAL_ERRNO, - Config, - Fetcher, - run_download, - sweep_part_files, - write_title, -) -from .estimate import EstimateSink, resume_from, run_estimate -from .session import ( - Abort, - assert_not_committable, - connected_client, - default_session_path, - load_api_credentials, - now_iso, - resolve_entity, - with_session_suffix, -) -from .sidecar import Sidecar -from .state import State -from .traversal import TraversalError, build_filters, iter_posts, parse_size - -log = logging.getLogger("telegram_exporter.cli") - -LOCK_NAME = ".export.lock" -DEFAULT_MIN_FREE = "2GiB" - - -def configure_logging(verbose: bool) -> None: - """--verbose means *our* logger only. - - telethon's request logging carries api_id, phone_number and - phone_code_hash, so its floor is unconditional and there is deliberately no - --debug-telethon flag. The root level is never touched. - """ - logging.basicConfig(level=logging.INFO, - format="%(asctime)s %(levelname)s %(message)s") - logging.getLogger("telegram_exporter").setLevel( - logging.DEBUG if verbose else logging.INFO) - logging.getLogger("telethon").setLevel(logging.WARNING) - logging.getLogger("asyncio").setLevel(logging.WARNING) - - -@contextmanager -def exclusive(path: Path): - """Single-instance lock, per export root and per session. - - flock releases on process death, kill -9 included, so there is no stale-lock - cleanup to get wrong. Contention fails fast with exit 3 - no wait, no - --force. The message names pid, host and start time so that "did it hang?" - gets an answer instead of a second process. - """ - path.parent.mkdir(parents=True, exist_ok=True) - f = open(path, "a+") - try: - fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB) - except BlockingIOError: - f.seek(0) - holder = f.read().strip() or "?" - f.close() - raise Abort( - f"busy: another tg-export holds {path} [{holder}]\n" - f"if you meant to export a second group at the same time, give it its " - f"own --session too: one SQLite session cannot serve two clients.", 3 - ) from None - f.seek(0) - f.truncate() - f.write(f"pid={os.getpid()} host={socket.gethostname()} started={now_iso()}\n") - f.flush() - try: - yield - finally: - f.close() - - -def _non_negative(text: str) -> int: - """A negative --limit is the one input where the estimator and the real run - would disagree (one stops before counting, the other after one post), so it - is rejected at the boundary rather than handled twice.""" - value = int(text) - if value < 0: - raise argparse.ArgumentTypeError(f"must be 0 or greater, got {value}") - return value - - -def build_parser() -> argparse.ArgumentParser: - p = argparse.ArgumentParser( - prog="tg-export", - description="Export all media from a Telegram group, grouped by post.", - epilog="credentials: export TG_API_ID and TG_API_HASH " - "(create an app at https://my.telegram.org). " - "The login code and any 2FA password are prompted for, never read " - "from the environment.") - p.add_argument("--group", required=True, - help="numeric id, @username, or t.me link") - p.add_argument("--out", required=True, metavar="DIR", - help="parent directory; the export goes in /g") - p.add_argument("--dry-run", action="store_true", - help="report what a real run would fetch, write nothing") - p.add_argument("--since", metavar="YYYY-MM-DD", help="messages on or after (UTC)") - p.add_argument("--until", metavar="YYYY-MM-DD", help="messages on or before (UTC)") - p.add_argument("--types", metavar="LIST", - help="comma-separated: photo,video,document,audio,voice") - p.add_argument("--max-size", metavar="SIZE", - help="skip media larger than this, e.g. 100MB") - p.add_argument("--include-text", action="store_true", - help="also record text-only messages in messages.jsonl with an " - "empty files[]; part of the stored filter set") - p.add_argument("--limit", type=_non_negative, metavar="N", - help="stop after N posts that contain files (0 = connect only)") - p.add_argument("--session", metavar="PATH", - help=f"session file (default: {default_session_path()})") - p.add_argument("--reset-state", action="store_true", - help="discard the cursor and rotate messages.jsonl; keeps " - "downloaded files") - p.add_argument("--min-free", default=DEFAULT_MIN_FREE, metavar="SIZE", - help=f"disk kept free (default {DEFAULT_MIN_FREE})") - p.add_argument("--max-flood-wait", type=int, metavar="SECONDS", - help="exit resumably (code 6) instead of sleeping through a " - "flood wait longer than this; unset means sleep as long " - "as Telegram demands") - p.add_argument("--verbose", action="store_true", - help="debug logging for this tool only, never for telethon") - return p - - -async def _run(args) -> int: - api_id, api_hash = load_api_credentials() - filters = build_filters(types=args.types, since=args.since, until=args.until, - max_size=args.max_size, include_text=args.include_text) - min_free = parse_size(args.min_free) - session_path = with_session_suffix( - Path(args.session).expanduser() if args.session else default_session_path()) - - # The session lock guards the SQLite session file, so it has to be held - # before anything opens that file - connect() and _login() both write to it. - # Taken deeper in, it locked *after* the damage: two runs sharing the default - # session each passed their own per-root lock, both opened the same database, - # and the loser reported the conflict having already caused it. - # - # --dry-run is inside the lock too. It writes nothing to the export tree, but - # it opens the same session file, which is the resource this lock is about - - # so a dry-run alongside a real export needs its own --session. - # - # Refuse a committable path before the lock, so a rejected run leaves no - # lock file behind next to a session it was never allowed to create. - assert_not_committable(session_path) - with exclusive(session_path.with_name(session_path.name + ".lock")): - async with connected_client(session_path, api_id, api_hash) as client: - entity, peer_id = await resolve_entity(client, args.group) - title = (getattr(entity, "title", None) - or getattr(entity, "username", None) or str(peer_id)) - root = paths.export_root(Path(args.out).expanduser(), peer_id) - assert_not_committable(root, is_dir=True) - log.info("group %r (id %s) -> %s", title, peer_id, root) - - if args.dry_run: - return await _dry_run(client, entity, root=root, title=title, - peer_id=peer_id, filters=filters, - min_free=min_free, args=args) - return await _real_run(client, entity, root=root, title=title, - peer_id=peer_id, filters=filters, - min_free=min_free, args=args) - - -async def _dry_run(client, entity, *, root: Path, title: str, peer_id: int, - filters, min_free: int, args) -> int: - """Takes no lock, creates no directories, mutates no state - so it can run - while a real export is in progress.""" - start = resume_from(root, filters) - posts = iter_posts(client, entity, after_id=start, filters=filters, - max_flood_wait=args.max_flood_wait) - return await run_estimate(posts, sink=EstimateSink(), root=root, title=title, - peer_id=peer_id, filters=filters, start_id=start, - min_free=min_free, limit=args.limit) - - -async def _real_run(client, entity, *, root: Path, title: str, peer_id: int, - filters, min_free: int, args) -> int: - root.mkdir(parents=True, exist_ok=True) - - # The export lock before the .part sweep: the sweep is the destructive - # operation, so locking after it would lock after the damage. Per-root rather - # than global, because exporting two different groups concurrently is - # legitimate - each with its own --session, which the session lock in _run - # now enforces rather than merely documents. - with exclusive(root / LOCK_NAME): - sweep_part_files(root) - write_title(root, title) - - # Rotate before the cursor is zeroed, and only under --reset-state - - # which is the one path with no compatibility check to fail, since the - # stored filters are being discarded. Opening state first made the zeroed - # cursor durable while the previous regime's sidecar was still in place, - # which is exactly the mixed-generation log that rotating exists to - # prevent. Without --reset-state nothing here is touched, so State.open - # still refuses a filter or chat mismatch (exit 7) before any write. - sidecar = Sidecar(root) - if args.reset_state: - sidecar.rotate() - state = State.open(root, chat_id=peer_id, chat_title=title, - filters=filters, reset=args.reset_state) - - with sidecar: - posts = iter_posts(client, entity, after_id=state.cursor_id, - filters=filters, max_flood_wait=args.max_flood_wait) - totals = await run_download( - posts, - fetcher=Fetcher(client, entity, args.max_flood_wait), - state=state, sidecar=sidecar, root=root, - # The flood ceiling reaches the network through the two - # primitives, not through Config: Fetcher above for the download - # path, iter_posts for the sweep. - cfg=Config(min_free=min_free, disk_path=root), - limit=args.limit) - log.info("done: %s", totals.summary()) - return 0 - - -def main() -> int: - os.umask(0o077) # FIRST statement: covers the session file and every sqlite - # sibling, the export tree, messages.jsonl, .part files, - # the state file and the lock - one line, no per-file chmod. - args = build_parser().parse_args() - configure_logging(args.verbose) - try: - return asyncio.run(_run(args)) - except Abort as a: - log.error("%s", a.reason) - return a.code - except TraversalError as e: - # An album split or a non-ascending sweep. Loud on purpose: the export on - # disk cannot be trusted, and the cursor is where the last good post was. - log.error("%s", e) - return 1 - except (ConnectionError, TimeoutError) as e: - # Before the OSError clause: both subclass it, and the sweep path has no - # retry of its own, so a mid-sweep reset arrives here. Calling that - # "filesystem" would send the operator to the wrong place entirely. - log.error("network: %s - the cursor is intact, re-run to resume", e) - return 1 - except OSError as e: - # The download path maps these itself; this catches the same failures - # arriving from the sidecar or state writes, so disk exhaustion exits 3 - # there too instead of printing a traceback. - code = 3 if e.errno in FATAL_ERRNO else 1 - log.error("filesystem: %s", e) - return code - except ValueError as e: # bad --types/--since/--max-size value - log.error("%s", e) - return 1 - except KeyboardInterrupt: - log.info("interrupted - re-run the same command to resume") - return 130 - except Exception: - # Last resort, so no failure can exit outside the documented contract. - # The traceback is still printed - CPython does not put locals in it, so - # this cannot leak api_hash or a phone number - but the exit code stays - # meaningful and the operator is told the cursor is safe. - log.exception("unexpected error - the cursor is intact, re-run to resume") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/src/telegram_exporter/downloader.py b/src/telegram_exporter/downloader.py deleted file mode 100644 index cbff5c4..0000000 --- a/src/telegram_exporter/downloader.py +++ /dev/null @@ -1,527 +0,0 @@ -"""The download path: fetch sequentially, write durably, dedupe, record. - -Governing principle: **the filesystem is the state; the state file is a hint; the -sidecar is the log.** Everything here follows from it. - -Telethon has no byte-level resume - download_media writes from offset 0 every -call - so resumability lives at *message* granularity, and completeness is proven -by arrival at the target path: - - per file: write -> flush -> fsync(fd) -> os.replace(tmp, target) - per post: fsync(post_dir) -> sidecar.append + fsync -> state.commit - -With fsync before os.replace, a file at the target path can only have arrived -fully downloaded. That is what licenses existence-based dedupe below. -""" - -from __future__ import annotations - -import asyncio -import errno -import logging -import os -import shutil -import time -from dataclasses import dataclass -from pathlib import Path - -from telethon.errors import ( - AuthKeyDuplicatedError, - AuthKeyUnregisteredError, - ChannelInvalidError, - ChannelPrivateError, - ChatForbiddenError, - FileIdInvalidError, - FileReferenceExpiredError, - FilerefUpgradeNeededError, - MediaEmptyError, - ServerError, - SessionRevokedError, - UserDeactivatedBanError, -) -from telethon.errors import common as telethon_common - -from . import paths -from .media import file_ext, has_media, media_kind, media_size -from .session import Abort, with_flood_retry -from .state import fsync_dir - -log = logging.getLogger("telegram_exporter.downloader") - -TRANSIENT_ERRNO = {errno.EAGAIN, errno.EINTR, errno.EIO, errno.ETIMEDOUT, - errno.ECONNRESET} -FATAL_ERRNO = {errno.ENOSPC, errno.EDQUOT, errno.EROFS, errno.EACCES} - -MAX_ATTEMPTS = 3 -PART_SUFFIX = ".part" -TITLE_NAME = "title.txt" - -# Commit the cursor at least this often even across a long filtered region, so a -# 100k-message sweep does not become 100k state writes. -COMMIT_INTERVAL_S = 5.0 - -DOWNLOADED, SKIPPED, FAILED = "DOWNLOADED", "SKIPPED", "FAILED" - -# Network trouble is retried; the unavailable-media errors are skipped; the -# access and auth errors end the run. FloodWaitError never appears here - it is -# owned by with_flood_retry, because a retry during an active wait is counted as -# a fresh violation and escalates the next wait. -# -# ServerError rather than only RpcCallFailError: the latter is one leaf of it, so -# a plain Telegram -500 would otherwise escape. BadMessageError arrives the same -# way - MTProtoSender sets it on the pending request's future, so it surfaces at -# the download await rather than in the receive loop. -# -# InvalidBufferError reaches an awaited request only for a transport-level 429, -# where retrying beats ending the run. InvalidChecksumError and TypeNotFoundError -# are deliberately absent: MTProtoSender consumes both inside its receive loop -# and neither can surface here, so listing them would imply coverage this tuple -# does not have. SecurityError is absent too - its one download-path member, -# CdnFileTamperedError, gets its own clause below, and no other security failure -# should be quietly retried. -_NETWORK_ERRORS = (ConnectionError, TimeoutError, asyncio.TimeoutError, - asyncio.IncompleteReadError, ServerError, - telethon_common.InvalidBufferError, - telethon_common.BadMessageError) - -# AuthKeyNotFound is the one that actually arrives: MTProtoSender._disconnect -# sets it on every in-flight request, and it subclasses plain Exception - so it -# matched no clause here and ended the run with a traceback instead of the -# documented exit 4. The RPC-level AuthKeyUnregisteredError is the rarer sibling. -_AUTH_ERRORS = (AuthKeyUnregisteredError, AuthKeyDuplicatedError, - SessionRevokedError, UserDeactivatedBanError, - telethon_common.AuthKeyNotFound) -_ACCESS_ERRORS = (ChannelPrivateError, ChatForbiddenError, ChannelInvalidError) -_UNAVAILABLE_ERRORS = (MediaEmptyError, FileIdInvalidError, FilerefUpgradeNeededError) - - -def backoff(attempt: int) -> float: - """5, 10 - and 0 after the final attempt, because sleeping before giving up - buys nothing and costs 20 s per exhausted file.""" - if attempt >= MAX_ATTEMPTS: - return 0.0 - return 5.0 * 2 ** (attempt - 1) - - -@dataclass(frozen=True) -class Config: - """Everything download_one needs from the CLI, so it takes no globals.""" - - min_free: int # bytes held in reserve, default 2 GiB - disk_path: Path # filesystem to measure - the export root - # No max_flood_wait here: the ceiling belongs to the two flood primitives, - # which Fetcher owns. A copy on this object would read as though check_disk - # and download_one honored it. - - -@dataclass -class Result: - """What happened to one message's media, as the sidecar records it.""" - - message_id: int - status: str - path: Path | None = None - size: int | None = None - declared_size: int | None = None - name: str | None = None - mime: str | None = None - kind: str | None = None - error: str | None = None - - def as_record(self, root: Path) -> dict: - record = { - "path": str(self.path.relative_to(root)), - "size": self.size, - "name": self.name, - "mime": self.mime, - "kind": self.kind, - } - # declared_size appears only when it differs, which incidentally - # accumulates empirical data on photo-variant accuracy. - if self.declared_size is not None and self.declared_size != self.size: - record["declared_size"] = self.declared_size - return record - - -@dataclass -class Totals: - """Per-run, in-memory counters. Deliberately not persisted: counters in the - state file double-count on every resume, because a re-processed album - re-increments them.""" - - posts: int = 0 - downloaded: int = 0 - skipped: int = 0 - failed: int = 0 - bytes_downloaded: int = 0 - - def summary(self) -> str: - line = (f"{self.posts} posts, {self.downloaded} downloaded " - f"({human_bytes(self.bytes_downloaded)}), {self.skipped} already present") - if self.failed: - line += f", {self.failed} files failed (see messages.jsonl)" - return line - - -def human_bytes(n: int) -> str: - step = 1024.0 - value = float(n) - for unit in ("B", "KiB", "MiB", "GiB", "TiB"): - if value < step or unit == "TiB": - return f"{value:,.1f} {unit}" if unit != "B" else f"{int(value):,} B" - value /= step - raise AssertionError("unreachable: the loop returns at TiB") - - -# --------------------------------------------------------------------------- # -# Fetcher - the only part that touches the network, so tests can replace it -# --------------------------------------------------------------------------- # - -class Fetcher: - """Wraps the two client calls download_one needs, each behind flood retry.""" - - def __init__(self, client, entity, max_flood_wait: int | None = None) -> None: - self.client = client - self.entity = entity - self.max_flood_wait = max_flood_wait - - async def fetch(self, msg, fh) -> None: - await with_flood_retry(lambda: self.client.download_media(msg, file=fh), - max_wait_s=self.max_flood_wait) - - async def refresh(self, message_id: int): - """Re-fetch a message to renew an expired file reference.""" - return await with_flood_retry( - lambda: self.client.get_messages(self.entity, ids=message_id), - max_wait_s=self.max_flood_wait) - - -# --------------------------------------------------------------------------- # -# Per-file decisions -# --------------------------------------------------------------------------- # - -def check_disk(msg, cfg: Config) -> None: - """Refuse before the first byte, comparing against *this file's* size. - - One statvfs is microseconds, unmeasurable against a multi-second download at - concurrency 1. Polling every N files would optimize a cost that does not - exist while opening a blind spot: a single Telegram file reaches 2-4 GB, so - ~100 GB could be written between two polls. An unknown size counts as 0 and - is absorbed by the reserve, which is why the reserve defaults to 2 GiB - one - entire unknown-size file at the non-premium upload cap. - """ - need = (media_size(msg) or 0) + cfg.min_free - free = shutil.disk_usage(cfg.disk_path).free - if free < need: - raise Abort(f"insufficient disk on {cfg.disk_path}: need " - f"{human_bytes(need)} (file + {human_bytes(cfg.min_free)} " - f"reserve), have {human_bytes(free)}", 3) - - -def validate(tmp: Path, msg) -> bool: - """Advisory only - never raises. True means accept. - - Documents, video and audio carry an exact document.size, so a mismatch there - is real truncation. Photos are the only approximate case: .size describes the - size *variant* Telethon selected. An exact-equality gate would delete every - completed photo on resume and then raise, which is why this is advisory and - why dedupe below tests existence rather than size. - """ - actual, declared = tmp.stat().st_size, media_size(msg) - if actual == 0: - # Nothing arrived. download_media returns without writing when the media - # resolves to an empty variant, and the photo excuse below would - # otherwise rename a 0-byte file to the target - where the dedupe rule - # ("zero bytes is never a complete download") could never see it again, - # because a completed export does not revisit existing targets. - log.warning("message %s produced 0 bytes - retrying", msg.id) - return False - if declared is None or actual == declared: - return True - if media_kind(msg) == "photo": - log.debug("photo %s declared %s, got %s - variant size, accepted", - msg.id, declared, actual) - return True - log.warning("size mismatch on message %s: declared %s, got %s - retrying", - msg.id, declared, actual) - return False - - -def sweep_part_files(root: Path) -> int: - """Remove stray .part files left by a killed run. - - Destructive, so the caller must already hold the export lock: sweeping first - and locking second would lock after the damage. - """ - removed = 0 - for stray in root.rglob(f"*{PART_SUFFIX}"): - stray.unlink(missing_ok=True) - removed += 1 - if removed: - log.info("swept %d stray %s file(s)", removed, PART_SUFFIX) - return removed - - -async def download_one(fetcher, post_dir_path: Path, msg, *, cfg: Config) -> Result: - """Download one message's media into post_dir_path. - - Every exit is one of: DOWNLOADED, SKIPPED, FAILED (recorded, run continues), - or Abort (run ends with a correct cursor). There is no path that lets an - unhandled exception kill an unattended 20-hour export. - """ - try: - target = paths.safe_join(post_dir_path, paths.derive_filename(msg)) - except (ValueError, TypeError) as e: - # sanitize should make this unreachable, so reaching it means sanitize has - # a hole worth finding - hence ERROR, not a silent repair. But one - # hostile filename must not be able to end the run: without the cursor - # advancing past this post, every later run would die on the same - # message. Recorded in errors[] and skipped, like unavailable media. - log.error("cannot build a safe path for message %s: %s - skipping", msg.id, e) - return _result(msg, FAILED, error=f"unsafe filename: {e}") - - # One stat, not three. An already-present file is the common case on every - # resume, and reusing this size as the recorded size keeps the whole skip - # path to a single syscall. - try: - present = target.stat().st_size - except FileNotFoundError: - present = None - if present is not None: - if present > 0: - return _result(msg, SKIPPED, target, size=present) - # Zero bytes means ENOSPC junk, never a complete download - re-fetch. - target.unlink() - - check_disk(msg, cfg) - post_dir_path.mkdir(parents=True, exist_ok=True) - tmp = target.with_name(target.name + PART_SUFFIX) - refreshed = False - - attempt = 0 - while attempt < MAX_ATTEMPTS: - attempt += 1 - try: - with open(tmp, "wb") as fh: - await fetcher.fetch(msg, fh) - fh.flush() - os.fsync(fh.fileno()) - if not validate(tmp, msg): - tmp.unlink(missing_ok=True) - await asyncio.sleep(backoff(attempt)) - continue - os.replace(tmp, target) - return _result(msg, DOWNLOADED, target) - - # ---- SKIP: this message is unavailable; the run continues ---------- # - except FileReferenceExpiredError: - # References expire in hours, so a 20-hour run *will* hit this. One - # refresh is the highest-value entry in this table. - tmp.unlink(missing_ok=True) - if refreshed: - return _result(msg, FAILED, error="file reference expired twice") - fresh = await fetcher.refresh(msg.id) - refreshed = True - if fresh is None or not has_media(fresh): - return _result(msg, FAILED, error="message deleted") - # Renewing a reference is not a failed attempt. Counting it burned - # the last slot when expiry landed on the final attempt, so the - # fresh reference was never actually fetched and the message was - # reported as "exhausted N attempts" having been tried N-1 times. - # The `refreshed` latch above bounds this arm, so it cannot loop. - attempt -= 1 - msg = fresh - continue - except _UNAVAILABLE_ERRORS as e: - tmp.unlink(missing_ok=True) - return _result(msg, FAILED, - error=f"media unavailable or expired: {type(e).__name__}") - - # ---- INTEGRITY: not network trouble, and not retryable noise ------ # - except telethon_common.CdnFileTamperedError: - # The CDN served bytes whose hash does not match what Telegram - # signed. It subclasses SecurityError and is raised on the download - # path, so folding it into the network clause would report a - # trust-boundary event as three lines of "network error" and a bland - # FAILED. Recorded distinctly and skipped; the run continues, because - # one tampered file is not evidence about the rest. - tmp.unlink(missing_ok=True) - log.error("message %s: CDN integrity check FAILED - the served bytes " - "do not match what Telegram signed; not retrying", msg.id) - return _result(msg, FAILED, error="cdn integrity check failed") - - # ---- ABORT: the run cannot continue ------------------------------- # - except _AUTH_ERRORS: - tmp.unlink(missing_ok=True) - raise Abort("session invalidated - delete the session file and re-login", 4) - except _ACCESS_ERRORS: - tmp.unlink(missing_ok=True) - raise Abort("lost access to the group", 5) - - # ---- RETRY: network trouble --------------------------------------- # - # Before the OSError clause on purpose: ConnectionError *is* an OSError, - # and a ConnectionResetError raised by asyncio can carry errno=None, - # which the fatal/transient errno test below would fall through to - # "unexpected OS error" and abort a healthy run. - except _NETWORK_ERRORS as e: - tmp.unlink(missing_ok=True) - log.warning("network error on message %s (attempt %d/%d): %s", - msg.id, attempt, MAX_ATTEMPTS, e) - await asyncio.sleep(backoff(attempt)) - continue - - # ---- filesystem: fatal checked before transient ------------------- # - except OSError as e: - tmp.unlink(missing_ok=True) - if e.errno in FATAL_ERRNO: - raise Abort(f"filesystem: {e}", 3) from None - if e.errno in TRANSIENT_ERRNO: - await asyncio.sleep(backoff(attempt)) - continue - raise Abort(f"unexpected OS error: {e}", 1) from None - - return _result(msg, FAILED, error=f"exhausted {MAX_ATTEMPTS} attempts") - - -def _result(msg, status: str, target: Path | None = None, - error: str | None = None, size: int | None = None) -> Result: - f = getattr(msg, "file", None) - if size is None and target is not None: - size = target.stat().st_size - return Result( - message_id=msg.id, - status=status, - path=target, - size=size, - declared_size=media_size(msg), - name=getattr(f, "name", None) or f"{media_kind(msg)}{file_ext(msg)}", - mime=getattr(f, "mime_type", None), - kind=media_kind(msg), - error=error, - ) - - -# --------------------------------------------------------------------------- # -# The sink loop - Invariant 2 by construction -# --------------------------------------------------------------------------- # - -async def run_download(posts, *, fetcher, state, sidecar, root: Path, - cfg: Config, limit: int | None = None) -> Totals: - """Drive the shared traversal, downloading as it goes. - - state.commit is reached only after a post is fully handled, and there is no - commit on any error path - so the cursor can never point past in-flight work. - "Commit the cursor and exit" is deliberately not a concept here; it was the - mechanism by which the disk guard could have violated Invariant 2. - - Abort and KeyboardInterrupt propagate to cli.main, which owns exit codes. - The cursor is already correct when they do. - """ - totals = Totals() - last_commit = time.monotonic() - last_complete = state.cursor_id - if limit == 0: - return totals # connectivity check, no work requested - - async for post in posts: - media = [m for m in post.messages if has_media(m)] - results: list[Result] = [] - - if media: - target_dir = paths.post_dir(root, post.post_id) - for msg in media: - result = await download_one(fetcher, target_dir, msg, cfg=cfg) - results.append(result) - _tally(totals, result, post.post_id) - # Only a rename needs its directory entry forced. When every file in - # the post was already present nothing was renamed, so a resume - # across a mostly-complete export no longer pays one fsync per post - # to make a directory entry durable that a previous run already did. - if any(r.status == DOWNLOADED for r in results) and target_dir.exists(): - fsync_dir(target_dir) - - # A post may be empty (everything filtered out) or text-only under - # --include-text; both still advance the cursor, and neither is counted. - if post.messages: - sidecar.append(post, results) - sidecar.fsync() - if media: - totals.posts += 1 - - # The post is now fully handled: files on disk, sidecar durable. - last_complete = post.max_message_id - wrote = any(r.status in (DOWNLOADED, FAILED) for r in results) - if wrote or (time.monotonic() - last_commit) > COMMIT_INTERVAL_S: - state.commit(last_complete) - last_commit = time.monotonic() - - if limit is not None and totals.posts >= limit: - log.info("--limit %d reached", limit) - state.commit(last_complete) - return totals - - # Flush the throttled tail before claiming the export finished. Without this, - # a run ending on a stretch of filtered or text-only posts shorter than the - # commit interval would record completed_at against a cursor still pointing - # at the last post that happened to contain a file - and the next run would - # re-sweep that whole tail to download nothing. - state.commit(last_complete) - if totals.failed > totals.downloaded + totals.skipped: - # Failures outnumber everything that worked. That is what a dead media DC - # looks like from in here - ConnectionError is retryable, so the loop - # walks the whole history failing everything and returns normally. - # Setting completed_at on that would answer "did my export finish?" with - # a confident yes over a mostly-empty tree. - # - # Compared against downloaded + skipped, not against zero: requiring - # *nothing* to have succeeded meant a single already-present file - # disabled the guard entirely, so any resume over a partly-complete - # export could fail every remaining file and still be marked finished. - # A legitimate tail of already-present files still outnumbers its own - # stray failures, so it completes normally. - # - # The cursor still stands either way: the posts were handled, and their - # failures are in the sidecar. - log.error("%d files failed, outnumbering the %d downloaded and %d already " - "present - not marking this export complete; check connectivity " - "and re-run", totals.failed, totals.downloaded, totals.skipped) - else: - state.mark_completed() - return totals - - -def _tally(totals: Totals, result: Result, post_id: int) -> None: - if result.status == DOWNLOADED: - totals.downloaded += 1 - totals.bytes_downloaded += result.size or 0 - log.info("%s/%s %s", post_id, result.path.name, human_bytes(result.size or 0)) - elif result.status == SKIPPED: - totals.skipped += 1 - log.debug("%s/%s already present", post_id, result.path.name) - else: - totals.failed += 1 - log.warning("%s message %s: %s", post_id, result.message_id, result.error) - - -def write_title(root: Path, title: str) -> None: - """The human-readable group name lives here, as data - never as a path - component. Written once; a later rename updates it without moving anything. - - Server-supplied and editable by any admin, so it is stripped of the - unprintable categories before landing on disk: the export root is protected - by never putting the title in a path at all, but `cat title.txt` would - otherwise feed ANSI escapes or a right-to-left override to the operator's - terminal. - - Atomic like every other write here. A torn title.txt is only cosmetic, but it - was the one file in the tree that could be observed half-written. - """ - root.mkdir(parents=True, exist_ok=True) - target = root / TITLE_NAME - tmp = target.with_name(target.name + PART_SUFFIX) - with open(tmp, "w", encoding="utf-8") as f: - f.write(paths.strip_unprintable(title or "") + "\n") - f.flush() - os.fsync(f.fileno()) - os.replace(tmp, target) - fsync_dir(root) diff --git a/src/telegram_exporter/estimate.py b/src/telegram_exporter/estimate.py deleted file mode 100644 index 3b8f558..0000000 --- a/src/telegram_exporter/estimate.py +++ /dev/null @@ -1,179 +0,0 @@ -"""--dry-run: sweep, download nothing, report what a real run would fetch *from -where it would actually start*. - -run_estimate drives the same iter_posts generator with the same Filters as -run_download. That shared generator is the anti-drift property: an estimator with -its own traversal would report numbers that do not match reality. There is no -Sink Protocol - two implementations with one call site and no shared behavior is -indirection describing a for loop - so both run modes own a plain `async for`, -and the duplication is two lines. -""" - -from __future__ import annotations - -import shutil -from dataclasses import dataclass, field -from pathlib import Path - -from .downloader import human_bytes -from .media import has_media, media_kind, media_size -from .state import State -from .traversal import Filters - -# Photo .size describes the size *variant* Telethon selected, so photo totals are -# approximate. Documents, video and audio carry an exact document.size. -_LABELS = {"photo": "photos", "video": "videos", "document": "documents", - "audio": "audio files", "voice": "voice notes"} - - -@dataclass -class EstimateSink: - """Counts posts *with files*, files, and bytes by kind. - - Empty posts (a fully filtered group, which iter_posts yields to keep the - cursor advancing) and text-only posts under --include-text are not counted, - on both sides of any comparison - so the dry-run and real-run reconciliation - in phase 7 compares like with like. - """ - - posts: int = 0 - files: int = 0 - by_kind: dict[str, list[int]] = field(default_factory=dict) # kind -> [count, bytes] - unknown_size_files: int = 0 - - def add(self, post) -> None: - media = [m for m in post.messages if has_media(m)] - if not media: - return - self.posts += 1 - for msg in media: - self.files += 1 - kind = media_kind(msg) # has_media guarantees this is set - row = self.by_kind.setdefault(kind, [0, 0]) - row[0] += 1 - size = media_size(msg) - if size is None: - # Never coerced to 0: an unbounded estimate error must be visible. - self.unknown_size_files += 1 - else: - row[1] += size - - @property - def total_bytes(self) -> int: - return sum(row[1] for row in self.by_kind.values()) - - -def _measurable_dir(p: Path) -> Path: - """Nearest existing ancestor - dry-run must not create the export root just - to ask how much space is left.""" - for candidate in [p, *p.parents]: - if candidate.is_dir(): - return candidate - return Path.cwd() - - -def describe_filters(filters: Filters) -> str: - # `is not None` throughout: --max-size 0 is an active filter that drops - # everything, and printing it as "(none)" would misdescribe the run. - parts = [] - if filters.types is not None: - parts.append("types=" + ",".join(sorted(filters.types))) - if filters.since is not None: - parts.append(f"since={filters.since.date()}") - if filters.until is not None: - parts.append(f"until={filters.until.date()}") - if filters.max_size is not None: - parts.append(f"max-size={human_bytes(filters.max_size)}") - if filters.include_text: - parts.append("include-text") - return " ".join(parts) or "(none)" - - -def stored_filters_differ(root: Path, filters: Filters) -> bool: - """Whether a real run would refuse (exit 7) rather than resume.""" - data = State.read(root) - return bool(data) and data.get("filters") != filters.to_state() - - -def resume_from(root: Path, filters: Filters) -> int: - """Start where the real run would. - - Without this, `tg-export --dry-run && tg-export` fails *closed* on every - resumed run: a 38 GB group with 30 GB already downloaded and 11 GB free would - re-measure the whole 38 GB, exit 2, and refuse to fetch the 8 GB that fits - comfortably - leaving the user no option but to bypass the guard they were - told to trust. - - Deliberately not scanning the export dir to subtract bytes already on disk: - with a cursor-based sweep everything before the cursor is already excluded, - and the residue is a handful of failed files and one partial post. - """ - data = State.read(root) - if not data: - return 0 - if data.get("filters") != filters.to_state(): - return 0 # different filters: the real run would refuse - return int(data.get("cursor_id", 0)) - - -def _mode_line(root: Path, filters: Filters, start_id: int) -> str: - """Say which run this describes. - - When the stored filters differ, the numbers below are honest for a *fresh* - export but misleading as "remaining work", because a real run would refuse - with exit 7 until --reset-state. Saying so beats printing a total the next - command cannot act on. - """ - if start_id: - return f"resume from message {start_id}" - if stored_filters_differ(root, filters): - return ("full history - stored filters differ, so a real run would refuse " - "(exit 7) until --reset-state") - return "full history (no prior state)" - - -async def run_estimate(posts, *, sink: EstimateSink, root: Path, - title: str, peer_id: int, filters: Filters, - start_id: int, min_free: int, - limit: int | None = None) -> int: - """Consume the shared traversal, print the report, return the exit code. - - The verdict is advisory. Phase 5's per-file check_disk is the sole - enforcement mechanism, so a SHORT verdict does not mean the run is unsafe - - it means the run will stop partway with a clean, resumable cursor. - """ - if limit != 0: - async for post in posts: - sink.add(post) - # Same stop rule as run_download, so `--dry-run --limit N` and - # `--limit N` from the same start describe the same work. - if limit is not None and sink.posts >= limit: - break - - free = shutil.disk_usage(_measurable_dir(root)).free - total = sink.total_bytes - need = total + min_free - - print(f"Group: {title or '(untitled)'} (id {peer_id})") - print(f"Filters: {describe_filters(filters)}") - print(f"Mode: {_mode_line(root, filters, start_id)}") - print() - print(f"Remaining: {sink.posts:,} posts / {sink.files:,} files") - for kind, (count, size) in sorted(sink.by_kind.items()): - note = " (approximate - size variant)" if kind == "photo" else "" - print(f" {_LABELS.get(kind, kind):<14}{count:>6,} files " - f"{human_bytes(size):>12}{note}") - if sink.unknown_size_files: - print(f" {'unknown size':<14}{sink.unknown_size_files:>6,} files " - f"{'(not counted)':>12}") - print() - print(f"Total: {human_bytes(total)}") - print(f"Free: {human_bytes(free)} Reserve: {human_bytes(min_free)}") - - if free < need: - print(f"Verdict: SHORT by {human_bytes(need - free)}") - print(f" advisory - the run would stop partway with a " - f"resumable cursor, not corrupt anything") - return 2 - print(f"Verdict: FITS ({human_bytes(free - need)} to spare)") - return 0 diff --git a/src/telegram_exporter/media.py b/src/telegram_exporter/media.py deleted file mode 100644 index d448472..0000000 --- a/src/telegram_exporter/media.py +++ /dev/null @@ -1,106 +0,0 @@ -"""Media classification and size, in one place because three consumers need it. - -`--types` filtering, filename derivation, the download validator, the disk guard -and the estimator all have to agree on "what kind of media is this" and "how big -does the server say it is". Two copies of that answer would drift, and a drift -between the estimator and the downloader is exactly the lie the shared-traversal -design exists to prevent. - -Duck-typed on purpose: every accessor uses getattr with a default, so unit tests -can build a stub message with just the attributes a case needs, offline. -""" - -from __future__ import annotations - -from pathlib import PurePosixPath - -# The kinds `--types` accepts. "video" absorbs video notes and gifs; stickers -# fall through to "document" unless they carry a video attribute. -KINDS = ("photo", "video", "document", "audio", "voice") - - -def media_kind(msg) -> str | None: - """The media kind, or None when the message carries no downloadable file. - - A link preview is deliberately not media. Telethon's `Message.photo` and - `Message.document` also return the *web page preview's* photo or document, - so classifying with those properties alone would make every message - containing a URL look like an attachment - inflating counts and downloading - thumbnails nobody asked for. Checking web_preview first excludes them. - """ - if getattr(msg, "web_preview", None) is not None: - return None - if getattr(msg, "photo", None) is not None: - return "photo" - if getattr(msg, "voice", None) is not None: - return "voice" # a voice note is an audio document - if (getattr(msg, "video", None) is not None - or getattr(msg, "video_note", None) is not None - or getattr(msg, "gif", None) is not None): - return "video" - if getattr(msg, "audio", None) is not None: - return "audio" - if getattr(msg, "document", None) is not None: - return "document" - return None # text, poll, geo, contact, dice... - - -def has_media(msg) -> bool: - return media_kind(msg) is not None - - -def media_size(msg) -> int | None: - """Server-declared size in bytes, or None when the server did not say. - - Single source of truth. None is not zero, and each consumer has its own - stated rule for it: - - --max-size filter include, warn once (unknown-size media is - pathological and near-always small; a silent drop - would violate the no-silent-gaps ethos) - download validator accept - there is nothing to check against - estimator counted on a separate "unknown size" line, never - coerced to 0, because an unbounded estimate error - has to be visible - disk guard treated as 0, absorbed by the --min-free reserve - - Comparing `None <= limit` unguarded raises TypeError inside the sweep and - kills a 20-hour export. That is the failure this helper exists to prevent. - """ - f = getattr(msg, "file", None) - if f is None: - return None - size = getattr(f, "size", None) - return size if isinstance(size, int) else None - - -def file_stem(msg) -> str: - """Stem for the filename, before sanitization. Untrusted: the name comes - from DocumentAttributeFilename, which any group member controls.""" - name = getattr(getattr(msg, "file", None), "name", None) - if name: - stem = PurePosixPath(str(name)).stem - if stem: - return stem - return media_kind(msg) or "media" # photos and voice notes carry no name - - -def file_ext(msg) -> str: - """Extension including the dot, falling back to .bin. - - Prefers the extension of the declared filename, then Telethon's mime-derived - File.ext. Never empty, so a name is always recognizable on disk. - """ - f = getattr(msg, "file", None) - name = getattr(f, "name", None) - if name: - suffix = PurePosixPath(str(name)).suffix - # Byte-counted, like the stem truncation in paths.sanitize: 16 multibyte - # characters would be 48+ bytes, and "{id}_" + 200 stem bytes + that - # would overflow ext4's 255-byte component limit. - if suffix and len(suffix.encode()) <= 16: - return suffix - ext = getattr(f, "ext", None) - if ext: - return ext if str(ext).startswith(".") else f".{ext}" - return ".bin" diff --git a/src/telegram_exporter/paths.py b/src/telegram_exporter/paths.py deleted file mode 100644 index bd6fed1..0000000 --- a/src/telegram_exporter/paths.py +++ /dev/null @@ -1,194 +0,0 @@ -"""All path construction, from the export root down to the filename leaf. - -Pure functions, no network, no I/O - which is why this is its own module rather -than a helper inside downloader.py, and why every rule below is table-tested -offline. - - The governing invariant: exactly one untrusted string ever becomes a path - component - the filename leaf. - -Every other component is an int, enforced by a type check, which makes -safe_join's "post_dir is trusted" precondition true by type rather than by -assumption. Two untrusted strings were in play before this module existed: - -1. DocumentAttributeFilename, supplied by arbitrary group members. - `../../../.ssh/authorized_keys` is a cheap, real attack. -2. The group title - server-supplied and editable by any admin. safe_join could - never have caught that one, because the escape happens in a parent component - before the join. It is fixed by never putting the title in a path at all. -""" - -from __future__ import annotations - -import os -import re -import unicodedata -from pathlib import Path - -from .media import file_ext, file_stem, media_kind - -# Characters removed outright rather than replaced: a NUL truncates the filename -# at the syscall boundary, so leaving a placeholder would only hide what -# happened. -# -# Cc C0 and C1 controls, including NUL and the ESC that starts an ANSI escape -# Cf format characters - U+202E RIGHT-TO-LEFT OVERRIDE is the one that -# matters: `agpj.exe` renders as `a.exe.jpg` in any terminal or file -# manager, so stripping only C0 would close the ANSI vector while leaving -# the older extension-spoofing one wide open -# Zl/Zp line and paragraph separators, which break line-oriented consumers of -# a file listing -# Cs lone surrogates, which cannot be encoded to UTF-8 at all -_STRIPPED_CATEGORIES = frozenset({"Cc", "Cf", "Zl", "Zp", "Cs"}) - - -def _is_disallowed(ch: str) -> bool: - return unicodedata.category(ch) in _STRIPPED_CATEGORIES - -# Path separators and shell/Windows reserved characters become underscores. -_RESERVED = str.maketrans({ch: "_" for ch in '/\\:*?"<>|'}) - -_WHITESPACE_RUN = re.compile(r"\s+") - -# ext4 accepts 255 bytes per component; 200 leaves room for the "{id}_" prefix -# and the extension, both of which are byte-counted too. -MAX_NAME_BYTES = 200 -MAX_EXT_BYTES = 16 - - -def strip_unprintable(text: str) -> str: - """Strip the categories that make a string unsafe to *print*, leaving it - otherwise intact. - - sanitize() below reduces an untrusted string to a single path component. - This does the much smaller job that untrusted text written as *data* needs - - title.txt - where spaces, punctuation and slashes are all legitimate content, - but an ANSI escape or a U+202E override still reaches the operator's terminal - the moment they `cat` the file. Same category set, so the two rules cannot - drift apart. - """ - return "".join(ch for ch in str(text) if not _is_disallowed(ch)) - - -def export_root(out_dir: Path, peer_id: int) -> Path: - """`/g` - the directory name is the chat id, never the title. - - peer_id comes from telethon.utils.get_peer_id(entity). An int cannot - traverse, cannot collapse into the parent when empty, cannot collide across - groups, and cannot strand an in-progress export when an admin renames the - group. The human-readable name is written to /title.txt as data. - - The `g` prefix only avoids a leading `-` in shell arguments; it is ergonomic, - not a security property. - """ - _require_int(peer_id, "peer_id") - return (Path(out_dir) / f"g{peer_id}").resolve() - - -def post_dir(root: Path, post_id: int) -> Path: - """One directory per logical post, named for the lowest message id in it.""" - _require_int(post_id, "post_id") - return root / str(post_id) - - -def _require_int(value: object, label: str) -> None: - # bool is an int subclass, and a bool here always means a caller bug. - if not isinstance(value, int) or isinstance(value, bool): - raise TypeError(f"{label} must be an int, got {type(value).__name__}: {value!r}") - - -def derive_filename(msg) -> str: - """`{message_id}_{sanitized stem}{ext}` - keyed on the message id. - - One Telegram message carries at most one media, so the message id makes - uniqueness inside a post folder structural rather than positional. - - A positional index is unstable under history mutation: delete one album - member between runs and every later index shifts, silently orphaning files - already on disk. It would also force Post to carry the full pre-filter - member list purely for naming, reintroducing the filter coupling that - Invariant 1 exists to remove. - - The redundancy in `1042/1042_photo.jpg` buys self-identifying files and lets - `grep 1043 messages.jsonl` join sidecar to disk with no index arithmetic. - """ - _require_int(getattr(msg, "id", None), "msg.id") - stem = sanitize(file_stem(msg), fallback=media_kind(msg) or "media") - return f"{msg.id}_{stem}{sanitize_ext(file_ext(msg))}" - - -def sanitize_ext(ext: str) -> str: - """The extension is part of the untrusted leaf and gets the same treatment. - - Sanitizing only the stem left a hole: the extension is derived from the same - attacker-supplied DocumentAttributeFilename, so `holiday.jpg` produced a - name containing a NUL, and the ValueError that Path.resolve then raised - escaped the download loop - halting the export on that message on every - subsequent run, since the cursor could not advance past it. Control - characters also meant an operator's terminal could be fed ANSI escapes from - a group member's filename. - """ - body = sanitize(str(ext).lstrip("."), fallback="bin", max_bytes=MAX_EXT_BYTES) - return f".{body}" - - -def sanitize(name: str, *, fallback: str, max_bytes: int = MAX_NAME_BYTES) -> str: - """Reduce an untrusted string to one safe path component. - - Order matters: control characters go before the leaf is taken, so a NUL - cannot confuse the split; the leaf is taken before separators are replaced, - so `../../../etc/passwd` loses its directories rather than becoming - `.._.._.._etc_passwd`. - """ - s = "".join(ch for ch in str(name) if not _is_disallowed(ch)) - s = _leaf(s) # discards every directory part, incl. .. - s = s.translate(_RESERVED) - s = _WHITESPACE_RUN.sub(" ", s).strip() - s = _strip_edges(s) - s = _truncate_utf8(s, max_bytes) - s = _strip_edges(s) # truncation can expose a new leading dot - return s or fallback - - -def _leaf(s: str) -> str: - """The final component only. Both separators, so a Windows-style path passed - to a Linux host still loses its directories.""" - return Path(s.replace("\\", "/")).name - - -def _strip_edges(s: str) -> str: - """Leading dots hide files; leading dashes make a filename look like a flag - to any tool the export is later piped through. Trailing dots and spaces are - stripped because they survive on Linux and silently vanish elsewhere.""" - return s.lstrip(". -\t").rstrip(". \t") - - -def _truncate_utf8(s: str, max_bytes: int) -> str: - """Truncate by UTF-8 bytes, preserving the extension. - - ext4's limit is 255 *bytes*, and a name of CJK or emoji characters overflows - it well before 255 characters - a character-counted truncation raises - OSError: File name too long partway through an export. - """ - if len(s.encode()) <= max_bytes: - return s - stem, ext = os.path.splitext(s) - ext_bytes = ext.encode() - if len(ext_bytes) > max_bytes: # pathological "extension"; drop it - stem, ext, ext_bytes = s, "", b"" - budget = max_bytes - len(ext_bytes) - # errors="ignore" discards a partial multi-byte character at the cut. - return stem.encode()[:budget].decode("utf-8", "ignore") + ext - - -def safe_join(post_dir: Path, filename: str) -> Path: - """Join and assert containment. Raises on escape - never silently repairs. - - sanitize should make this unreachable. Reaching it means sanitize has a hole, - so repairing here would convert a discovered bug into a silent one. - """ - base = Path(post_dir).resolve() - resolved = (base / filename).resolve() - if resolved == base or not resolved.is_relative_to(base): - raise ValueError(f"path escape attempt: {filename!r} against {post_dir}") - return resolved diff --git a/src/telegram_exporter/session.py b/src/telegram_exporter/session.py deleted file mode 100644 index cdcde8f..0000000 --- a/src/telegram_exporter/session.py +++ /dev/null @@ -1,303 +0,0 @@ -"""Credentials, session file preparation, the authenticated client, and the two -flood-wait primitives every network call routes through. - -Credential policy (the one authoritative copy): - - api_id, api_hash | env TG_API_ID / TG_API_HASH only | needed every run - phone | prompt; TG_PHONE optional | PII, not a secret - login code | prompt only | single-use, 5 min TTL - 2FA password | getpass only - no env, no file, no flag - -Nothing here reads a .env file. The shell already does that -(`set -a; . ./.env; set +a`). -""" - -from __future__ import annotations - -import asyncio -import logging -import os -import random -import stat -import subprocess -from contextlib import asynccontextmanager -from datetime import UTC, datetime, timedelta -from getpass import getpass -from pathlib import Path -from typing import AsyncIterator, Callable - -from telethon import TelegramClient, utils -from telethon.errors import FloodWaitError, SessionPasswordNeededError - -log = logging.getLogger("telegram_exporter.session") - -SUFFIX = ".session" - -# Routine short waits are absorbed inside Telethon rather than bubbling up. -FLOOD_SLEEP_THRESHOLD = 120 - - -class Abort(Exception): - """A run-ending condition that carries the process exit code. - - Raised instead of calling sys.exit deep in the call stack, so the caller - decides when to exit and the resume cursor is never written from an error - path. - """ - - def __init__(self, reason: str, code: int) -> None: - super().__init__(reason) - self.reason = reason - self.code = code - - -# --------------------------------------------------------------------------- # -# Session path -# --------------------------------------------------------------------------- # - -def default_session_path() -> Path: - """Outside the repo by default, so the safe location needs no opt-in and the - git check only bites a deliberately chosen path.""" - base = os.environ.get("XDG_STATE_HOME") or str(Path.home() / ".local" / "state") - return Path(base) / "tg-export" / f"default{SUFFIX}" - - -def with_session_suffix(p: Path) -> Path: - """Telethon appends '.session' when absent, so every consumer - the privacy - assertion, the git check, the lock file - has to agree on the real filename. - Pure: creates nothing, so callers can check a path before it exists.""" - return p if p.name.endswith(SUFFIX) else p.with_name(p.name + SUFFIX) - - -def prepare_session_path(p: Path) -> Path: - """Create the session file 0600 *before* Telethon can create it 0644. - - sqlite3.connect() on a fresh path yields 0644 under a default umask, and the - auth key is written during login - so a post-hoc chmod is too late. cli.main - sets umask(0o077) first; this is the belt to that braces. - """ - p = with_session_suffix(p) - p.parent.mkdir(parents=True, exist_ok=True) - try: - os.close(os.open(p, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)) - except FileExistsError: - os.chmod(p, 0o600) - return p - - -def assert_session_private(p: Path) -> None: - """Every sqlite sibling too: -journal, -wal and -shm hold the same secrets.""" - for f in sorted(p.parent.glob(p.name + "*")): - mode = stat.S_IMODE(f.stat().st_mode) - if mode & 0o077: - raise SystemExit(f"insecure mode on {f}: {oct(mode)}") - - -def _nearest_existing_dir(p: Path) -> Path | None: - """First existing ancestor directory, used as git's cwd. - - The target usually does not exist yet - an export root on a first run, or a - session path under a directory we have not created. Running git from a - nonexistent cwd raises FileNotFoundError before git can answer. - """ - for candidate in [p, *p.parents]: - if candidate.is_dir(): - return candidate - return None - - -def _check_ignore(target: str | Path, cwd: Path) -> int: - """git check-ignore rc: 0 = ignored, 1 = NOT ignored, 128 = not a work tree. - Returns 0 when git is unavailable, so a missing git never blocks a run.""" - try: - return subprocess.run(["git", "check-ignore", "-q", str(target)], - cwd=cwd, capture_output=True).returncode - except (FileNotFoundError, NotADirectoryError): - return 0 # no git installed, or the tree moved under us - not our problem - - -def assert_not_committable(p: Path, *, is_dir: bool = False) -> None: - """Refuse a session file or export root that git would happily commit. - - Adding .gitignore patterns cannot help when --out and --session point - anywhere, so this is checked at runtime against the actual path. - - is_dir must be True only for a path that will become a directory. The - trailing-slash retry below is what makes a directory-only pattern match - before the directory exists - and asking about a *file* that way would be - unsound in the other direction: a `.gitignore` line of `creds.session/` - would then excuse a file named `creds.session`, which git would commit. - """ - target = p.absolute() - cwd = _nearest_existing_dir(target) - if cwd is None: - return - rc = _check_ignore(target, cwd) - if rc == 1 and is_dir and not target.exists(): - # A directory-only pattern such as `exports/` does not match a path git - # cannot see is a directory, so an export root would be refused on the - # first run and accepted on every run after it. The trailing slash tells - # git to read it as a directory - passed as a string, because Path drops - # it. Verified empirically against git's exit codes. - rc = _check_ignore(f"{target}{os.sep}", cwd) - if rc == 1: - raise SystemExit( - f"refusing: {p} is inside a git repo and is not gitignored.\n" - f"add it to .gitignore, or choose a path outside the repo.") - - -# --------------------------------------------------------------------------- # -# Flood-wait primitives - two, because one cannot cover both shapes -# --------------------------------------------------------------------------- # - -async def _flood_sleep(e: FloodWaitError, max_wait_s: int | None, - where: str = "") -> None: - """Sleep out one flood wait, or abort if it exceeds an explicit ceiling. - - max_wait_s is None by default: sleep however long Telegram demands. Always - log the computed WAKE TIME, not just the duration - an unattended four-hour - sleep has to read as a sleep rather than as a hang. That log line is the - mitigation for the no-ceiling default. - """ - if max_wait_s is not None and e.seconds > max_wait_s: - raise Abort(f"flood wait {e.seconds}s exceeds --max-flood-wait {max_wait_s}s", 6) - wake = datetime.now(UTC) + timedelta(seconds=e.seconds) - log.warning("flood wait %ss%s - sleeping until %s", - e.seconds, where, wake.isoformat(timespec="seconds")) - await asyncio.sleep(e.seconds + random.uniform(1.0, 3.0)) - - -async def with_flood_retry(make_coro: Callable[[], object], *, - max_wait_s: int | None = None): - """Download path. Never tight-retry: a retry during an active wait is a - fresh violation that escalates the next wait.""" - while True: - try: - return await make_coro() - except FloodWaitError as e: - await _flood_sleep(e, max_wait_s) - - -async def aiter_with_flood_retry(make_agen, *, start_id: int, - max_wait_s: int | None = None): - """Sweep path. - - iter_messages is consumed with `async for`, and Telethon raises - FloodWaitError on the *next page fetch* - deep inside the loop, where a - wrapper guarding only construction never sees it. Rebuilding the generator - from the last id actually yielded is what keeps the sweep gap-free: with - reverse=True, offset_id is an exclusive lower bound, so no message is - skipped or replayed. - """ - last = start_id - while True: - try: - async for item in make_agen(last): - last = item.id - yield item - return - except FloodWaitError as e: - await _flood_sleep(e, max_wait_s, where=f" mid-sweep after id {last}") - - -# --------------------------------------------------------------------------- # -# Credentials and client -# --------------------------------------------------------------------------- # - -def load_api_credentials() -> tuple[int, str]: - api_id, api_hash = os.environ.get("TG_API_ID"), os.environ.get("TG_API_HASH") - if not api_id or not api_hash: - raise Abort( - "missing credentials: set TG_API_ID and TG_API_HASH.\n" - " create an app at https://my.telegram.org -> API development tools\n" - " then: export TG_API_ID=... TG_API_HASH=...", 1) - try: - return int(api_id), api_hash - except ValueError: - raise Abort(f"TG_API_ID must be an integer, got {api_id!r}", 1) from None - - -async def _login(client: TelegramClient) -> None: - """Interactive first login. Later runs return immediately.""" - if await client.is_user_authorized(): - return - phone = os.environ.get("TG_PHONE") or input("phone (+countrycode): ").strip() - if not phone: - raise Abort("no phone number given", 1) - await client.send_code_request(phone) - code = input("login code (sent via Telegram): ").strip() - try: - await client.sign_in(phone, code) - except SessionPasswordNeededError: - # 2FA password: getpass only. Never argv, never env, never a file. - await client.sign_in(password=getpass("2FA password: ")) - - -@asynccontextmanager -async def connected_client(session_path: Path, api_id: int, - api_hash: str) -> AsyncIterator[TelegramClient]: - """Connected, authorized client that always disconnects. - - Telethon does not flush SQLite session state without disconnect(), so this - is a context manager rather than a discipline. - """ - # Check before creating: a refused path should not be left holding an empty - # session file. - session_path = with_session_suffix(session_path) - assert_not_committable(session_path) - session_path = prepare_session_path(session_path) - client = TelegramClient(str(session_path), api_id, api_hash) - client.flood_sleep_threshold = FLOOD_SLEEP_THRESHOLD - await client.connect() - try: - await _login(client) - assert_session_private(session_path) - yield client - finally: - await client.disconnect() - - -async def resolve_entity(client: TelegramClient, spec: str): - """Accept a numeric id, @username, or t.me link. Returns (entity, peer_id). - - peer_id is telethon.utils.get_peer_id(entity) - the canonical int that - becomes the export directory name. - """ - if "/+" in spec or "joinchat/" in spec: - raise Abort( - f"{spec} is a private invite link, which cannot be resolved without " - f"joining.\njoin the group in a Telegram client first, then pass its " - f"numeric id or @username.", 5) - - target = spec.strip() - try: - entity = await client.get_entity(int(target)) - except ValueError as e: - # Not an int, or an int Telethon has never seen. Raw ids resolve only - # from the session cache, so fall back to a dialog scan before giving up. - try: - entity = await client.get_entity(target) - except ValueError: - entity = await _find_in_dialogs(client, target) - if entity is None: - raise Abort( - f"cannot resolve group {spec!r}: {e}\n" - f"confirm this account is a member, and try the @username or " - f"the numeric id.", 5) from None - return entity, utils.get_peer_id(entity) - - -async def _find_in_dialogs(client: TelegramClient, target: str): - """Last resort for a numeric id absent from the session cache.""" - try: - wanted = int(target) - except ValueError: - return None - async for dialog in client.iter_dialogs(): - if utils.get_peer_id(dialog.entity) == wanted: - return dialog.entity - return None - - -def now_iso() -> str: - return datetime.now(UTC).isoformat(timespec="seconds") diff --git a/src/telegram_exporter/sidecar.py b/src/telegram_exporter/sidecar.py deleted file mode 100644 index f5c4554..0000000 --- a/src/telegram_exporter/sidecar.py +++ /dev/null @@ -1,167 +0,0 @@ -"""messages.jsonl - the append-only log linking every file on disk to its message. - -A record is an event: "the files written for this message in this run". The -correct consumer rule is therefore the UNION of all records for a message_id, -not last-write-wins. Last-write-wins is actively wrong: after a filter change it -would report a narrower file list than what is actually on disk. -""" - -from __future__ import annotations - -import json -import logging -import os -from datetime import UTC, datetime -from pathlib import Path - -from .state import fsync_dir - -log = logging.getLogger("telegram_exporter.sidecar") - -SIDECAR_NAME = "messages.jsonl" - - -class Sidecar: - """Append-only JSONL writer. Opened for the life of the run.""" - - def __init__(self, root: Path) -> None: - self.root = root - self.path = root / SIDECAR_NAME - self._fh = None - - # ---- lifecycle ------------------------------------------------------ # - - def open(self) -> Sidecar: - self.root.mkdir(parents=True, exist_ok=True) - self.repair() - self._fh = open(self.path, "a", encoding="utf-8") - return self - - def close(self) -> None: - if self._fh is not None: - self._fh.close() - self._fh = None - - def __enter__(self) -> Sidecar: - return self.open() - - def __exit__(self, *exc) -> None: - self.close() - - def repair(self) -> None: - """Truncate a partial trailing line written when the host died mid-write. - - One unterminated line breaks every JSONL consumer, and the damage is - always confined to the tail, so scanning back from EOF for the last - newline is both sufficient and cheap on a 100k-line file. - """ - if not self.path.exists() or self.path.stat().st_size == 0: - return - with open(self.path, "rb+") as f: - f.seek(0, os.SEEK_END) - end = f.tell() - # Grow the window until a newline is actually found. A record longer - # than the window contains no newline at all, and truncating to - # `end - window` would then cut in the middle of an earlier, valid - # record - leaving the file just as unreadable, one megabyte shorter. - window = min(end, 1 << 20) - while True: - f.seek(end - window) - tail = f.read(window) - cut = tail.rfind(b"\n") - if cut != -1 or window == end: - break - window = min(end, window * 2) - trailing = tail[cut + 1:] - if not trailing: - return # file ends on a newline: intact - try: - json.loads(trailing) - except (json.JSONDecodeError, UnicodeDecodeError): - offset = end - len(trailing) - f.truncate(offset) - log.warning("repaired %s: truncated %d trailing bytes of a partial " - "record at offset %d", self.path, len(trailing), offset) - return - # Valid JSON but no terminating newline - complete record, just add one. - f.write(b"\n") - - def rotate(self) -> Path | None: - """--reset-state rotates rather than deletes, so the current sidecar always - describes exactly one filter regime while the old one stays inspectable.""" - if not self.path.exists(): - return None - stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") - target = self.root / f"messages-{stamp}.jsonl" - # Two resets inside the same second must not clobber the first archive: - # rotating instead of deleting is only worth doing if the old log stays - # inspectable, and os.replace would have overwritten it silently. - serial = 1 - while target.exists(): - target = self.root / f"messages-{stamp}-{serial}.jsonl" - serial += 1 - os.replace(self.path, target) - fsync_dir(self.root) # a rename is not durable until its dir entry is - log.info("rotated %s -> %s", self.path.name, target.name) - return target - - # ---- writing -------------------------------------------------------- # - - def append(self, post, results) -> None: - """One record per surviving message in the post. - - Text-only messages under --include-text produce the same shape with - "files": [] - consumers distinguish media from context by files being - empty, not by a separate record type. - """ - by_message = {} - for r in results: - by_message.setdefault(r.message_id, []).append(r) - - for msg in post.messages: - rs = by_message.get(msg.id, []) - record = { - "message_id": msg.id, - "post_id": post.post_id, - "grouped_id": getattr(msg, "grouped_id", None), - "date": _iso(getattr(msg, "date", None)), - "sender_id": getattr(msg, "sender_id", None), - "sender_name": _sender_name(msg), - "caption": (getattr(msg, "message", None) or None), - "reply_to": _reply_to(msg), - "files": [r.as_record(self.root) for r in rs if r.path is not None], - "errors": [r.error for r in rs if r.error is not None], - } - self._fh.write(json.dumps(record, ensure_ascii=False) + "\n") - - def fsync(self) -> None: - """Called before the cursor commits: the sidecar must be durable before - the state file claims the post is done.""" - self._fh.flush() - os.fsync(self._fh.fileno()) - - -def _iso(value) -> str | None: - return value.isoformat() if isinstance(value, datetime) else None - - -def _reply_to(msg) -> int | None: - header = getattr(msg, "reply_to", None) - return getattr(header, "reply_to_msg_id", None) if header is not None else None - - -def _sender_name(msg) -> str | None: - """Only from an already-cached sender. - - Never an extra get_entity call: on a 100k-message sweep that is 100k extra - RPCs and a flood ban, in exchange for a display string. - """ - sender = getattr(msg, "sender", None) - if sender is None: - return None - title = getattr(sender, "title", None) - if title: - return title - parts = [getattr(sender, "first_name", None), getattr(sender, "last_name", None)] - name = " ".join(p for p in parts if p) - return name or getattr(sender, "username", None) diff --git a/src/telegram_exporter/state.py b/src/telegram_exporter/state.py deleted file mode 100644 index 9108447..0000000 --- a/src/telegram_exporter/state.py +++ /dev/null @@ -1,172 +0,0 @@ -"""The resume cursor, and nothing else. - -state.py and sidecar.py are separate modules on purpose. The correctness-critical -part is the *ordering* between them - sidecar first, cursor second - and that -ordering belongs to the caller. Two objects in two modules makes the sequence -visible at one call site, and makes a "helpful" commit_and_append() impossible to -write without noticing that it destroys Invariant 2. -""" - -from __future__ import annotations - -import json -import logging -import os -from pathlib import Path - -from .session import Abort, now_iso -from .traversal import Filters - -log = logging.getLogger("telegram_exporter.state") - -STATE_NAME = ".export-state.json" -VERSION = 1 - - -def fsync_dir(path: Path) -> None: - """A rename is not durable until its directory entry is. - - Lives here rather than in downloader.py because durability is what this - module exists for; downloader.py imports it for the post-directory fsync that - has to happen before the cursor claims a post is done. - """ - fd = os.open(path, os.O_RDONLY) - try: - os.fsync(fd) - finally: - os.close(fd) - - -class State: - """cursor_id means: every post with max_message_id <= cursor_id has been - handled *under these filters*. - - One cursor field, not two. The sweep is a generator driven by the sink at - concurrency 1, so it can never run ahead of the downloader - a second - "swept_to" field would invent a divergence to manage. - """ - - def __init__(self, path: Path, data: dict) -> None: - self.path = path - self.data = data - - # ---- construction -------------------------------------------------- # - - @classmethod - def path_for(cls, root: Path) -> Path: - return root / STATE_NAME - - @classmethod - def read(cls, root: Path) -> dict | None: - """Read-only load for --dry-run. Never creates anything.""" - p = cls.path_for(root) - if not p.exists(): - return None - try: - return json.loads(p.read_text()) - except (json.JSONDecodeError, OSError) as e: - raise Abort(f"cannot read {p}: {e}\n" - f"inspect it, or use --reset-state to start over", 7) from None - - @classmethod - def open(cls, root: Path, *, chat_id: int, chat_title: str, - filters: Filters, reset: bool = False) -> State: - """Load and validate, or create. Refuses a mismatch rather than guessing.""" - path = cls.path_for(root) - existing = None if reset else cls.read(root) - - if existing is not None: - _assert_compatible(existing, chat_id=chat_id, filters=filters) - existing["chat_title"] = chat_title # titles change; that is fine - existing["completed_at"] = None # this run has not finished yet - existing["updated_at"] = now_iso() - state = cls(path, existing) - else: - state = cls(path, { - "version": VERSION, - "chat_id": chat_id, - "chat_title": chat_title, - "filters": filters.to_state(), - "cursor_id": 0, - "completed_at": None, - "started_at": now_iso(), - "updated_at": now_iso(), - }) - state.save() - return state - - # ---- accessors ------------------------------------------------------ # - - @property - def cursor_id(self) -> int: - return int(self.data["cursor_id"]) - - # ---- mutation ------------------------------------------------------- # - - def commit(self, cursor_id: int) -> None: - """Invariant 2: only ever called after a COMPLETE post, and unreachable - from every error exit, so the cursor can never point past in-flight work.""" - if cursor_id < self.cursor_id: - log.debug("ignoring cursor regression %s -> %s", self.cursor_id, cursor_id) - return - self.data["cursor_id"] = int(cursor_id) - self.save() - - def mark_completed(self) -> None: - """Set only on clean exhaustion of the sweep. The only way to answer - "did my 20-hour export finish?" without guessing.""" - self.data["completed_at"] = now_iso() - self.save() - - def save(self) -> None: - """Atomic: tmp -> fsync -> replace -> fsync(dir). A half-written state - file would be indistinguishable from a corrupted cursor.""" - self.data["updated_at"] = now_iso() - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp = self.path.with_name(self.path.name + ".tmp") - with open(tmp, "w") as f: - json.dump(self.data, f, indent=2, sort_keys=True) - f.write("\n") - f.flush() - os.fsync(f.fileno()) - os.replace(tmp, self.path) - fsync_dir(self.path.parent) - - -def _assert_compatible(existing: dict, *, chat_id: int, filters: Filters) -> None: - """Invariant 3: filters are stored verbatim, and a mismatch names what changed. - - A sha256 fingerprint would be smaller to store and useless here: the whole - point is to *print the diff*, and a digest is one-way. - - --limit is deliberately absent from the filter set. It changes when we stop, - not what a post contains, so including it would let a `--limit 20` test run - poison every later full run. - """ - if int(existing.get("chat_id", 0)) != chat_id: - raise Abort( - f"chat mismatch: state file belongs to chat {existing.get('chat_id')}, " - f"not {chat_id}\nexport each group to its own --out directory", 7) - - if not isinstance(existing.get("filters"), dict): - # Absent or null is not the same condition as changed, and reporting it - # as a diff sends the user chasing a flag they never passed. Only a - # hand-edited or truncated state file gets here. - raise Abort( - f"state file has no filter set recorded, so the cursor's meaning is " - f"unknown - it means 'handled through here under these filters'.\n" - f"use --reset-state to start over", 7) - - was, now = existing["filters"], filters.to_state() - changed = [k for k in sorted(set(was) | set(now)) if was.get(k) != now.get(k)] - if changed: - lines = "\n".join(f" {k+':':12} {_show(was.get(k))} -> {_show(now.get(k))}" - for k in changed) - raise Abort( - f"filter mismatch:\n{lines}\n" - f"the cursor means 'handled through here under these filters'.\n" - f"re-run with the original filters, or --reset-state to start over", 7) - - -def _show(v: object) -> str: - return "(none)" if v is None else repr(v) diff --git a/src/telegram_exporter/traversal.py b/src/telegram_exporter/traversal.py deleted file mode 100644 index 4426a7a..0000000 --- a/src/telegram_exporter/traversal.py +++ /dev/null @@ -1,239 +0,0 @@ -"""The single traversal generator both run modes consume. - -Chronological sweep, album buffering, filter application. Yields one Post per -logical post a human would recognize. - -`run_download()` and `run_estimate()` drive *this* generator with the same -Filters. That shared generator - not any class hierarchy - is what guarantees the -dry-run estimate describes what a real run would download. A forked traversal -would drift and lie, so never fork it. -""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass, field -from datetime import UTC, datetime -from typing import Any - -from .media import KINDS, media_kind, media_size -from .session import aiter_with_flood_retry - -log = logging.getLogger("telegram_exporter.traversal") - -# An album is capped at 10 members by Telegram; 20 is slack for a protocol -# change, and exceeding it means the grouping logic is wrong, not that Telegram -# got generous. -MAX_ALBUM_BUFFER = 20 - -_SIZE_UNITS = {"": 1, "B": 1, "K": 10**3, "KB": 10**3, "KIB": 2**10, - "M": 10**6, "MB": 10**6, "MIB": 2**20, - "G": 10**9, "GB": 10**9, "GIB": 2**30, - "T": 10**12, "TB": 10**12, "TIB": 2**40} - - -class TraversalError(Exception): - """The sweep is not the shape the whole design assumes. Never repaired: a - wrong assumption here produces silent corruption that looks like success.""" - - -class AlbumSplitError(TraversalError): - """A grouped_id reopened after its run closed - the album was split across a - non-adjacent boundary, so folder identity is no longer trustworthy.""" - - -class SweepOrderError(TraversalError): - """The sweep is not strictly ascending, or an album exceeded its bound. - - A real exception rather than a bare assert: `python -O` strips assertions, - and these two tripwires guard the same class of silent corruption as - AlbumSplitError sitting beside them. - """ - - -@dataclass(frozen=True) -class Post: - """One logical post: a standalone message, or every member of one album. - - grouped_id is deliberately absent. The sidecar reads it off the message and - folder identity uses post_id, so a field here would be a third copy of a - fact with two owners already. - """ - - post_id: int # lowest message id in the FULL group (pre-filter) - messages: list[Any] # filter-surviving members, ascending by id - max_message_id: int # highest id in the FULL group - the cursor value - - -@dataclass(frozen=True) -class Filters: - """What a post *contains*. Stored verbatim in the state file, because the - cursor means "handled through here under these filters".""" - - types: frozenset[str] | None = None - since: datetime | None = None - until: datetime | None = None - max_size: int | None = None - include_text: bool = False - - # Set once, so the unknown-size warning does not repeat per file. - _warned: set[str] = field(default_factory=set, compare=False, repr=False) - - def to_state(self) -> dict: - """The stored form. Verbatim and comparable - no digest, because two - call sites have to *name* what changed, and a hash cannot. - - `is not None`, not truthiness: an empty set is a filter that drops - everything, and storing it as "no filter" would let a later unfiltered - run inherit an end-of-history cursor and report success having downloaded - nothing. build_filters refuses to construct one, so this is the second - line of defense on Invariant 3. - """ - return { - "types": sorted(self.types) if self.types is not None else None, - "since": self.since.isoformat() if self.since else None, - "until": self.until.isoformat() if self.until else None, - "max_size": self.max_size, - "include_text": self.include_text, - } - - -def parse_size(text: str) -> int: - """`100MB`, `2GiB`, `1500000`. Decimal units are powers of 10, `*iB` are - powers of 2 - the same convention `ls -h` and `df -h` use.""" - raw = str(text).strip().replace(" ", "") - i = 0 - while i < len(raw) and (raw[i].isdigit() or raw[i] == "."): - i += 1 - number, unit = raw[:i], raw[i:].upper() - if not number or unit not in _SIZE_UNITS: - raise ValueError(f"cannot parse size {text!r}; try 100MB, 2GiB or a byte count") - return int(float(number) * _SIZE_UNITS[unit]) - - -def parse_date(text: str) -> datetime: - """`--since`/`--until` as UTC-aware, because msg.date is UTC-aware and a - naive comparison raises TypeError mid-sweep.""" - value = datetime.fromisoformat(str(text).strip()) - return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) - - -def build_filters(*, types: str | None = None, since: str | None = None, - until: str | None = None, max_size: str | None = None, - include_text: bool = False) -> Filters: - kinds = None - if types: - kinds = frozenset(t.strip().lower() for t in types.split(",") if t.strip()) - unknown = kinds - set(KINDS) - if unknown: - raise ValueError(f"unknown media types {sorted(unknown)}; " - f"choose from {', '.join(KINDS)}") - if not kinds: - # `--types " "` or `--types ,` - easy to produce from an unset shell - # variable. Left alone it is a filter that drops every media kind, - # which would sweep the whole history, download nothing, and commit - # an end-of-history cursor. - raise ValueError(f"--types {types!r} names no media kind; " - f"choose from {', '.join(KINDS)}, or omit the flag") - return Filters( - types=kinds, - since=parse_date(since) if since else None, - until=parse_date(until) if until else None, - max_size=parse_size(max_size) if max_size else None, - include_text=include_text, - ) - - -def keep(msg, filters: Filters) -> bool: - """Whether this message survives the filters. - - Text-only messages are dropped unless --include-text, in which case they - reach the sidecar with an empty files[] so captions and replies keep their - surrounding conversation. They never create a post folder and never reach - download_one. - """ - kind = media_kind(msg) - if kind is None: - return filters.include_text - if filters.types is not None and kind not in filters.types: - return False - date = getattr(msg, "date", None) - if filters.since is not None and (date is None or date < filters.since): - return False - if filters.until is not None and (date is None or date > filters.until): - return False - if filters.max_size is not None: - size = media_size(msg) - if size is None: - # Include and warn once. A silent drop violates the no-silent-gaps - # ethos, and unknown-size media is pathological and near-always small. - if "max_size_none" not in filters._warned: - filters._warned.add("max_size_none") - log.warning("some media reports no size; --max-size cannot apply " - "to it and it is being included") - elif size > filters.max_size: - return False - return True - - -async def iter_posts(client, entity, *, after_id: int = 0, - filters: Filters, max_flood_wait: int | None = None): - """Sweep oldest -> newest, yielding one Post per logical post. - - after_id is an exclusive lower bound: with reverse=True Telethon bumps - offset_id by one before the first request and again per page (verified - against telethon 1.44.0's _MessagesIter), so passing the last handled id - resumes without replaying it. The explicit `msg.id <= after_id` guard makes - the sweep correct even if that internal detail ever changes. - """ - buf: list[Any] = [] - closed: set[int] = set() - prev = 0 - - def agen(since: int): - return client.iter_messages(entity, reverse=True, offset_id=since) - - async for msg in aiter_with_flood_retry(agen, start_id=after_id, - max_wait_s=max_flood_wait): - if msg.id <= after_id: - continue - if msg.id <= prev: - raise SweepOrderError( - f"sweep not strictly ascending: {msg.id} after {prev}. " - f"Album grouping and the cursor both assume ascending order.") - prev = msg.id - - gid = getattr(msg, "grouped_id", None) - if buf and (gid is None or gid != buf[0].grouped_id): - yield _close(buf, closed, filters) - buf = [] - if gid is None: - yield _close([msg], closed, filters) - else: - buf.append(msg) - if len(buf) > MAX_ALBUM_BUFFER: - raise SweepOrderError( - f"album buffer exceeded {MAX_ALBUM_BUFFER} for grouped_id " - f"{gid} - grouping logic is wrong, refusing to keep buffering") - if buf: - yield _close(buf, closed, filters) - - -def _close(buf: list[Any], closed: set[int], filters: Filters) -> Post: - """Turn a finished run of messages into a Post. - - Invariant 1 - identity is filter-invariant. post_id and max_message_id come - from the complete grouped_id group, before filters drop any member, so - `--types photo` and `--types video` address the same folder. Phase 4's - message-id filenames close the same loop for files. - """ - gid = getattr(buf[0], "grouped_id", None) - if gid is not None: - if gid in closed: - raise AlbumSplitError( - f"grouped_id {gid} reopened at msg {buf[0].id} after being closed - " - f"album split across a non-adjacent boundary. Do not trust this export.") - closed.add(gid) - return Post(post_id=buf[0].id, - messages=[m for m in buf if keep(m, filters)], - max_message_id=buf[-1].id) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/support.py b/tests/support.py deleted file mode 100644 index 6763d2a..0000000 --- a/tests/support.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Synthetic message stubs and async helpers, so every test runs offline. - -media.py is duck-typed precisely so these stubs can exist: a test message needs -only the attributes its case is about. -""" - -from __future__ import annotations - -import asyncio -from dataclasses import dataclass -from datetime import UTC, datetime - -_DEFAULT_EXT = {"photo": ".jpg", "video": ".mp4", "audio": ".mp3", - "voice": ".ogg", "document": ".bin"} -_MIME = {"photo": "image/jpeg", "video": "video/mp4", "audio": "audio/mpeg", - "voice": "audio/ogg", "document": "application/octet-stream"} - - -@dataclass -class FakeFile: - name: str | None = None - ext: str | None = None - size: int | None = None - mime_type: str | None = None - - -class FakeMsg: - """A Telethon Message as far as this codebase is concerned.""" - - def __init__(self, id: int, *, kind: str | None = None, - grouped_id: int | None = None, date: datetime | None = None, - name: str | None = None, ext: str | None = None, - size: int | None = 1000, text: str | None = None, - web_preview: object | None = None, sender_id: int | None = None, - sender: object | None = None) -> None: - self.id = id - self.grouped_id = grouped_id - self.date = date or datetime(2026, 1, 1, tzinfo=UTC) - self.message = text - self.sender_id = sender_id - self.sender = sender - self.reply_to = None - self.web_preview = web_preview - # Every media property media_kind() consults, default absent. - self.photo = self.video = self.document = None - self.audio = self.voice = self.gif = self.video_note = None - self.file = None - if kind is not None: - setattr(self, kind, object()) # the property media_kind reads - self.file = FakeFile(name=name, - ext=ext or _DEFAULT_EXT.get(kind, ".bin"), - size=size, - mime_type=_MIME.get(kind)) - - -class FakeClient: - """Serves a fixed ascending message list, honoring offset_id exclusively - - the semantics verified against telethon 1.44.0's _MessagesIter.""" - - def __init__(self, messages: list[FakeMsg]) -> None: - self.messages = messages - self.calls: list[int] = [] - - def iter_messages(self, entity, *, reverse=False, offset_id=0, **kwargs): - assert reverse is True, "the sweep must always be chronological" - self.calls.append(offset_id) - - async def gen(): - for m in self.messages: - if m.id > offset_id: - yield m - - return gen() - - -def collect(agen) -> list: - """Drain an async generator from a synchronous test.""" - - async def drain(): - return [item async for item in agen] - - return asyncio.run(drain()) - - -def run(coro): - return asyncio.run(coro) - - -async def as_agen(items): - for item in items: - yield item diff --git a/tests/test_cli_dispatch.py b/tests/test_cli_dispatch.py deleted file mode 100644 index 024735e..0000000 --- a/tests/test_cli_dispatch.py +++ /dev/null @@ -1,239 +0,0 @@ -"""End-to-end dispatch through cli.main() against a fake Telegram client. - -This file exists because of a specific escape: `Config.max_flood_wait` was -removed from the dataclass while `cli._real_run` kept passing it, so every real -export died with an uncaught TypeError after taking both locks and writing the -state file - and 180 tests still passed, because every one of them built Config -by hand and none of them ever called main(). - -Anything that only the wiring can break belongs here. -""" - -from __future__ import annotations - -import json -from pathlib import Path - -import pytest - -from telegram_exporter import cli, session as session_mod -from telegram_exporter.state import STATE_NAME -from tests.support import FakeMsg - - -class FakeClient: - """Enough of a TelegramClient for a whole run, with no network.""" - - def __init__(self, messages, entity): - self.messages = messages - self.entity = entity - self.downloads: list[int] = [] - self.flood_sleep_threshold = None - self.disconnected = False - - def iter_messages(self, entity, *, reverse=False, offset_id=0, **kwargs): - async def gen(): - for m in self.messages: - if m.id > offset_id: - yield m - return gen() - - async def download_media(self, msg, file=None): - self.downloads.append(msg.id) - file.write(b"x" * (msg.file.size or 10)) - return file - - async def get_messages(self, entity, ids=None): - return next((m for m in self.messages if m.id == ids), None) - - async def connect(self): - return None - - async def is_user_authorized(self): - return True # a second run is non-interactive - - async def disconnect(self): - self.disconnected = True - - -class FakeEntity: - id, title, username = 1234567890, "My Group", "mygroup" - - -@pytest.fixture -def wired(monkeypatch, tmp_path): - """Patch out only the network boundary: credentials, client, entity.""" - monkeypatch.setenv("TG_API_ID", "12345") - monkeypatch.setenv("TG_API_HASH", "0" * 32) - monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) - - messages = [FakeMsg(10, kind="photo", grouped_id=7, size=100), - FakeMsg(11, kind="photo", grouped_id=7, size=100), - FakeMsg(12, text="just talking"), - FakeMsg(13, kind="video", size=500)] - client = FakeClient(messages, FakeEntity()) - - class FakeClientFactory: - def __init__(self, *a, **kw): - pass - - def fake_telegram_client(session, api_id, api_hash): - return client - - monkeypatch.setattr(session_mod, "TelegramClient", fake_telegram_client) - - async def fake_resolve(_client, _spec): - return client.entity, -1001234567890 - - monkeypatch.setattr(cli, "resolve_entity", fake_resolve) - return client - - -def run_cli(monkeypatch, *argv) -> int: - monkeypatch.setattr("sys.argv", ["tg-export", *argv]) - return cli.main() - - -# --------------------------------------------------------------------------- # -# The real run - the path C1 broke -# --------------------------------------------------------------------------- # - -def test_a_real_export_runs_end_to_end(wired, monkeypatch, tmp_path): - out = tmp_path / "exports" - code = run_cli(monkeypatch, "--group", "@mygroup", "--out", str(out)) - - assert code == 0 - root = out / "g-1001234567890" - assert sorted(p.name for p in (root / "10").iterdir()) == \ - ["10_photo.jpg", "11_photo.jpg"] - assert (root / "13" / "13_video.mp4").exists() - assert (root / "title.txt").read_text().strip() == "My Group" - assert json.loads((root / STATE_NAME).read_text())["completed_at"] is not None - assert wired.disconnected, "disconnect must run even on the success path" - - -def test_a_real_export_honors_the_flood_ceiling_flag(wired, monkeypatch, tmp_path): - # The ceiling reaches the network through Fetcher and iter_posts, not - # through Config. Passing the flag must not blow up the wiring. - code = run_cli(monkeypatch, "--group", "@mygroup", - "--out", str(tmp_path / "e"), "--max-flood-wait", "3600") - assert code == 0 - - -def test_a_second_run_downloads_nothing(wired, monkeypatch, tmp_path): - out = str(tmp_path / "exports") - assert run_cli(monkeypatch, "--group", "@mygroup", "--out", out) == 0 - before = len(wired.downloads) - assert run_cli(monkeypatch, "--group", "@mygroup", "--out", out) == 0 - assert len(wired.downloads) == before, "resume re-downloaded completed files" - - -def test_limit_zero_is_a_non_interactive_connectivity_check(wired, monkeypatch, - tmp_path): - code = run_cli(monkeypatch, "--group", "@mygroup", - "--out", str(tmp_path / "e"), "--limit", "0") - assert code == 0 - assert wired.downloads == [] - - -# --------------------------------------------------------------------------- # -# Dry run -# --------------------------------------------------------------------------- # - -def test_dry_run_writes_nothing_and_downloads_nothing(wired, monkeypatch, - tmp_path, capsys): - out = tmp_path / "exports" - code = run_cli(monkeypatch, "--group", "@mygroup", "--out", str(out), - "--dry-run", "--min-free", "0") - - assert code == 0 - assert wired.downloads == [] - assert not out.exists() - assert "Remaining:" in capsys.readouterr().out - - -def test_dry_run_exits_two_when_it_will_not_fit(wired, monkeypatch, tmp_path): - code = run_cli(monkeypatch, "--group", "@mygroup", - "--out", str(tmp_path / "e"), "--dry-run", - "--min-free", "1000GiB") - assert code == 2 - - -# --------------------------------------------------------------------------- # -# Exit-code contract through the real dispatcher -# --------------------------------------------------------------------------- # - -def test_a_filter_change_exits_seven_and_names_the_change(wired, monkeypatch, - tmp_path, caplog): - out = str(tmp_path / "exports") - assert run_cli(monkeypatch, "--group", "@mygroup", "--out", out, - "--types", "photo") == 0 - code = run_cli(monkeypatch, "--group", "@mygroup", "--out", out, - "--types", "video") - - assert code == 7 - assert "filter mismatch" in caplog.text and "types" in caplog.text - - -def test_a_token_less_types_value_never_reaches_a_run(wired, monkeypatch, tmp_path): - code = run_cli(monkeypatch, "--group", "@mygroup", - "--out", str(tmp_path / "e"), "--types", " ") - assert code == 1 - assert wired.downloads == [] - - -def test_missing_credentials_exit_one(monkeypatch, tmp_path): - monkeypatch.delenv("TG_API_ID", raising=False) - monkeypatch.delenv("TG_API_HASH", raising=False) - assert run_cli(monkeypatch, "--group", "@g", "--out", str(tmp_path)) == 1 - - -def test_an_unexpected_exception_still_honors_the_exit_contract(wired, monkeypatch, - tmp_path): - async def boom(_client, _spec): - raise RuntimeError("something nobody predicted") - - monkeypatch.setattr(cli, "resolve_entity", boom) - code = run_cli(monkeypatch, "--group", "@mygroup", "--out", str(tmp_path / "e")) - assert code == 1 # not a bare traceback with no code - - -def test_a_second_concurrent_run_exits_three(wired, monkeypatch, tmp_path): - out = tmp_path / "exports" - root = out / "g-1001234567890" - root.mkdir(parents=True) - with cli.exclusive(root / cli.LOCK_NAME): - code = run_cli(monkeypatch, "--group", "@mygroup", "--out", str(out)) - assert code == 3 - - -def test_a_session_lock_conflict_is_refused_before_the_session_is_opened( - wired, monkeypatch, tmp_path): - # The session lock was taken after connect() and _login() had already written - # the shared SQLite session, so the loser reported the conflict having caused - # it. The session file must not exist at all when the lock is contended. - session = tmp_path / "state" / "tg-export" / "default.session" - session.parent.mkdir(parents=True, exist_ok=True) - - with cli.exclusive(session.with_name(session.name + ".lock")): - code = run_cli(monkeypatch, "--group", "@mygroup", - "--out", str(tmp_path / "exports")) - - assert code == 3 - assert not session.exists() - assert wired.downloads == [] - - -def test_dry_run_contends_for_the_session_lock_too(wired, monkeypatch, tmp_path): - # --dry-run writes nothing to the export tree, but it opens the same session - # file - which is the resource this lock is about - so it needs its own - # --session to run alongside a real export. - session = tmp_path / "state" / "tg-export" / "default.session" - session.parent.mkdir(parents=True, exist_ok=True) - - with cli.exclusive(session.with_name(session.name + ".lock")): - code = run_cli(monkeypatch, "--group", "@mygroup", "--dry-run", - "--out", str(tmp_path / "exports")) - - assert code == 3 - assert not session.exists() diff --git a/tests/test_download_decisions.py b/tests/test_download_decisions.py deleted file mode 100644 index 326acb1..0000000 --- a/tests/test_download_decisions.py +++ /dev/null @@ -1,624 +0,0 @@ -"""Per-file download decisions and the sink loop, against a tmpdir and a fake -fetcher. No network. - -The cases here are the ones where "looks like success" and "is success" can -diverge: dedupe, partial files, the disk guard, and Invariant 2. -""" - -from __future__ import annotations - -import errno -import json -import os -import stat - -import pytest -from telethon.errors import ( - AuthKeyUnregisteredError, - ChannelPrivateError, - FileReferenceExpiredError, - MediaEmptyError, - ServerError, -) -from telethon.errors.common import ( - AuthKeyNotFound, - CdnFileTamperedError, - InvalidBufferError, -) - -from telegram_exporter import downloader, paths -from telegram_exporter.downloader import ( - DOWNLOADED, - FAILED, - SKIPPED, - Config, - download_one, - run_download, - sweep_part_files, - TITLE_NAME, - write_title, -) -from telegram_exporter.session import Abort -from telegram_exporter.sidecar import Sidecar -from telegram_exporter.state import State -from telegram_exporter.traversal import Post, build_filters -from tests.support import FakeMsg, as_agen, run - -PAYLOAD = b"x" * 1000 -KIB = 1024 - - -@pytest.fixture(autouse=True) -def no_real_backoff(monkeypatch): - """The retry ladder is 5/10 s in production; tests assert the ladder is - walked, not that it sleeps. Yields the real function for the one test that - is about the ladder's shape.""" - real = downloader.backoff - monkeypatch.setattr(downloader, "backoff", lambda attempt: 0) - return real - - -class FakeFetcher: - def __init__(self, payload: bytes = PAYLOAD, raises=None, refresh_to="same"): - self.payload = payload - self.raises = list(raises or []) # exceptions to raise, in order - self.refresh_to = refresh_to - self.calls = 0 - self.refresh_calls = 0 - - async def fetch(self, msg, fh) -> None: - self.calls += 1 - if self.raises: - raise self.raises.pop(0) - fh.write(self.payload) - - async def refresh(self, message_id): - self.refresh_calls += 1 - return FakeMsg(message_id, kind="photo") if self.refresh_to == "same" \ - else self.refresh_to - - -def cfg(tmp_path, *, min_free=0): - return Config(min_free=min_free, disk_path=tmp_path) - - -def post_dir_for(tmp_path, post_id=1042): - root = paths.export_root(tmp_path, -1001) - target = paths.post_dir(root, post_id) - target.mkdir(parents=True, exist_ok=True) - return root, target - - -# --------------------------------------------------------------------------- # -# The plan's decision table -# --------------------------------------------------------------------------- # - -def test_target_absent_downloads(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher() - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert result.path.read_bytes() == PAYLOAD - assert fetcher.calls == 1 - assert list(target_dir.glob("*.part")) == [] - - -def test_an_existing_non_empty_target_is_skipped_without_a_download(tmp_path): - # Existence, not size equality. fsync before os.replace means a file at the - # target path can only have arrived fully downloaded - and a size-equality - # gate would delete every completed photo on resume, because photo .size - # describes the variant Telethon picked. - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=999_999) - (target_dir / "1042_photo.jpg").write_bytes(b"already here") - fetcher = FakeFetcher() - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == SKIPPED - assert fetcher.calls == 0 - - -def test_a_zero_byte_target_is_unlinked_and_refetched(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - (target_dir / "1042_photo.jpg").touch() - fetcher = FakeFetcher() - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert fetcher.calls == 1 - assert result.path.read_bytes() == PAYLOAD - - -def test_stray_part_files_are_swept(tmp_path): - root, target_dir = post_dir_for(tmp_path) - (target_dir / "1042_photo.jpg.part").write_bytes(b"half") - (root / "9/9_video.mp4.part").parent.mkdir(parents=True) - (root / "9/9_video.mp4.part").write_bytes(b"half") - - assert sweep_part_files(root) == 2 - assert list(root.rglob("*.part")) == [] - - -def test_a_valid_target_wins_over_a_leftover_part_file(tmp_path): - root, target_dir = post_dir_for(tmp_path) - (target_dir / "1042_photo.jpg").write_bytes(PAYLOAD) - (target_dir / "1042_photo.jpg.part").write_bytes(b"half") - sweep_part_files(root) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - - result = run(download_one(FakeFetcher(), target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == SKIPPED - assert (target_dir / "1042_photo.jpg").read_bytes() == PAYLOAD - assert list(target_dir.glob("*.part")) == [] - - -def test_unknown_declared_size_is_accepted_by_the_validator(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="document", size=None) - - result = run(download_one(FakeFetcher(), target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert result.declared_size is None - - -def test_a_hostile_declared_filename_lands_inside_the_post_dir(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="document", name="../../../etc/passwd", - size=len(PAYLOAD)) - - result = run(download_one(FakeFetcher(), target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert result.path.parent == target_dir.resolve() - assert result.path.name == "1042_passwd.bin" - - -def test_the_disk_guard_aborts_before_writing_a_single_byte(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="video", size=10 * KIB) - fetcher = FakeFetcher() - - with pytest.raises(Abort) as excinfo: - run(download_one(fetcher, target_dir, msg, - cfg=cfg(tmp_path, min_free=1 << 62))) - - assert excinfo.value.code == 3 - assert fetcher.calls == 0 - assert list(target_dir.iterdir()) == [] - - -# --------------------------------------------------------------------------- # -# Validator and error taxonomy -# --------------------------------------------------------------------------- # - -def test_a_photo_size_mismatch_is_accepted_as_a_variant(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD) + 431) - - result = run(download_one(FakeFetcher(), target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert result.declared_size != result.size # recorded, not hidden - - -def test_a_document_size_mismatch_retries_then_fails_without_raising(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="document", size=len(PAYLOAD) + 1) - fetcher = FakeFetcher() - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == FAILED - assert fetcher.calls == downloader.MAX_ATTEMPTS - assert list(target_dir.glob("*.part")) == [] - - -def test_an_expired_file_reference_is_refreshed_once_then_succeeds(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(raises=[FileReferenceExpiredError(request=None)]) - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert fetcher.refresh_calls == 1 - - -def test_a_deleted_message_is_recorded_and_skipped(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(raises=[FileReferenceExpiredError(request=None)], - refresh_to=None) - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == FAILED - assert result.error == "message deleted" - - -def test_unavailable_media_is_recorded_and_skipped(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(raises=[MediaEmptyError(request=None)]) - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == FAILED - assert "unavailable" in result.error - - -def test_a_transient_network_error_retries_and_then_succeeds(tmp_path): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - # ConnectionResetError with errno=None is the case that would fall through - # the errno test into "unexpected OS error" if OSError were caught first. - fetcher = FakeFetcher(raises=[ConnectionResetError()]) - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert fetcher.calls == 2 - - -def test_a_zero_byte_download_is_never_accepted(tmp_path): - """download_media returns without writing when the media resolves to an empty - variant. The photo excuse in validate() would otherwise rename a 0-byte file - to the target, where the "zero bytes is junk" dedupe rule could never see it - again - a completed export does not revisit existing targets.""" - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(payload=b"") - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == FAILED - assert fetcher.calls == downloader.MAX_ATTEMPTS - assert not (target_dir / "1042_photo.jpg").exists() - assert list(target_dir.glob("*.part")) == [] - - -def test_an_unsafe_filename_skips_one_file_instead_of_ending_the_run(tmp_path): - # Defense in depth behind the extension sanitizer: one hostile message must - # never be able to halt an export permanently, which is what happened while - # the ValueError from safe_join escaped and the cursor could not advance. - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher() - - # Simulate a hole in the sanitizer rather than asserting one exists. - with pytest.MonkeyPatch.context() as mp: - mp.setattr(paths, "derive_filename", lambda m: "../escape.jpg") - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == FAILED - assert "unsafe filename" in result.error - assert fetcher.calls == 0 - assert not (target_dir.parent / "escape.jpg").exists() - - -@pytest.mark.parametrize("exc", [ - # A plain -500: RpcCallFailError is only one leaf of ServerError, so catching - # the leaf alone let the parent escape and end the run. - ServerError(request=None, message="INTERNAL_SERVER_ERROR"), - InvalidBufferError(payload=b"\x00\x00\x00\x00"), # corrupt packet on a flaky link -]) -def test_transport_level_errors_retry_rather_than_killing_the_run(tmp_path, exc): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(raises=[exc]) - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert fetcher.calls == 2 - - -def test_a_cdn_integrity_failure_is_named_rather_than_called_a_network_error( - tmp_path, caplog): - """CdnFileTamperedError subclasses SecurityError and is raised on the - download path. Folding it into the network clause reported a trust-boundary - event as three lines of "network error" and a bland FAILED.""" - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(raises=[CdnFileTamperedError()]) - - with caplog.at_level("ERROR"): - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == FAILED - assert result.error == "cdn integrity check failed" - assert fetcher.calls == 1 # not retried as if it were noise - assert "integrity check FAILED" in caplog.text - - -@pytest.mark.parametrize("exc,code", [ - (AuthKeyUnregisteredError(request=None), 4), - # The one that actually arrives: MTProtoSender sets it on every in-flight - # request when the connection drops, and it subclasses plain Exception - so - # it used to escape every handler and end the run with a traceback. - (AuthKeyNotFound(), 4), - (ChannelPrivateError(request=None), 5), - (OSError(errno.ENOSPC, "No space left on device"), 3), -]) -def test_terminal_errors_abort_with_their_own_exit_code(tmp_path, exc, code): - _, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - - with pytest.raises(Abort) as excinfo: - run(download_one(FakeFetcher(raises=[exc]), target_dir, msg, - cfg=cfg(tmp_path))) - - assert excinfo.value.code == code - assert list(target_dir.glob("*.part")) == [] - - -# --------------------------------------------------------------------------- # -# The sink loop - Invariant 2 -# --------------------------------------------------------------------------- # - -def drive(tmp_path, posts, *, limit=None, fetcher=None, filters=None): - root = paths.export_root(tmp_path, -1001) - root.mkdir(parents=True, exist_ok=True) - state = State.open(root, chat_id=-1001, chat_title="G", - filters=filters or build_filters()) - with Sidecar(root) as sidecar: - totals = run(run_download(as_agen(posts), fetcher=fetcher or FakeFetcher(), - state=state, sidecar=sidecar, root=root, - cfg=cfg(tmp_path), limit=limit)) - records = [json.loads(line) - for line in (root / "messages.jsonl").read_text().splitlines()] - return root, state, totals, records - - -def album(post_id, ids, *, kind="photo"): - messages = [FakeMsg(i, kind=kind, grouped_id=7, size=len(PAYLOAD)) for i in ids] - return Post(post_id=post_id, messages=messages, max_message_id=max(ids)) - - -def test_a_completed_post_commits_the_cursor_and_writes_the_sidecar(tmp_path): - root, state, totals, records = drive(tmp_path, [album(1042, [1042, 1043])]) - - assert state.cursor_id == 1043 - assert totals.downloaded == 2 - assert totals.posts == 1 - assert [r["message_id"] for r in records] == [1042, 1043] - assert records[0]["files"][0]["path"] == "1042/1042_photo.jpg" - assert (root / "1042" / "1042_photo.jpg").exists() - - -def test_the_cursor_is_never_committed_from_an_abort_path(tmp_path): - # Invariant 2: state.commit is unreachable from every error exit, so the - # cursor can never point past an in-flight post. This is the failure that - # would look exactly like success. - posts = [album(1042, [1042]), album(2000, [2000])] - root = paths.export_root(tmp_path, -1001) - root.mkdir(parents=True, exist_ok=True) - state = State.open(root, chat_id=-1001, chat_title="G", filters=build_filters()) - - class FailSecond(FakeFetcher): - async def fetch(self, msg, fh): - if msg.id == 2000: - raise ChannelPrivateError(request=None) - await super().fetch(msg, fh) - - with Sidecar(root) as sidecar: - with pytest.raises(Abort) as excinfo: - run(run_download(as_agen(posts), fetcher=FailSecond(), state=state, - sidecar=sidecar, root=root, cfg=cfg(tmp_path))) - - assert excinfo.value.code == 5 - assert state.cursor_id == 1042 # the first post, not the second - assert State.read(root)["completed_at"] is None - - -def test_a_second_run_re_downloads_nothing(tmp_path): - posts = [album(1042, [1042, 1043])] - drive(tmp_path, posts) - fetcher = FakeFetcher() - drive(tmp_path, posts, fetcher=fetcher) - - assert fetcher.calls == 0 - - -def test_empty_and_text_only_posts_advance_the_cursor_without_counting(tmp_path): - posts = [Post(post_id=10, messages=[], max_message_id=11), - Post(post_id=12, messages=[FakeMsg(12, text="hi")], max_message_id=12)] - root, state, totals, records = drive( - tmp_path, posts, filters=build_filters(include_text=True)) - - assert state.cursor_id == 12 - assert totals.posts == 0 and totals.downloaded == 0 - assert [r["message_id"] for r in records] == [12] - assert records[0]["files"] == [] # context, not media - assert not (root / "12").exists() # no folder for a text message - - -def test_limit_counts_posts_with_files(tmp_path): - posts = [Post(post_id=10, messages=[], max_message_id=10), - album(11, [11]), album(12, [12]), album(13, [13])] - _, state, totals, _ = drive(tmp_path, posts, limit=2) - - assert totals.posts == 2 - assert state.cursor_id == 12 # stopped after the second, not the third - - -def test_completed_at_is_set_when_the_sweep_is_exhausted(tmp_path): - root, _, _, _ = drive(tmp_path, [album(1042, [1042])]) - assert State.read(root)["completed_at"] is not None - - -def test_a_run_where_everything_failed_is_never_marked_complete(tmp_path, caplog): - """A dead session looks exactly like this from inside the loop: ConnectionError - is retryable, so the sweep walks the whole history failing every file and - returns normally. Setting completed_at there would answer "did my export - finish?" with a confident yes over an empty tree.""" - class AlwaysFails(FakeFetcher): - async def fetch(self, msg, fh): - self.calls += 1 - raise ConnectionError("Cannot send requests while disconnected") - - with caplog.at_level("ERROR"): - root, state, totals, _ = drive( - tmp_path, [album(1, [1]), album(2, [2])], fetcher=AlwaysFails()) - - assert totals.failed == 2 and totals.downloaded == 0 - assert State.read(root)["completed_at"] is None - assert "not marking this export complete" in caplog.text - # The cursor still stands: those posts were handled, and the failures are - # recorded in the sidecar. - assert state.cursor_id == 2 - - -def test_a_partly_failed_run_is_still_complete(tmp_path): - class FailsOne(FakeFetcher): - async def fetch(self, msg, fh): - self.calls += 1 - if msg.id == 2: - raise MediaEmptyError(request=None) - fh.write(self.payload) - - root, _, totals, _ = drive(tmp_path, [album(1, [1]), album(2, [2])], - fetcher=FailsOne()) - - assert totals.downloaded == 1 and totals.failed == 1 - assert State.read(root)["completed_at"] is not None - - -def test_the_retry_ladder_does_not_sleep_after_the_final_attempt(no_real_backoff): - # 20 s per exhausted file bought nothing; on a dead connection it was the - # difference between hours and days. - backoff = no_real_backoff - assert backoff(1) == 5.0 - assert backoff(2) == 10.0 - assert backoff(downloader.MAX_ATTEMPTS) == 0.0 - - -def test_the_whole_export_tree_is_private_under_the_process_umask(tmp_path): - """plan.md's "export tree 0700, every file 0600" rested on umask alone, with - nothing observing the result. The sidecar carries other people's message text - and names, so this is the criterion worth asserting rather than assuming.""" - old = os.umask(0o077) - try: - root, _, _, _ = drive(tmp_path, [album(1042, [1042, 1043])]) - write_title(root, "My Group") # cli writes this; drive() does not - finally: - os.umask(old) - - checked = [root, root / "1042", root / "1042" / "1042_photo.jpg", - root / "messages.jsonl", root / ".export-state.json", - root / "title.txt"] - for path in checked: - assert path.exists(), path - assert stat.S_IMODE(path.stat().st_mode) & 0o077 == 0, \ - f"{path} is group/world accessible: {oct(path.stat().st_mode)}" - - -def test_limit_zero_connects_and_does_nothing(tmp_path): - root, state, totals, _ = drive(tmp_path, [album(1042, [1042])], limit=0) - assert totals.posts == 0 - assert state.cursor_id == 0 - assert not (root / "1042").exists() - - -def test_a_reference_renewed_on_the_final_attempt_is_actually_fetched(tmp_path): - # Renewing a file reference is not a failed attempt. Counting it consumed - # the last slot, so expiry landing on the final attempt reported "exhausted - # 3 attempts" for a message that was only ever tried twice - and the docstring - # on that arm says a 20-hour run *will* reach it. - root, target_dir = post_dir_for(tmp_path) - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher(raises=[ConnectionResetError(), ConnectionResetError(), - FileReferenceExpiredError(request=None)]) - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == DOWNLOADED - assert fetcher.refresh_calls == 1 - assert (target_dir / "1042_photo.jpg").read_bytes() == PAYLOAD - - -def test_an_already_present_file_reports_its_size_without_restatting(tmp_path): - root, target_dir = post_dir_for(tmp_path) - (target_dir / "1042_photo.jpg").write_bytes(b"abcdef") - msg = FakeMsg(1042, kind="photo", size=len(PAYLOAD)) - fetcher = FakeFetcher() - - result = run(download_one(fetcher, target_dir, msg, cfg=cfg(tmp_path))) - - assert result.status == SKIPPED - assert result.size == 6 - assert fetcher.calls == 0 - - -def test_a_post_whose_files_are_all_present_does_not_fsync_its_directory( - tmp_path, monkeypatch): - # The post directory is fsynced to make a rename durable. When every file was - # already on disk nothing was renamed, so a resume across a mostly-complete - # export should not pay one fsync per post for a directory entry an earlier - # run already made durable. - calls = [] - monkeypatch.setattr(downloader, "fsync_dir", lambda p: calls.append(p)) - root = paths.export_root(tmp_path, -1001) - for message_id in (1042, 1043): - path = paths.post_dir(root, 1042) / f"{message_id}_photo.jpg" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(PAYLOAD) - - _, _, totals, _ = drive(tmp_path, [album(1042, [1042, 1043])]) - - assert totals.skipped == 2 and totals.downloaded == 0 - assert calls == [] - - -def test_a_post_with_one_new_file_still_fsyncs_its_directory(tmp_path, monkeypatch): - calls = [] - monkeypatch.setattr(downloader, "fsync_dir", lambda p: calls.append(p)) - - _, _, totals, _ = drive(tmp_path, [album(1042, [1042, 1043])]) - - assert totals.downloaded == 2 - assert calls == [paths.post_dir(paths.export_root(tmp_path, -1001), 1042)] - - -def test_a_resume_where_failures_outnumber_successes_is_not_marked_complete(tmp_path): - # One already-present file used to disable this guard outright: it required - # that *nothing* had succeeded, so any resume across a partly-complete export - # could fail every remaining file and still record the export as finished. - class MediaDcIsDead(FakeFetcher): - async def fetch(self, msg, fh): - self.calls += 1 - raise ConnectionError("Cannot send requests while disconnected") - - root = paths.export_root(tmp_path, -1001) - present = paths.post_dir(root, 1) / "1_photo.jpg" - present.parent.mkdir(parents=True, exist_ok=True) - present.write_bytes(PAYLOAD) - - root, state, totals, _ = drive( - tmp_path, [album(1, [1]), album(2, [2]), album(3, [3])], - fetcher=MediaDcIsDead()) - - assert totals.skipped == 1 and totals.failed == 2 and totals.downloaded == 0 - assert State.read(root)["completed_at"] is None - assert state.cursor_id == 3 # the cursor still stands - - -def test_the_group_title_is_written_without_terminal_escapes(tmp_path): - # Server-supplied and admin-editable. The export root is safe because the - # title never becomes a path component, but `cat title.txt` would still have - # handed ANSI escapes and a right-to-left override to the operator. - root = paths.export_root(tmp_path, -1001) - write_title(root, "My \x1b[31mGroup\x1b[0m ‮gpj.exe") - - text = (root / "title.txt").read_text() - assert "\x1b" not in text and "‮" not in text - assert "Group" in text and "gpj.exe" in text - assert not (root / (TITLE_NAME + ".part")).exists() # atomic, nothing left over diff --git a/tests/test_estimate.py b/tests/test_estimate.py deleted file mode 100644 index f8ae7af..0000000 --- a/tests/test_estimate.py +++ /dev/null @@ -1,180 +0,0 @@ -"""The dry-run estimator. - -The property under test is anti-drift: the estimate has to describe exactly what -a real run from the same start would fetch, because that number is the one users -make disk decisions on. -""" - -from __future__ import annotations - -import pytest - -from telegram_exporter import paths -from telegram_exporter.downloader import run_download -from telegram_exporter.estimate import EstimateSink, describe_filters, resume_from, run_estimate -from telegram_exporter.sidecar import Sidecar -from telegram_exporter.state import STATE_NAME, State -from telegram_exporter.traversal import Post, build_filters -from tests.support import FakeMsg, as_agen, run -from tests.test_download_decisions import PAYLOAD, FakeFetcher, cfg - -GIB = 1 << 30 - - -def album(post_id, ids, *, kind="photo", size=len(PAYLOAD)): - return Post(post_id=post_id, - messages=[FakeMsg(i, kind=kind, grouped_id=7, size=size) - for i in ids], - max_message_id=max(ids)) - - -def estimate(tmp_path, posts, *, min_free=0, limit=None, filters=None): - root = paths.export_root(tmp_path, -1001) - sink = EstimateSink() - filters = filters or build_filters() - code = run(run_estimate(as_agen(posts), sink=sink, root=root, title="My Group", - peer_id=-1001, filters=filters, - start_id=resume_from(root, filters), - min_free=min_free, limit=limit)) - return sink, code, root - - -# --------------------------------------------------------------------------- # -# Counting -# --------------------------------------------------------------------------- # - -def test_counts_are_broken_down_by_kind(tmp_path): - posts = [album(1, [1, 2], kind="photo", size=100), - album(3, [3], kind="video", size=5000)] - sink, code, _ = estimate(tmp_path, posts) - - assert sink.posts == 2 and sink.files == 3 - assert sink.by_kind == {"photo": [2, 200], "video": [1, 5000]} - assert sink.total_bytes == 5200 - assert code == 0 - - -def test_unknown_size_files_get_their_own_tally_never_a_zero(tmp_path): - # An unbounded estimate error must be visible, so these are counted on their - # own line rather than coerced into the byte total. - posts = [album(1, [1], kind="document", size=None)] - sink, _, _ = estimate(tmp_path, posts) - - assert sink.unknown_size_files == 1 - assert sink.files == 1 - assert sink.total_bytes == 0 - - -def test_empty_and_text_only_posts_are_not_counted(tmp_path): - posts = [Post(post_id=1, messages=[], max_message_id=2), - Post(post_id=3, messages=[FakeMsg(3, text="hi")], max_message_id=3), - album(4, [4])] - sink, _, _ = estimate(tmp_path, posts) - - assert sink.posts == 1 and sink.files == 1 - - -# --------------------------------------------------------------------------- # -# Verdict -# --------------------------------------------------------------------------- # - -def test_the_verdict_is_short_and_exit_two_when_it_will_not_fit(tmp_path, capsys): - _, code, _ = estimate(tmp_path, [album(1, [1])], min_free=1 << 62) - assert code == 2 - out = capsys.readouterr().out - assert "SHORT by" in out - # A SHORT verdict is advisory: the per-file check_disk is the enforcement, - # and the run stops partway with a resumable cursor rather than corrupting. - assert "advisory" in out - - -def test_the_verdict_fits_when_there_is_room(tmp_path, capsys): - _, code, _ = estimate(tmp_path, [album(1, [1])], min_free=0) - assert code == 0 - assert "FITS" in capsys.readouterr().out - - -def test_photo_totals_are_labelled_approximate(tmp_path, capsys): - estimate(tmp_path, [album(1, [1], kind="photo"), album(2, [2], kind="video")]) - out = capsys.readouterr().out - photo_line = next(line for line in out.splitlines() if "photos" in line) - video_line = next(line for line in out.splitlines() if "videos" in line) - assert "approximate" in photo_line - assert "approximate" not in video_line - - -# --------------------------------------------------------------------------- # -# The cursor fix, and no side effects -# --------------------------------------------------------------------------- # - -def test_dry_run_writes_nothing_at_all(tmp_path): - before = sorted(p.name for p in tmp_path.iterdir()) - _, _, root = estimate(tmp_path, [album(1, [1])]) - - assert not root.exists() # not even the export root - assert sorted(p.name for p in tmp_path.iterdir()) == before - - -def test_dry_run_starts_where_the_real_run_would(tmp_path): - # Without reading the cursor, `--dry-run && run` fails closed on every - # resumed export: a mostly-downloaded group would be re-measured in full, - # exit 2, and refuse work that fits comfortably. - root = paths.export_root(tmp_path, -1001) - root.mkdir(parents=True) - filters = build_filters(types="photo") - State.open(root, chat_id=-1001, chat_title="G", filters=filters).commit(1042) - - assert resume_from(root, filters) == 1042 - - -def test_a_filter_change_makes_dry_run_measure_from_zero(tmp_path): - # The real run would refuse (exit 7), so reusing its cursor would describe a - # run that cannot happen. - root = paths.export_root(tmp_path, -1001) - root.mkdir(parents=True) - State.open(root, chat_id=-1001, chat_title="G", - filters=build_filters(types="photo")).commit(1042) - - assert resume_from(root, build_filters(types="video")) == 0 - - -def test_resume_from_is_zero_with_no_prior_state(tmp_path): - assert resume_from(paths.export_root(tmp_path, -1001), build_filters()) == 0 - assert not (tmp_path / STATE_NAME).exists() - - -# --------------------------------------------------------------------------- # -# Anti-drift: the estimate and the real run describe the same work -# --------------------------------------------------------------------------- # - -@pytest.mark.parametrize("limit", [None, 2]) -def test_dry_run_and_real_run_agree_on_posts_and_files(tmp_path, limit): - posts = [album(1, [1, 2]), album(3, [3]), album(4, [4, 5, 6]), - Post(post_id=7, messages=[], max_message_id=7), album(8, [8])] - - sink, _, _ = estimate(tmp_path, posts, limit=limit) - - root = paths.export_root(tmp_path / "real", -1001) - root.mkdir(parents=True) - state = State.open(root, chat_id=-1001, chat_title="G", filters=build_filters()) - with Sidecar(root) as sidecar: - totals = run(run_download(as_agen(posts), fetcher=FakeFetcher(), state=state, - sidecar=sidecar, root=root, - cfg=cfg(tmp_path), limit=limit)) - - assert sink.posts == totals.posts - assert sink.files == totals.downloaded - # Bytes agree exactly here because the fake payload matches the declared - # size; against the live API only photo sizes are approximate. - assert sink.total_bytes == totals.bytes_downloaded - - -def test_filters_are_described_for_the_report_header(): - described = describe_filters(build_filters(types="video,photo", - since="2026-01-01", - max_size="100MB", - include_text=True)) - assert "types=photo,video" in described - assert "since=2026-01-01" in described - assert "max-size" in described and "include-text" in described - assert describe_filters(build_filters()) == "(none)" diff --git a/tests/test_lock.py b/tests/test_lock.py deleted file mode 100644 index 983f778..0000000 --- a/tests/test_lock.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Single-instance lock under real process contention. - -flock is tested with a second process because that is the only thing that -exercises it: two acquisitions inside one process would succeed on most -platforms and prove nothing. -""" - -from __future__ import annotations - -import os -import subprocess -import sys -from pathlib import Path - -import pytest - -from telegram_exporter.cli import exclusive -from telegram_exporter.session import Abort - -REPO_SRC = str(Path(__file__).resolve().parents[1] / "src") - -HOLDER = """ -import sys, time -from pathlib import Path -from telegram_exporter.cli import exclusive -with exclusive(Path(sys.argv[1])): - print("HELD", flush=True) - time.sleep(60) -""" - -CONTENDER = """ -import sys -from pathlib import Path -from telegram_exporter.cli import exclusive -from telegram_exporter.session import Abort -try: - with exclusive(Path(sys.argv[1])): - sys.exit(0) -except Abort as a: - print(a.reason, file=sys.stderr) - sys.exit(a.code) -""" - - -def _env(): - env = dict(os.environ) - env["PYTHONPATH"] = REPO_SRC + os.pathsep + env.get("PYTHONPATH", "") - return env - - -@pytest.fixture -def holder(tmp_path): - lock = tmp_path / ".export.lock" - proc = subprocess.Popen([sys.executable, "-c", HOLDER, str(lock)], - stdout=subprocess.PIPE, text=True, env=_env()) - assert proc.stdout.readline().strip() == "HELD", "holder failed to acquire" - try: - yield lock - finally: - proc.kill() - proc.wait(timeout=10) - - -def test_a_second_acquire_exits_three(holder): - result = subprocess.run([sys.executable, "-c", CONTENDER, str(holder)], - capture_output=True, text=True, env=_env(), timeout=30) - assert result.returncode == 3 - assert "busy" in result.stderr - - -def test_contention_names_the_holder(holder): - with pytest.raises(Abort) as excinfo: - with exclusive(holder): - pass - assert excinfo.value.code == 3 - for field in ("pid=", "host=", "started="): - assert field in excinfo.value.reason - - -def test_contention_touches_no_file(holder, tmp_path): - before = sorted(p.name for p in tmp_path.iterdir()) - with pytest.raises(Abort): - with exclusive(holder): - pass - assert sorted(p.name for p in tmp_path.iterdir()) == before - - -def test_the_lock_is_released_when_the_holder_dies(tmp_path): - lock = tmp_path / ".export.lock" - proc = subprocess.Popen([sys.executable, "-c", HOLDER, str(lock)], - stdout=subprocess.PIPE, text=True, env=_env()) - assert proc.stdout.readline().strip() == "HELD" - proc.kill() # kill -9: flock releases on process death, - proc.wait(timeout=10) # so there is no stale-lock cleanup to get wrong - with exclusive(lock): - pass - - -def test_different_export_roots_do_not_contend(holder, tmp_path): - # Exporting two different groups concurrently is legitimate, so the lock is - # per-root rather than global. - with exclusive(tmp_path / "other" / ".export.lock"): - pass diff --git a/tests/test_logging.py b/tests/test_logging.py deleted file mode 100644 index 99b5a92..0000000 --- a/tests/test_logging.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Logging policy and the credential surface. - -telethon's request logging carries api_id, phone_number and phone_code_hash, so -its floor is enforced by this test rather than by a checkbox. -""" - -from __future__ import annotations - -import logging -from pathlib import Path - -import pytest - -from telegram_exporter.cli import build_parser, configure_logging - -SRC = Path(__file__).resolve().parents[1] / "src" / "telegram_exporter" -SOURCES = sorted(SRC.glob("*.py")) - - -@pytest.mark.parametrize("verbose", [False, True]) -def test_the_telethon_logger_never_drops_below_warning(verbose): - configure_logging(verbose=verbose) - assert logging.getLogger("telethon").getEffectiveLevel() >= logging.WARNING - assert logging.getLogger("asyncio").getEffectiveLevel() >= logging.WARNING - - -def test_verbose_affects_only_our_own_logger(): - configure_logging(verbose=True) - assert logging.getLogger("telegram_exporter").level == logging.DEBUG - configure_logging(verbose=False) - assert logging.getLogger("telegram_exporter").level == logging.INFO - - -def test_there_is_no_flag_that_unmutes_telethon(): - options = [action.option_strings for action in build_parser()._actions] - flat = [opt for group in options for opt in group] - assert not any("telethon" in opt or "debug" in opt for opt in flat) - - -def test_the_2fa_password_is_reachable_only_through_getpass(): - """No flag, no env var, no file - it is needed once per session lifetime, and - prompting costs nothing.""" - joined = "\n".join(p.read_text() for p in SOURCES) - assert joined.count("getpass(") == 1 - for forbidden in ["TG_PASSWORD", "TG_2FA", "TG_PASS", "--password", "--2fa"]: - assert forbidden not in joined - - -def test_no_credential_is_read_from_anywhere_but_the_documented_sources(): - joined = "\n".join(p.read_text() for p in SOURCES) - env_reads = {"TG_API_ID", "TG_API_HASH", "TG_PHONE", "XDG_STATE_HOME"} - found = {token for token in env_reads if token in joined} - assert found == env_reads # and nothing else - # The app does not read .env; the shell already handles that. - assert "dotenv" not in joined - - -def test_no_credential_appears_in_the_captured_log_output(caplog): - configure_logging(verbose=True) - log = logging.getLogger("telegram_exporter.test") - with caplog.at_level(logging.DEBUG): - log.debug("group %r (id %s)", "My Group", -1001234567890) - assert "api_hash" not in caplog.text - assert "+1555" not in caplog.text diff --git a/tests/test_no_concurrency.py b/tests/test_no_concurrency.py deleted file mode 100644 index a4ac641..0000000 --- a/tests/test_no_concurrency.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Concurrency 1, enforced by an AST scan rather than by discipline. - -Telethon does not parallelize transfers, so parallelism buys nothing and costs -flood-wait escalation: waits climb from ~7 s to 4 h+ once Telegram decides an -account is abusive. The temptation arrives precisely when an export feels slow, -which is why this is a test and not a comment. - -There is deliberately no --workers flag for this test to have to police. -""" - -from __future__ import annotations - -import ast -from pathlib import Path - -import pytest - -SRC = Path(__file__).resolve().parents[1] / "src" / "telegram_exporter" - -FORBIDDEN = { - "gather", "create_task", "ensure_future", "as_completed", "wait_for_all", - "to_thread", "run_in_executor", "TaskGroup", - "ThreadPoolExecutor", "ProcessPoolExecutor", "Thread", "Process", "Pool", -} - - -def _called_names(tree: ast.AST): - for node in ast.walk(tree): - if isinstance(node, ast.Call): - func = node.func - if isinstance(func, ast.Attribute): - yield func.attr, node.lineno - elif isinstance(func, ast.Name): - yield func.id, node.lineno - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - yield alias.name, node.lineno - elif isinstance(node, ast.AsyncWith): - pass - - -@pytest.mark.parametrize("path", sorted(SRC.glob("*.py")), ids=lambda p: p.name) -def test_no_parallelism_primitive_is_used(path): - tree = ast.parse(path.read_text(), filename=str(path)) - hits = [(name, line) for name, line in _called_names(tree) if name in FORBIDDEN] - assert not hits, f"{path.name} uses a parallelism primitive: {hits}" - - -def test_the_scan_would_actually_catch_something(): - # A guard that cannot fail is worse than no guard: it makes passing the - # default. This is the same shape of mistake the album-adjacency test had. - tree = ast.parse("import asyncio\n" - "async def f():\n" - " await asyncio.gather(g(), h())\n") - assert any(name in FORBIDDEN for name, _ in _called_names(tree)) - - -def test_there_is_no_workers_flag(): - from telegram_exporter.cli import build_parser - flat = [opt for action in build_parser()._actions for opt in action.option_strings] - assert not any("worker" in opt or "concurren" in opt or "parallel" in opt - for opt in flat) diff --git a/tests/test_paths.py b/tests/test_paths.py deleted file mode 100644 index f71181a..0000000 --- a/tests/test_paths.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Path safety. Every row of the plan's test table, plus the invariant that no -sanitized name can escape its post directory.""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from telegram_exporter.paths import ( - MAX_NAME_BYTES, - derive_filename, - export_root, - post_dir, - safe_join, - sanitize, -) -from tests.support import FakeMsg - - -@pytest.mark.parametrize("raw,expected", [ - ("../../../etc/passwd", "passwd"), - ("a/b/c.jpg", "c.jpg"), - ("..", "FALLBACK"), - (".", "FALLBACK"), - ("", "FALLBACK"), - ("...", "FALLBACK"), - ("\x00evil.jpg", "evil.jpg"), - ("....jpg", "jpg"), - ("-rf.jpg", "rf.jpg"), - ("--force me.jpg", "force me.jpg"), - ("a\\b\\c.jpg", "c.jpg"), - ("we:ird*na?me.jpg", "we_ird_na_me.jpg"), - ("tab\tsep.jpg", "tabsep.jpg"), # control chars are removed, not spaced - ("sp aced.jpg", "sp aced.jpg"), # whitespace runs collapse - ("/etc/shadow", "shadow"), -]) -def test_sanitize_table(raw, expected): - assert sanitize(raw, fallback="FALLBACK") == expected - - -def test_truncation_is_byte_counted_and_keeps_the_extension(): - # 300 CJK characters is ~900 bytes: a character-counted truncation would - # produce a name ext4 rejects with OSError: File name too long. - name = "あ" * 300 + ".jpg" - out = sanitize(name, fallback="x") - assert len(out.encode()) <= MAX_NAME_BYTES - assert out.endswith(".jpg") - assert len(out) > 10 # not collapsed to the fallback - - -def test_truncation_of_a_pathological_extension_still_yields_a_usable_name(): - out = sanitize("stem." + "e" * 400, fallback="fb") - assert 0 < len(out.encode()) <= MAX_NAME_BYTES - - -def test_no_sanitized_name_escapes_its_post_dir(tmp_path): - hostile = ["../../../etc/passwd", "..", "/etc/shadow", "a/../../b", - "\x00../x", "....//..//etc/passwd"] - base = post_dir(export_root(tmp_path, -1001), 1042) - for raw in hostile: - joined = safe_join(base, sanitize(raw, fallback="fb")) - assert joined.is_relative_to(base.resolve()) - - -def test_safe_join_raises_rather_than_repairing(): - base = Path("/tmp/exports/g-1001/1042") - for escape in ["../x", "../../etc/passwd", "/etc/passwd", ".."]: - with pytest.raises(ValueError): - safe_join(base, escape) - - -def test_export_root_is_the_chat_id_never_the_title(tmp_path): - assert export_root(tmp_path, -1001234567890).name == "g-1001234567890" - - -@pytest.mark.parametrize("bad", ["..", "../../etc", "", "1042; rm -rf", None, 1.5, True]) -def test_int_only_path_components(tmp_path, bad): - # A group title cannot reach a path component: there is no code path from - # title to export_root, and anything non-int is refused by type. - with pytest.raises(TypeError): - export_root(tmp_path, bad) - with pytest.raises(TypeError): - post_dir(tmp_path, bad) - - -def test_derive_filename_is_keyed_on_message_id(): - msg = FakeMsg(1043, kind="photo") - assert derive_filename(msg) == "1043_photo.jpg" - - -def test_hostile_declared_filename_becomes_a_leaf(): - msg = FakeMsg(7, kind="document", name="../../../etc/passwd", ext=".bin") - assert derive_filename(msg) == "7_passwd.bin" - - -def test_two_media_in_one_post_get_distinct_names(): - a, b = FakeMsg(1042, kind="photo", grouped_id=9), FakeMsg(1043, kind="photo", - grouped_id=9) - assert derive_filename(a) != derive_filename(b) - - -def test_filename_is_identical_across_filter_sets(): - # derive_filename takes no filter argument, which is the point: the same - # message addresses the same file whether --types photo or --types video was - # used, so dedupe survives a filter change. - msg = FakeMsg(1044, kind="video", name="clip.mp4") - assert derive_filename(msg) == derive_filename(msg) == "1044_clip.mp4" - - -@pytest.mark.parametrize("name,expected", [ - ("holiday.\x00jpg", "7_holiday.jpg"), # NUL: used to escape as ValueError - # The ESC byte is removed; the remaining "[31m" is inert text, not an escape. - ("clip.\x1b[31mmp4", "7_clip.[31mmp4"), - ("doc.j:pg", "7_doc.j_pg"), # reserved character - ("shot.jp g", "7_shot.jp g"), - ("archive.tar.gz", "7_archive.tar.gz"), # a real double extension survives - ("weird.", "7_weird.bin"), # empty extension falls back -]) -def test_the_extension_is_sanitized_like_the_stem(name, expected): - """The extension comes from the same attacker-supplied filename attribute as - the stem. Sanitizing only the stem left a hole: a NUL in the extension made - Path.resolve raise inside safe_join, and that ValueError escaped the download - loop - so the cursor never advanced past the message and every later run died - on it.""" - assert derive_filename(FakeMsg(7, kind="document", name=name)) == expected - - -@pytest.mark.parametrize("raw,forbidden", [ - ("a‮gpj.exe", "‮"), # RIGHT-TO-LEFT OVERRIDE: renders a.exe.jpg - ("‮report.pdf", "‮"), - ("a​b.jpg", "​"), # zero-width space - ("a
b.jpg", "
"), # line separator: breaks `find` consumers - ("a…b.jpg", "…"), # C1 NEL -]) -def test_bidi_and_format_characters_cannot_spoof_an_extension(raw, forbidden): - """Stripping only C0 escapes closed the ANSI vector while leaving the older - extension-spoofing one open: makes a .exe render as a .jpg in any - terminal or file manager.""" - out = derive_filename(FakeMsg(12345, kind="document", name=raw)) - assert forbidden not in out - - -def test_no_control_character_survives_anywhere_in_a_filename(): - for raw in ["a\x00.b\x00c", "\x1b].jpg", "x.\ny", "n.\r\tz"]: - out = derive_filename(FakeMsg(3, kind="document", name=raw)) - assert not any(ord(ch) < 0x20 or ord(ch) == 0x7F for ch in out), out - - -def test_a_whole_filename_stays_inside_ext4s_255_byte_component_limit(): - # The stem is capped at 200 UTF-8 bytes, so the "{id}_" prefix and the - # extension have to be byte-counted too or the total overflows. - worst = FakeMsg(9_999_999_999_999_999, - kind="document", - name="あ" * 300 + "." + "ん" * 16) - assert len(derive_filename(worst).encode()) <= 255 - - -def test_missing_name_falls_back_to_the_media_kind(): - assert derive_filename(FakeMsg(5, kind="voice")) == "5_voice.ogg" - - -def test_fallbacks_never_include_a_timestamp_or_anything_nondeterministic(): - msg = FakeMsg(11, kind="document", name=" ", ext=".pdf") - assert derive_filename(msg) == derive_filename(msg) == "11_document.pdf" diff --git a/tests/test_session.py b/tests/test_session.py deleted file mode 100644 index aad270e..0000000 --- a/tests/test_session.py +++ /dev/null @@ -1,249 +0,0 @@ -"""Session file privacy, the git-safety refusal, and the flood-wait primitives. - -Phase 2's success criteria name all three; none of them belongs in the logging or -lock test files, and every one is a silent-failure mode if it regresses. -""" - -from __future__ import annotations - -import os -import re -import stat -import subprocess -from pathlib import Path - -import pytest -from telethon.errors import FloodWaitError - -from telegram_exporter import session as session_mod -from telegram_exporter.session import ( - Abort, - _flood_sleep, - aiter_with_flood_retry, - assert_not_committable, - assert_session_private, - default_session_path, - load_api_credentials, - prepare_session_path, - with_flood_retry, -) -from tests.support import FakeMsg, collect, run - -CLI_SRC = Path(__file__).resolve().parents[1] / "src" / "telegram_exporter" / "cli.py" - - -@pytest.fixture -def no_sleep(monkeypatch): - """Assert the wait *happens* without waiting for it.""" - slept = [] - - async def fake_sleep(seconds): - slept.append(seconds) - - monkeypatch.setattr(session_mod.asyncio, "sleep", fake_sleep) - return slept - - -# --------------------------------------------------------------------------- # -# Session file privacy -# --------------------------------------------------------------------------- # - -def test_the_session_suffix_is_added_so_the_right_file_is_protected(tmp_path): - # Telethon appends '.session' when absent, so chmod-ing the un-suffixed path - # passed to --session would protect nothing. - created = prepare_session_path(tmp_path / "mine") - assert created.name == "mine.session" - assert created.exists() - - -def test_a_new_session_file_is_created_private(tmp_path): - created = prepare_session_path(tmp_path / "s.session") - assert stat.S_IMODE(created.stat().st_mode) & 0o077 == 0 - - -def test_an_existing_loose_session_file_is_tightened(tmp_path): - loose = tmp_path / "s.session" - loose.touch() - os.chmod(loose, 0o644) - - prepare_session_path(loose) - - assert stat.S_IMODE(loose.stat().st_mode) == 0o600 - - -def test_every_sqlite_sibling_is_checked_not_just_the_session(tmp_path): - # -journal, -wal and -shm hold the same secrets as the database itself. - created = prepare_session_path(tmp_path / "s.session") - sibling = tmp_path / "s.session-journal" - sibling.touch() - os.chmod(sibling, 0o644) - - with pytest.raises(SystemExit, match="insecure mode"): - assert_session_private(created) - - -def test_a_private_session_passes_the_assertion(tmp_path): - created = prepare_session_path(tmp_path / "s.session") - (tmp_path / "s.session-journal").touch(mode=0o600) - assert_session_private(created) - - -def test_the_default_session_path_is_outside_any_repo(): - # The safe location is the default, so the git check only bites a - # deliberately chosen path. - assert default_session_path().is_absolute() - assert "tg-export" in str(default_session_path()) - - -def test_umask_is_the_first_statement_of_main(): - # It has to precede anything that can create a file: sqlite writes the auth - # key *during* login, so a post-hoc chmod is provably too late. - body = CLI_SRC.read_text().split("def main()")[1] - first = next(line.strip() for line in body.splitlines()[1:] if line.strip()) - assert first.startswith("os.umask(0o077)") - - -# --------------------------------------------------------------------------- # -# Git safety - robust to --out / --session pointing anywhere -# --------------------------------------------------------------------------- # - -def _git_repo(tmp_path: Path) -> Path: - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", "-q"], cwd=repo, check=True) - (repo / ".gitignore").write_text("ignored/\n*.session\n") - return repo - - -def test_a_committable_path_inside_a_repo_is_refused(tmp_path): - repo = _git_repo(tmp_path) - with pytest.raises(SystemExit, match="refusing"): - assert_not_committable(repo / "exports", is_dir=True) - - -def test_an_ignored_path_inside_a_repo_is_allowed(tmp_path): - repo = _git_repo(tmp_path) - (repo / "ignored").mkdir() - assert_not_committable(repo / "ignored") - assert_not_committable(repo / "creds.session") - - -def test_a_directory_pattern_matches_before_the_directory_exists(tmp_path): - # git check-ignore returns "not ignored" for `exports` when it cannot see - # that it is a directory, so without the trailing-slash retry the export root - # would be refused on the first run and accepted on every run after it. - repo = _git_repo(tmp_path) - assert not (repo / "ignored").exists() - assert_not_committable(repo / "ignored", is_dir=True) - assert_not_committable(repo / "ignored" / "g-1001", is_dir=True) - - -def test_the_directory_retry_never_excuses_a_file(tmp_path): - # The retry asks git to read the path as a directory, which would be unsound - # for a session file: a `.gitignore` line of `creds.session/` must not excuse - # a *file* named creds.session, which git would happily commit. - repo = tmp_path / "repo" - repo.mkdir() - subprocess.run(["git", "init", "-q"], cwd=repo, check=True) - (repo / ".gitignore").write_text("creds.session/\n") - - assert_not_committable(repo / "creds.session", is_dir=True) # a dir: ignored - with pytest.raises(SystemExit): - assert_not_committable(repo / "creds.session") # a file: refused - - -def test_a_path_outside_any_repo_is_allowed(tmp_path): - assert_not_committable(tmp_path / "anywhere" / "exports") - - -def test_a_nonexistent_path_is_still_checked(tmp_path): - # An export root does not exist on a first run, which is exactly when this - # check matters; running git from a nonexistent cwd would raise instead. - repo = _git_repo(tmp_path) - with pytest.raises(SystemExit): - assert_not_committable(repo / "deep" / "not" / "created" / "yet") - - -# --------------------------------------------------------------------------- # -# Credentials -# --------------------------------------------------------------------------- # - -def test_missing_credentials_produce_an_actionable_error(monkeypatch): - monkeypatch.delenv("TG_API_ID", raising=False) - monkeypatch.delenv("TG_API_HASH", raising=False) - with pytest.raises(Abort) as excinfo: - load_api_credentials() - assert "my.telegram.org" in excinfo.value.reason - - -def test_a_non_numeric_api_id_is_named(monkeypatch): - monkeypatch.setenv("TG_API_ID", "abc") - monkeypatch.setenv("TG_API_HASH", "hash") - with pytest.raises(Abort, match="must be an integer"): - load_api_credentials() - - -# --------------------------------------------------------------------------- # -# Flood-wait primitives -# --------------------------------------------------------------------------- # - -def test_a_flood_wait_is_slept_through_once_by_default(no_sleep): - calls = [] - - async def flaky(): - calls.append(1) - if len(calls) == 1: - raise FloodWaitError(request=None, capture=30) - return "ok" - - assert run(with_flood_retry(flaky)) == "ok" - assert len(calls) == 2 - assert no_sleep and no_sleep[0] >= 30 # the wait plus jitter, never a tight retry - - -def test_a_wait_beyond_the_explicit_ceiling_exits_six(no_sleep): - with pytest.raises(Abort) as excinfo: - run(_flood_sleep(FloodWaitError(request=None, capture=7200), 3600)) - assert excinfo.value.code == 6 - assert not no_sleep # aborted instead of sleeping - - -def test_with_no_ceiling_even_a_four_hour_wait_is_slept(no_sleep): - run(_flood_sleep(FloodWaitError(request=None, capture=4 * 3600), None)) - assert no_sleep[0] >= 4 * 3600 - - -def test_the_flood_log_names_the_computed_wake_time(no_sleep, caplog): - # The mitigation for the no-ceiling default: an unattended four-hour sleep - # has to read as a sleep rather than as a hang. - with caplog.at_level("WARNING"): - run(_flood_sleep(FloodWaitError(request=None, capture=90), None)) - assert "sleeping until" in caplog.text - assert re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", caplog.text) - - -def test_a_mid_sweep_flood_wait_resumes_from_the_last_yielded_id(no_sleep): - """The single-value wrapper cannot cover `async for`: Telethon raises on the - *next page fetch*, deep inside the loop. Rebuilding from the last yielded id - is what keeps the sweep gap-free.""" - messages = [FakeMsg(i, kind="photo") for i in (10, 11, 12, 13)] - attempts = [] - - def make_agen(since): - attempts.append(since) - - async def gen(): - for m in messages: - if m.id <= since: - continue - if m.id == 12 and len(attempts) == 1: - raise FloodWaitError(request=None, capture=5) - yield m - - return gen() - - got = collect(aiter_with_flood_retry(make_agen, start_id=0)) - - assert [m.id for m in got] == [10, 11, 12, 13] # no gap, no replay - assert attempts == [0, 11] # rebuilt from the last yielded - assert len(no_sleep) == 1 diff --git a/tests/test_sidecar.py b/tests/test_sidecar.py deleted file mode 100644 index b845e9d..0000000 --- a/tests/test_sidecar.py +++ /dev/null @@ -1,153 +0,0 @@ -"""messages.jsonl: partial-line repair, rotation, and union semantics. - -Not in the plan's original file list, but phase 5's success criteria name both -behaviours ("a truncated final line is repaired at startup", "--reset-state -rotates the sidecar"), and neither belongs in the download-decision table. -""" - -from __future__ import annotations - -import json - -from telegram_exporter.sidecar import SIDECAR_NAME, Sidecar -from telegram_exporter.traversal import Post -from tests.support import FakeMsg - - -def lines(root): - return (root / SIDECAR_NAME).read_text().splitlines() - - -def test_a_partial_trailing_line_is_truncated_at_startup(tmp_path): - # A host crash mid-write otherwise leaves a fragment that breaks every JSONL - # consumer downstream. - path = tmp_path / SIDECAR_NAME - path.write_text('{"message_id": 1}\n{"message_id": 2, "fi') - - with Sidecar(tmp_path): - pass - - assert lines(tmp_path) == ['{"message_id": 1}'] - - -def test_a_complete_final_line_without_a_newline_is_kept(tmp_path): - path = tmp_path / SIDECAR_NAME - path.write_text('{"message_id": 1}\n{"message_id": 2}') - - with Sidecar(tmp_path): - pass - - assert [json.loads(line)["message_id"] for line in lines(tmp_path)] == [1, 2] - - -def test_an_intact_file_is_left_alone(tmp_path): - path = tmp_path / SIDECAR_NAME - path.write_text('{"message_id": 1}\n') - - with Sidecar(tmp_path): - pass - - assert lines(tmp_path) == ['{"message_id": 1}'] - - -def test_an_empty_or_absent_file_is_not_a_repair_case(tmp_path): - with Sidecar(tmp_path): - pass - assert lines(tmp_path) == [] - - -def test_rotation_preserves_the_old_records_under_a_timestamp(tmp_path): - (tmp_path / SIDECAR_NAME).write_text('{"message_id": 1}\n') - sidecar = Sidecar(tmp_path) - - rotated = sidecar.rotate() - - assert rotated is not None and rotated.exists() - assert rotated.name.startswith("messages-") and rotated.suffix == ".jsonl" - assert not (tmp_path / SIDECAR_NAME).exists() - # Rotation, not deletion: the current sidecar describes exactly one filter - # regime while the previous one stays inspectable. - assert json.loads(rotated.read_text())["message_id"] == 1 - - -def test_rotating_a_missing_sidecar_is_a_no_op(tmp_path): - assert Sidecar(tmp_path).rotate() is None - - -def test_records_are_append_only_events_so_consumers_union_them(tmp_path): - """Two runs under different filters both record message 1043. Reading only - the last record would report a narrower file list than what is on disk.""" - msg_photo = FakeMsg(1043, kind="photo", grouped_id=8) - post = Post(post_id=1042, messages=[msg_photo], max_message_id=1043) - - for _ in range(2): - with Sidecar(tmp_path) as sidecar: - sidecar.append(post, []) - sidecar.fsync() - - records = [json.loads(line) for line in lines(tmp_path)] - assert [r["message_id"] for r in records] == [1043, 1043] - assert all(r["post_id"] == 1042 and r["grouped_id"] == 8 for r in records) - - -def test_a_text_only_message_gets_the_same_shape_with_no_files(tmp_path): - post = Post(post_id=12, messages=[FakeMsg(12, text="hello")], max_message_id=12) - with Sidecar(tmp_path) as sidecar: - sidecar.append(post, []) - sidecar.fsync() - - record = json.loads(lines(tmp_path)[0]) - assert record["files"] == [] and record["caption"] == "hello" - assert record["errors"] == [] - - -def test_sender_name_is_only_taken_from_an_already_cached_sender(tmp_path): - # Never an extra get_entity call: on a 100k-message sweep that is 100k extra - # RPCs and a flood ban, in exchange for a display string. - class Sender: - first_name, last_name, username, title = "Alice", None, "alice", None - - with_sender = FakeMsg(1, kind="photo", sender=Sender(), sender_id=777) - without = FakeMsg(2, kind="photo", sender=None, sender_id=778) - post = Post(post_id=1, messages=[with_sender, without], max_message_id=2) - - with Sidecar(tmp_path) as sidecar: - sidecar.append(post, []) - sidecar.fsync() - - records = [json.loads(line) for line in lines(tmp_path)] - assert records[0]["sender_name"] == "Alice" - assert records[1]["sender_name"] is None - assert records[1]["sender_id"] == 778 - - -def test_a_partial_record_larger_than_the_scan_window_keeps_earlier_records(tmp_path): - # Scanning back a fixed window for the last newline finds none when the - # partial record is bigger than the window. Truncating to `end - window` - # then cuts inside the partial record and leaves the file just as unreadable, - # one window shorter - so the window has to grow until a newline is found. - path = tmp_path / SIDECAR_NAME - good = '{"message_id": 1}' - partial = '{"message_id": 2, "caption": "' + "x" * (2 << 20) - path.write_text(good + "\n" + partial) - - with Sidecar(tmp_path): - pass - - assert lines(tmp_path) == [good] - - -def test_two_rotations_in_the_same_second_both_stay_inspectable(tmp_path): - # os.replace onto a one-second stamp silently overwrote the first archive, - # which is the whole reason --reset-state rotates instead of deleting. - path = tmp_path / SIDECAR_NAME - path.write_text('{"message_id": 1}\n') - first = Sidecar(tmp_path).rotate() - path.write_text('{"message_id": 2}\n') - second = Sidecar(tmp_path).rotate() - - assert first is not None and second is not None - assert first != second - assert first.exists() and second.exists() - assert json.loads(first.read_text())["message_id"] == 1 - assert json.loads(second.read_text())["message_id"] == 2 diff --git a/tests/test_state.py b/tests/test_state.py deleted file mode 100644 index 5247973..0000000 --- a/tests/test_state.py +++ /dev/null @@ -1,122 +0,0 @@ -"""The resume cursor: Invariant 3 (filters stored verbatim, mismatch refused with -a named diff) and atomic saves.""" - -from __future__ import annotations - -import json - -import pytest - -from telegram_exporter.session import Abort -from telegram_exporter.state import STATE_NAME, State -from telegram_exporter.traversal import build_filters - -CHAT = -1001234567890 - - -def open_state(root, *, filters=None, chat_id=CHAT, reset=False): - return State.open(root, chat_id=chat_id, chat_title="My Group", - filters=filters or build_filters(), reset=reset) - - -def test_a_new_state_file_starts_at_zero(tmp_path): - state = open_state(tmp_path) - assert state.cursor_id == 0 - assert (tmp_path / STATE_NAME).exists() - assert json.loads((tmp_path / STATE_NAME).read_text())["chat_id"] == CHAT - - -def test_commit_persists_and_leaves_no_temp_file(tmp_path): - state = open_state(tmp_path) - state.commit(1042) - assert json.loads((tmp_path / STATE_NAME).read_text())["cursor_id"] == 1042 - assert list(tmp_path.glob("*.tmp")) == [] - assert open_state(tmp_path).cursor_id == 1042 - - -def test_a_cursor_regression_is_ignored(tmp_path): - state = open_state(tmp_path) - state.commit(1042) - state.commit(7) - assert state.cursor_id == 1042 - - -def test_completed_at_is_set_only_on_clean_exhaustion(tmp_path): - state = open_state(tmp_path) - assert state.data["completed_at"] is None - state.mark_completed() - assert state.data["completed_at"] is not None - # A new run clears it: the field answers "did *this* export finish?". - assert open_state(tmp_path).data["completed_at"] is None - - -def test_a_filter_change_is_refused_and_names_what_changed(tmp_path): - open_state(tmp_path, filters=build_filters(types="photo,video", - max_size="100MB")).commit(1042) - with pytest.raises(Abort) as excinfo: - open_state(tmp_path, filters=build_filters(types="video")) - assert excinfo.value.code == 7 - reason = excinfo.value.reason - assert "filter mismatch" in reason - assert "types" in reason and "max_size" in reason - assert "--reset-state" in reason - - -def test_toggling_include_text_is_a_filter_change(tmp_path): - # It changes what a post *contains*, so running without it and then with it - # against the same cursor would silently skip every text record. - open_state(tmp_path, filters=build_filters()).commit(50) - with pytest.raises(Abort) as excinfo: - open_state(tmp_path, filters=build_filters(include_text=True)) - assert excinfo.value.code == 7 - assert "include_text" in excinfo.value.reason - - -def test_a_different_chat_against_an_existing_state_file_is_refused(tmp_path): - open_state(tmp_path) - with pytest.raises(Abort) as excinfo: - open_state(tmp_path, chat_id=-100999) - assert excinfo.value.code == 7 - assert "chat mismatch" in excinfo.value.reason - - -def test_reset_state_starts_over_without_complaining(tmp_path): - open_state(tmp_path, filters=build_filters(types="photo")).commit(1042) - fresh = open_state(tmp_path, filters=build_filters(types="video"), reset=True) - assert fresh.cursor_id == 0 - assert fresh.data["filters"]["types"] == ["video"] - - -def test_limit_is_not_part_of_the_stored_filter_set(tmp_path): - # A `--limit 20` run is a resumable partial export; if --limit were stored, - # a limited test run would poison every later full run. - assert "limit" not in open_state(tmp_path).data["filters"] - - -@pytest.mark.parametrize("filters_value", ["MISSING", None, "not-a-dict"]) -def test_a_state_file_with_no_filter_set_is_refused_as_its_own_condition( - tmp_path, filters_value): - data = {"version": 1, "chat_id": CHAT, "cursor_id": 99, "filters": filters_value} - if filters_value == "MISSING": - del data["filters"] - (tmp_path / STATE_NAME).write_text(json.dumps(data)) - - with pytest.raises(Abort) as excinfo: - open_state(tmp_path) - - assert excinfo.value.code == 7 - assert "no filter set recorded" in excinfo.value.reason - # Not reported as a phantom diff on a flag the user never passed. - assert "include_text:" not in excinfo.value.reason - - -def test_a_corrupt_state_file_is_refused_rather_than_ignored(tmp_path): - (tmp_path / STATE_NAME).write_text("{not json") - with pytest.raises(Abort) as excinfo: - open_state(tmp_path) - assert excinfo.value.code == 7 - - -def test_read_is_side_effect_free_for_dry_run(tmp_path): - assert State.read(tmp_path) is None - assert not (tmp_path / STATE_NAME).exists() diff --git a/tests/test_traversal.py b/tests/test_traversal.py deleted file mode 100644 index 54f90e1..0000000 --- a/tests/test_traversal.py +++ /dev/null @@ -1,251 +0,0 @@ -"""Album grouping, filter-invariant identity, and the album-split tripwire. - -All synthetic: no client, no network. -""" - -from __future__ import annotations - -from datetime import UTC, datetime - -import pytest - -from telegram_exporter.media import media_size -from telegram_exporter.traversal import ( - AlbumSplitError, - Filters, - SweepOrderError, - TraversalError, - build_filters, - iter_posts, - keep, - parse_date, - parse_size, -) -from tests.support import FakeClient, FakeMsg, collect - -NO_FILTERS = Filters() - - -def sweep(messages, *, filters=NO_FILTERS, after_id=0): - return collect(iter_posts(FakeClient(messages), object(), - after_id=after_id, filters=filters)) - - -# --------------------------------------------------------------------------- # -# Grouping -# --------------------------------------------------------------------------- # - -def test_a_three_photo_album_is_one_post(): - posts = sweep([FakeMsg(i, kind="photo", grouped_id=7) for i in (10, 11, 12)]) - assert len(posts) == 1 - assert posts[0].post_id == 10 - assert [m.id for m in posts[0].messages] == [10, 11, 12] - assert posts[0].max_message_id == 12 - - -def test_two_adjacent_albums_do_not_merge(): - posts = sweep([FakeMsg(10, kind="photo", grouped_id=7), - FakeMsg(11, kind="photo", grouped_id=7), - FakeMsg(12, kind="photo", grouped_id=8), - FakeMsg(13, kind="photo", grouped_id=8)]) - assert [p.post_id for p in posts] == [10, 12] - - -def test_a_standalone_message_between_albums_gets_its_own_post(): - posts = sweep([FakeMsg(10, kind="photo", grouped_id=7), - FakeMsg(11, kind="video"), - FakeMsg(12, kind="photo", grouped_id=8)]) - assert [p.post_id for p in posts] == [10, 11, 12] - - -def test_an_album_at_the_end_of_history_is_flushed(): - posts = sweep([FakeMsg(10, kind="video"), - FakeMsg(11, kind="photo", grouped_id=9), - FakeMsg(12, kind="photo", grouped_id=9)]) - assert [p.post_id for p in posts] == [10, 11] - assert len(posts[-1].messages) == 2 - - -def test_after_id_is_an_exclusive_lower_bound(): - msgs = [FakeMsg(i, kind="photo") for i in (10, 11, 12)] - assert [p.post_id for p in sweep(msgs, after_id=11)] == [12] - - -# --------------------------------------------------------------------------- # -# Invariant 1 - identity is filter-invariant -# --------------------------------------------------------------------------- # - -def test_identity_survives_a_filter_dropping_members(): - album = [FakeMsg(10, kind="photo", grouped_id=7), - FakeMsg(11, kind="video", grouped_id=7), - FakeMsg(12, kind="photo", grouped_id=7)] - photos = sweep(album, filters=build_filters(types="photo"))[0] - videos = sweep(album, filters=build_filters(types="video"))[0] - - assert photos.post_id == videos.post_id == 10 - assert photos.max_message_id == videos.max_message_id == 12 - assert [m.id for m in photos.messages] == [10, 12] - assert [m.id for m in videos.messages] == [11] - - -def test_a_fully_filtered_group_still_yields_a_post_so_the_cursor_advances(): - # Skipping empty posts stalls the cursor: a --types video run over a - # photo-heavy tail would advance only at videos, so every resume re-sweeps - # tens of thousands of messages to download nothing. - posts = sweep([FakeMsg(10, kind="photo", grouped_id=7), - FakeMsg(11, kind="photo", grouped_id=7)], - filters=build_filters(types="video")) - assert len(posts) == 1 - assert posts[0].messages == [] - assert posts[0].max_message_id == 11 - - -# --------------------------------------------------------------------------- # -# The tripwire and the ordering assertion -# --------------------------------------------------------------------------- # - -def test_a_reopened_grouped_id_raises_album_split_error(): - with pytest.raises(AlbumSplitError, match="grouped_id 7 reopened"): - sweep([FakeMsg(10, kind="photo", grouped_id=7), - FakeMsg(11, kind="photo", grouped_id=8), - FakeMsg(12, kind="photo", grouped_id=9), - FakeMsg(13, kind="photo", grouped_id=7)]) # A B C A - - -def test_the_tripwire_catches_an_immediately_reopened_album(): - with pytest.raises(AlbumSplitError): - sweep([FakeMsg(10, kind="photo", grouped_id=7), - FakeMsg(11, kind="video"), - FakeMsg(12, kind="photo", grouped_id=7)]) # A B A - - -def test_a_non_ascending_stream_is_refused(): - class Descending(FakeClient): - def iter_messages(self, entity, **kwargs): - async def gen(): - for m in self.messages: - yield m - return gen() - - with pytest.raises(SweepOrderError, match="not strictly ascending"): - collect(iter_posts(Descending([FakeMsg(12, kind="photo"), - FakeMsg(10, kind="photo")]), - object(), filters=NO_FILTERS)) - - -def test_the_album_buffer_is_bounded(): - with pytest.raises(SweepOrderError, match="album buffer exceeded"): - sweep([FakeMsg(i, kind="photo", grouped_id=7) for i in range(100, 130)]) - - -def test_the_sweep_tripwires_survive_python_dash_o(): - # `python -O` strips bare asserts. These two guard the same silent corruption - # as AlbumSplitError beside them, so they are real exceptions. - assert issubclass(SweepOrderError, TraversalError) - assert issubclass(AlbumSplitError, TraversalError) - assert not issubclass(SweepOrderError, AssertionError) - - -# --------------------------------------------------------------------------- # -# Filters -# --------------------------------------------------------------------------- # - -def test_text_only_messages_are_dropped_by_default(): - posts = sweep([FakeMsg(10, text="hello"), FakeMsg(11, kind="photo")]) - assert [p.post_id for p in posts] == [10, 11] - assert posts[0].messages == [] # yielded, but empty - assert len(posts[1].messages) == 1 - - -def test_include_text_keeps_text_messages_without_media(): - posts = sweep([FakeMsg(10, text="hello")], - filters=build_filters(include_text=True)) - assert len(posts[0].messages) == 1 - assert posts[0].messages[0].file is None # nothing to download - - -def test_a_link_preview_is_not_media(): - # Telethon's Message.photo also returns the web preview's photo, so without - # the web_preview guard every message containing a URL would look like an - # attachment. - msg = FakeMsg(10, kind="photo", text="look", web_preview=object()) - assert keep(msg, NO_FILTERS) is False - assert keep(msg, build_filters(include_text=True)) is True - - -def test_filters_compose(): - filters = build_filters(types="video", since="2026-01-01", max_size="50MB") - inside = FakeMsg(1, kind="video", size=1_000_000, - date=datetime(2026, 6, 1, tzinfo=UTC)) - too_old = FakeMsg(2, kind="video", size=1_000_000, - date=datetime(2025, 6, 1, tzinfo=UTC)) - too_big = FakeMsg(3, kind="video", size=99_000_000, - date=datetime(2026, 6, 1, tzinfo=UTC)) - wrong_kind = FakeMsg(4, kind="photo", size=1_000, - date=datetime(2026, 6, 1, tzinfo=UTC)) - assert keep(inside, filters) is True - assert not any(keep(m, filters) for m in (too_old, too_big, wrong_kind)) - - -def test_unknown_size_is_included_by_max_size_never_silently_dropped(): - msg = FakeMsg(1, kind="document", size=None) - assert media_size(msg) is None - assert keep(msg, build_filters(max_size="10MB")) is True - - -def test_media_size_is_none_rather_than_zero_for_all_three_consumers(): - # The estimator, the validator and the disk guard each have their own rule - # for None; what matters here is that the helper never raises and never - # invents a 0. - assert media_size(FakeMsg(1, kind="document", size=None)) is None - assert media_size(FakeMsg(2, text="hi")) is None - assert media_size(FakeMsg(3, kind="photo", size=17)) == 17 - - -def test_unknown_types_are_rejected_at_parse_time(): - with pytest.raises(ValueError, match="unknown media types"): - build_filters(types="photo,sticker") - - -@pytest.mark.parametrize("spec", [" ", ",", " , ", ",,"]) -def test_a_token_less_types_value_is_refused(spec): - """`--types " "` (easy to produce from an unset shell variable) used to build - an empty kind set: a filter that drops every media kind while serializing as - "no filter". A run would sweep all history, download nothing, commit an - end-of-history cursor and set completed_at - and a later correct run would see - matching filters, resume from that cursor and report success having fetched - nothing. Silent total data loss, so it is refused at the boundary.""" - with pytest.raises(ValueError, match="names no media kind"): - build_filters(types=spec) - - -def test_an_empty_kind_set_could_not_masquerade_as_no_filter(): - # Second line of defense, in case a caller bypasses build_filters. - assert Filters(types=frozenset()).to_state()["types"] == [] - assert Filters(types=None).to_state()["types"] is None - - -@pytest.mark.parametrize("text,expected", [ - ("100MB", 100_000_000), ("2GiB", 2 * 2**30), ("1500000", 1_500_000), - ("50 MB", 50_000_000), ("1KiB", 1024), ("1.5GB", 1_500_000_000), -]) -def test_parse_size(text, expected): - assert parse_size(text) == expected - - -def test_parse_size_rejects_nonsense(): - with pytest.raises(ValueError): - parse_size("plenty") - - -def test_dates_are_utc_aware_so_comparison_cannot_raise(): - # msg.date is UTC-aware; a naive bound would raise TypeError mid-sweep. - assert parse_date("2026-01-01").tzinfo is not None - assert parse_date("2026-01-01T00:00:00+07:00").tzinfo is not None - - -def test_stored_filters_are_verbatim_and_comparable(): - stored = build_filters(types="video,photo", max_size="100MB", - include_text=True).to_state() - assert stored == {"types": ["photo", "video"], "since": None, "until": None, - "max_size": 100_000_000, "include_text": True}