From 913b0d0b7d62fbec5c7a6c9c6a0b758f4850884b Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Thu, 3 Sep 2026 22:42:32 +0700 Subject: [PATCH] feat: verify export completeness and drive runs until complete run.sh finishes when tdl finishes, which does not mean every file arrived: a dropped session, a stalled remote or an interrupted pass leaves gaps that nothing reported. verify-export.sh rebuilds the filename tdl produces for each media message in the export JSON and checks the remote and staging for it. Messages carrying no media are skipped rather than counted as gaps. A zero-byte file counts as missing because rclone overwrites a size-mismatched destination, so a retry repairs it; files under 1 KiB are reported but not retried, since real media is sometimes that small and retrying them would never terminate. export-until-complete.sh loops that check with run.sh, narrowing the export to the outstanding ids each pass. It stops when the verifier reports complete, when a pass fetches nothing new, when the remote runs low on space, or on Ctrl-C. tdl's progress bar is shown on a terminal and suppressed when output is redirected, so logs stay readable. Exports, staging and logs are now ignored: they hold real message ids, file names and text from an account. --- .gitignore | 16 ++++++ README.md | 57 ++++++++++++++++++++ export-until-complete.sh | 111 +++++++++++++++++++++++++++++++++++++++ verify-export.sh | 89 +++++++++++++++++++++++++++++++ 4 files changed, 273 insertions(+) create mode 100644 .gitignore create mode 100755 export-until-complete.sh create mode 100755 verify-export.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f86285a --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# tdl chat exports and the narrowed subsets built from them. These hold real +# message ids, file names and text from an account, and are specific to one run. +# The repo ships no JSON of its own, so the whole extension is excluded. +*.json + +# Ids still to fetch, written by verify-export.sh. +missing-ids.txt + +# Staging holds media in flight between tdl and the remote. +staging/ + +# Run logs carry chat names and progress output. +*.log + +# Session plans and reports. +plans/ diff --git a/README.md b/README.md index d8d24e9..3a555f5 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,63 @@ cross-run deduplication rests on tdl's own `--continue` tracking. If you must start from a fresh export, narrow it to the missing message-id range (`-T id -i ,`) rather than re-downloading everything. +## Verifying an export + +`run.sh` finishes when tdl finishes, which is not the same as every file having +arrived: a dropped session, a stalled remote, or an interrupted pass all leave +gaps. `verify-export.sh` settles it by rebuilding the filename tdl produces for +each media message in the export JSON and checking the remote for it. + +```bash +./verify-export.sh -f export-mygroup.json -r remote:telegram/media +``` + +``` +messages in export : 18193 + text-only (skip) : 38 + media expected : 12000 +present and intact : 12000 + absent : 0 + zero-byte : 0 + +COMPLETE: every media message is present and non-empty. +``` + +Messages with no media are skipped; they carry an empty `file` and were never +download targets. A zero-byte file counts as missing, because rclone overwrites a +size-mismatched destination and a retry repairs it. Files under 1 KiB are +reported but not retried, since some real media is genuinely that small. Exit +status is 0 when complete and 1 otherwise, with the outstanding message ids +written to `missing-ids.txt`. + +## Running until complete + +`export-until-complete.sh` drives `run.sh` in a loop: verify what is already +there, narrow the export to the ids still missing, run the pipeline on that +subset, and repeat. + +```bash +./export-until-complete.sh -r remote:telegram/media -c @mygroup +``` + +| Flag | Meaning | +|------|---------| +| `-r REMOTE:PATH` | **Required.** rclone destination | +| `-c CHAT` | Chat to export metadata for on the first pass | +| `-f FILE` | Export JSON (default `export-.json`) | +| `-d DIR` | Staging directory (default `./staging`) | +| `-i SECONDS` | rclone sweep interval (default `300`) | +| `-p N` | Maximum passes (default `20`) | +| `-q GIB` | Stop if remote free space falls below this (default `5`) | + +It stops when the verifier reports complete (exit `0`), when a pass fetches +nothing new (exit `1` — the remaining media is no longer available from +Telegram), when the remote runs low on space (exit `3`), or on Ctrl-C (exit +`130`, after the current pass shuts down cleanly). + +tdl's progress bar is shown when stdout is a terminal and suppressed when output +is redirected, so a log file stays readable without a flag. + ## What it guards against - **Partial uploads.** tdl writes `.tmp` and renames on completion, so diff --git a/export-until-complete.sh b/export-until-complete.sh new file mode 100755 index 0000000..becff63 --- /dev/null +++ b/export-until-complete.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Drive run.sh repeatedly until every media file in a chat is on the remote. +# +# Each pass verifies what is already there, narrows the export JSON to just the +# ids still needed, and runs the pipeline on that subset. It stops when the +# verifier reports complete, when a pass makes no progress (the remaining media +# is genuinely unavailable on Telegram's side), or when the remote runs low on +# space. +# +# Usage: ./export-until-complete.sh -r REMOTE:PATH -c CHAT [options] +# -r REMOTE:PATH rclone destination (required) +# -c CHAT chat id/username, used for the initial metadata export +# -f FILE export JSON (default export-.json) +# -d DIR staging directory (default ./staging) +# -i SECONDS rclone sweep interval (default 300) +# -p N maximum passes (default 20) +# -q GIB stop if remote free space falls below this (default 5) + +set -euo pipefail + +remote='' chat='' export_file='' staging='./staging' interval=300 +max_passes=20 min_free_gib=5 + +while getopts ':r:c:f:d:i:p:q:h' o; do case $o in + r) remote=$OPTARG ;; c) chat=$OPTARG ;; f) export_file=$OPTARG ;; + d) staging=$OPTARG ;; i) interval=$OPTARG ;; p) max_passes=$OPTARG ;; + q) min_free_gib=$OPTARG ;; + h) sed -n '2,18p' "$0"; exit 0 ;; + *) echo "usage: $0 -r REMOTE:PATH -c CHAT [-f FILE] [-i SECS] [-p N] [-q GIB]" >&2; exit 2 ;; +esac; done + +log() { printf '\n=== %s [driver] %s ===\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; } + +[[ -n $remote ]] || { echo "-r REMOTE:PATH is required" >&2; exit 2; } +[[ -n $chat || -n $export_file ]] || { echo "-c CHAT or -f FILE is required" >&2; exit 2; } +[[ -n $export_file ]] || export_file="export-${chat}.json" + +# Stop the whole loop on Ctrl-C rather than rolling into the next pass. run.sh +# installs its own handlers, so a pass shuts down cleanly before we exit. +interrupted=0 +trap 'interrupted=1' INT TERM + +# tdl's progress bar is ANSI redraws: great on a terminal, unreadable in a log. +# Show it when stdout is a TTY, suppress it when output is redirected. +tdl_quiet=() +[[ -t 1 ]] || tdl_quiet=(--disable-progress-ps) + +# The metadata export must exist before the first verify has anything to compare. +if [[ ! -f $export_file ]]; then + [[ -n $chat ]] || { echo "$export_file missing and no -c CHAT to create it" >&2; exit 2; } + log "exporting $chat metadata to $export_file" + tdl chat export -c "$chat" --all --with-content -o "$export_file" +fi + +free_gib() { + rclone about "${remote%%:*}:" --json 2>/dev/null \ + | python3 -c 'import json,sys; print(int(json.load(sys.stdin).get("free",0))//2**30)' 2>/dev/null \ + || echo 999999 # backends without quota reporting must not block the run +} + +prev_todo=-1 +for ((pass = 1; pass <= max_passes; pass++)); do + ((interrupted)) && { log 'interrupted; stopping'; exit 130; } + + log "pass $pass/$max_passes: verifying" + if ./verify-export.sh -f "$export_file" -r "$remote" -d "$staging"; then + log "complete after $((pass - 1)) download pass(es)" + exit 0 + fi + + todo=$(wc -l < missing-ids.txt) + if ((todo == prev_todo)); then + log "no progress in the last pass; $todo file(s) look permanently unavailable" + log 'ids left in missing-ids.txt' + exit 1 + fi + prev_todo=$todo + + free=$(free_gib) + if ((free < min_free_gib)); then + log "remote has only ${free} GiB free (limit ${min_free_gib}); stopping before it fills" + exit 3 + fi + log "$todo file(s) to fetch; ${free} GiB free on remote" + + # Narrow the full export to the outstanding ids, preserving its top-level shape + # so tdl reads it exactly like the original. + python3 - "$export_file" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +want = {int(l) for l in open('missing-ids.txt') if l.strip()} +d['messages'] = [m for m in d['messages'] if m['id'] in want] +json.dump(d, open('gap.json', 'w')) +print(f"gap.json: {len(d['messages'])} messages") +PY + + rc=0 + ./run.sh -r "$remote" -f gap.json -d "$staging" -i "$interval" \ + -- --group=false -t 4 -l 2 ${tdl_quiet[@]+"${tdl_quiet[@]}"} || rc=$? + log "pass $pass finished (run.sh exit $rc)" + + case $rc in + 0|1) ;; # done or partial: verify decides + 3) log 'rclone failure (remote full or unreachable); stopping'; exit 3 ;; + 130|143) log 'run interrupted; stopping'; exit 130 ;; + esac +done + +log "hit the $max_passes-pass limit; run again to continue" +exit 1 diff --git a/verify-export.sh b/verify-export.sh new file mode 100755 index 0000000..16734ef --- /dev/null +++ b/verify-export.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Verify a tdl/rclone export is complete and every file is intact. +# +# Rebuilds the filename tdl produces for each media message in the export JSON +# ({DialogID}_{MessageID}_{FileName}) and checks it exists on the remote or in +# staging. Zero-byte files count as missing: rclone overwrites a size-mismatched +# destination, so re-running repairs them. Files under 1 KiB are reported for +# inspection but trusted, since some real media is genuinely that small. +# +# Writes every id needing another attempt to missing-ids.txt. +# Exit 0 = complete, 1 = incomplete, 2 = usage error. +# +# Usage: ./verify-export.sh -f export-.json -r REMOTE:PATH [-d STAGING] + +set -euo pipefail + +export_file='' remote='' staging='./staging' +while getopts ':f:r:d:h' o; do case $o in + f) export_file=$OPTARG ;; r) remote=$OPTARG ;; d) staging=$OPTARG ;; + h) sed -n '2,16p' "$0"; exit 0 ;; + *) echo "usage: $0 -f FILE -r REMOTE:PATH [-d STAGING]" >&2; exit 2 ;; +esac; done + +[[ -f $export_file ]] || { echo "no such export file: $export_file" >&2; exit 2; } +[[ -n $remote ]] || { echo "-r REMOTE:PATH is required" >&2; exit 2; } + +listing=$(mktemp); trap 'rm -f "$listing"' EXIT +# lsl gives sizes as well as names, so truncated uploads are detectable. +rclone lsl "$remote" > "$listing" + +python3 - "$export_file" "$listing" "$staging" <<'PY' +import json, os, re, sys +export_file, listing, staging = sys.argv[1], sys.argv[2], sys.argv[3] + +sizes = {} +for line in open(listing): + m = re.match(r'^\s*(\d+)\s+\S+\s+\S+\s+(.*)$', line.rstrip('\n')) + if m: + sizes[m.group(2)] = int(m.group(1)) +if os.path.isdir(staging): + for f in os.listdir(staging): + if not f.endswith('.tmp'): + sizes.setdefault(f, os.path.getsize(os.path.join(staging, f))) + +msgs = json.load(open(export_file))['messages'] +# Text-only messages carry an empty "file" and are not download targets. +media = [m for m in msgs if m.get('file')] + +dialog = str(json.load(open(export_file)).get('id', '')).lstrip('-') +if not dialog.isdigit(): + ids = {n.split('_')[0] for n in sizes if '_' in n} + dialog = ids.pop() if len(ids) == 1 else '' +if not dialog: + sys.exit('cannot determine dialog id') + +absent, empty, tiny = [], [], [] +for m in media: + name = f"{dialog}_{m['id']}_{m['file']}" + if name not in sizes: + absent.append(m['id']) + elif sizes[name] == 0: + empty.append(m['id']) + elif sizes[name] < 1024: + tiny.append((m['id'], sizes[name], name)) + +todo = sorted(absent + empty) +print(f"messages in export : {len(msgs)}") +print(f" text-only (skip) : {len(msgs) - len(media)}") +print(f" media expected : {len(media)}") +print(f"present and intact : {len(media) - len(todo)}") +print(f" absent : {len(absent)}") +print(f" zero-byte : {len(empty)}") +if tiny: + print(f" under 1KiB (check, not retried): {len(tiny)}") + for i, s, n in tiny[:5]: + print(f" id {i} {s} B {n}") + +if todo: + with open('missing-ids.txt', 'w') as fh: + fh.write('\n'.join(map(str, todo)) + '\n') + print(f"\nneeds another pass : {len(todo)} (ids {min(todo)}–{max(todo)})") + print("written to missing-ids.txt") + sys.exit(1) + +if os.path.exists('missing-ids.txt'): + os.remove('missing-ids.txt') +print("\nCOMPLETE: every media message is present and non-empty.") +PY