diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f86285a --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# tdl chat exports and the narrowed subsets built from them. These hold real +# message ids, file names and text from an account, and are specific to one run. +# The repo ships no JSON of its own, so the whole extension is excluded. +*.json + +# Ids still to fetch, written by verify-export.sh. +missing-ids.txt + +# Staging holds media in flight between tdl and the remote. +staging/ + +# Run logs carry chat names and progress output. +*.log + +# Session plans and reports. +plans/ diff --git a/README.md b/README.md index d8d24e9..3a555f5 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,63 @@ cross-run deduplication rests on tdl's own `--continue` tracking. If you must start from a fresh export, narrow it to the missing message-id range (`-T id -i ,`) rather than re-downloading everything. +## Verifying an export + +`run.sh` finishes when tdl finishes, which is not the same as every file having +arrived: a dropped session, a stalled remote, or an interrupted pass all leave +gaps. `verify-export.sh` settles it by rebuilding the filename tdl produces for +each media message in the export JSON and checking the remote for it. + +```bash +./verify-export.sh -f export-mygroup.json -r remote:telegram/media +``` + +``` +messages in export : 18193 + text-only (skip) : 38 + media expected : 12000 +present and intact : 12000 + absent : 0 + zero-byte : 0 + +COMPLETE: every media message is present and non-empty. +``` + +Messages with no media are skipped; they carry an empty `file` and were never +download targets. A zero-byte file counts as missing, because rclone overwrites a +size-mismatched destination and a retry repairs it. Files under 1 KiB are +reported but not retried, since some real media is genuinely that small. Exit +status is 0 when complete and 1 otherwise, with the outstanding message ids +written to `missing-ids.txt`. + +## Running until complete + +`export-until-complete.sh` drives `run.sh` in a loop: verify what is already +there, narrow the export to the ids still missing, run the pipeline on that +subset, and repeat. + +```bash +./export-until-complete.sh -r remote:telegram/media -c @mygroup +``` + +| Flag | Meaning | +|------|---------| +| `-r REMOTE:PATH` | **Required.** rclone destination | +| `-c CHAT` | Chat to export metadata for on the first pass | +| `-f FILE` | Export JSON (default `export-.json`) | +| `-d DIR` | Staging directory (default `./staging`) | +| `-i SECONDS` | rclone sweep interval (default `300`) | +| `-p N` | Maximum passes (default `20`) | +| `-q GIB` | Stop if remote free space falls below this (default `5`) | + +It stops when the verifier reports complete (exit `0`), when a pass fetches +nothing new (exit `1` — the remaining media is no longer available from +Telegram), when the remote runs low on space (exit `3`), or on Ctrl-C (exit +`130`, after the current pass shuts down cleanly). + +tdl's progress bar is shown when stdout is a terminal and suppressed when output +is redirected, so a log file stays readable without a flag. + ## What it guards against - **Partial uploads.** tdl writes `.tmp` and renames on completion, so diff --git a/export-until-complete.sh b/export-until-complete.sh new file mode 100755 index 0000000..becff63 --- /dev/null +++ b/export-until-complete.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Drive run.sh repeatedly until every media file in a chat is on the remote. +# +# Each pass verifies what is already there, narrows the export JSON to just the +# ids still needed, and runs the pipeline on that subset. It stops when the +# verifier reports complete, when a pass makes no progress (the remaining media +# is genuinely unavailable on Telegram's side), or when the remote runs low on +# space. +# +# Usage: ./export-until-complete.sh -r REMOTE:PATH -c CHAT [options] +# -r REMOTE:PATH rclone destination (required) +# -c CHAT chat id/username, used for the initial metadata export +# -f FILE export JSON (default export-.json) +# -d DIR staging directory (default ./staging) +# -i SECONDS rclone sweep interval (default 300) +# -p N maximum passes (default 20) +# -q GIB stop if remote free space falls below this (default 5) + +set -euo pipefail + +remote='' chat='' export_file='' staging='./staging' interval=300 +max_passes=20 min_free_gib=5 + +while getopts ':r:c:f:d:i:p:q:h' o; do case $o in + r) remote=$OPTARG ;; c) chat=$OPTARG ;; f) export_file=$OPTARG ;; + d) staging=$OPTARG ;; i) interval=$OPTARG ;; p) max_passes=$OPTARG ;; + q) min_free_gib=$OPTARG ;; + h) sed -n '2,18p' "$0"; exit 0 ;; + *) echo "usage: $0 -r REMOTE:PATH -c CHAT [-f FILE] [-i SECS] [-p N] [-q GIB]" >&2; exit 2 ;; +esac; done + +log() { printf '\n=== %s [driver] %s ===\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; } + +[[ -n $remote ]] || { echo "-r REMOTE:PATH is required" >&2; exit 2; } +[[ -n $chat || -n $export_file ]] || { echo "-c CHAT or -f FILE is required" >&2; exit 2; } +[[ -n $export_file ]] || export_file="export-${chat}.json" + +# Stop the whole loop on Ctrl-C rather than rolling into the next pass. run.sh +# installs its own handlers, so a pass shuts down cleanly before we exit. +interrupted=0 +trap 'interrupted=1' INT TERM + +# tdl's progress bar is ANSI redraws: great on a terminal, unreadable in a log. +# Show it when stdout is a TTY, suppress it when output is redirected. +tdl_quiet=() +[[ -t 1 ]] || tdl_quiet=(--disable-progress-ps) + +# The metadata export must exist before the first verify has anything to compare. +if [[ ! -f $export_file ]]; then + [[ -n $chat ]] || { echo "$export_file missing and no -c CHAT to create it" >&2; exit 2; } + log "exporting $chat metadata to $export_file" + tdl chat export -c "$chat" --all --with-content -o "$export_file" +fi + +free_gib() { + rclone about "${remote%%:*}:" --json 2>/dev/null \ + | python3 -c 'import json,sys; print(int(json.load(sys.stdin).get("free",0))//2**30)' 2>/dev/null \ + || echo 999999 # backends without quota reporting must not block the run +} + +prev_todo=-1 +for ((pass = 1; pass <= max_passes; pass++)); do + ((interrupted)) && { log 'interrupted; stopping'; exit 130; } + + log "pass $pass/$max_passes: verifying" + if ./verify-export.sh -f "$export_file" -r "$remote" -d "$staging"; then + log "complete after $((pass - 1)) download pass(es)" + exit 0 + fi + + todo=$(wc -l < missing-ids.txt) + if ((todo == prev_todo)); then + log "no progress in the last pass; $todo file(s) look permanently unavailable" + log 'ids left in missing-ids.txt' + exit 1 + fi + prev_todo=$todo + + free=$(free_gib) + if ((free < min_free_gib)); then + log "remote has only ${free} GiB free (limit ${min_free_gib}); stopping before it fills" + exit 3 + fi + log "$todo file(s) to fetch; ${free} GiB free on remote" + + # Narrow the full export to the outstanding ids, preserving its top-level shape + # so tdl reads it exactly like the original. + python3 - "$export_file" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +want = {int(l) for l in open('missing-ids.txt') if l.strip()} +d['messages'] = [m for m in d['messages'] if m['id'] in want] +json.dump(d, open('gap.json', 'w')) +print(f"gap.json: {len(d['messages'])} messages") +PY + + rc=0 + ./run.sh -r "$remote" -f gap.json -d "$staging" -i "$interval" \ + -- --group=false -t 4 -l 2 ${tdl_quiet[@]+"${tdl_quiet[@]}"} || rc=$? + log "pass $pass finished (run.sh exit $rc)" + + case $rc in + 0|1) ;; # done or partial: verify decides + 3) log 'rclone failure (remote full or unreachable); stopping'; exit 3 ;; + 130|143) log 'run interrupted; stopping'; exit 130 ;; + esac +done + +log "hit the $max_passes-pass limit; run again to continue" +exit 1 diff --git a/verify-export.sh b/verify-export.sh new file mode 100755 index 0000000..16734ef --- /dev/null +++ b/verify-export.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# +# Verify a tdl/rclone export is complete and every file is intact. +# +# Rebuilds the filename tdl produces for each media message in the export JSON +# ({DialogID}_{MessageID}_{FileName}) and checks it exists on the remote or in +# staging. Zero-byte files count as missing: rclone overwrites a size-mismatched +# destination, so re-running repairs them. Files under 1 KiB are reported for +# inspection but trusted, since some real media is genuinely that small. +# +# Writes every id needing another attempt to missing-ids.txt. +# Exit 0 = complete, 1 = incomplete, 2 = usage error. +# +# Usage: ./verify-export.sh -f export-.json -r REMOTE:PATH [-d STAGING] + +set -euo pipefail + +export_file='' remote='' staging='./staging' +while getopts ':f:r:d:h' o; do case $o in + f) export_file=$OPTARG ;; r) remote=$OPTARG ;; d) staging=$OPTARG ;; + h) sed -n '2,16p' "$0"; exit 0 ;; + *) echo "usage: $0 -f FILE -r REMOTE:PATH [-d STAGING]" >&2; exit 2 ;; +esac; done + +[[ -f $export_file ]] || { echo "no such export file: $export_file" >&2; exit 2; } +[[ -n $remote ]] || { echo "-r REMOTE:PATH is required" >&2; exit 2; } + +listing=$(mktemp); trap 'rm -f "$listing"' EXIT +# lsl gives sizes as well as names, so truncated uploads are detectable. +rclone lsl "$remote" > "$listing" + +python3 - "$export_file" "$listing" "$staging" <<'PY' +import json, os, re, sys +export_file, listing, staging = sys.argv[1], sys.argv[2], sys.argv[3] + +sizes = {} +for line in open(listing): + m = re.match(r'^\s*(\d+)\s+\S+\s+\S+\s+(.*)$', line.rstrip('\n')) + if m: + sizes[m.group(2)] = int(m.group(1)) +if os.path.isdir(staging): + for f in os.listdir(staging): + if not f.endswith('.tmp'): + sizes.setdefault(f, os.path.getsize(os.path.join(staging, f))) + +msgs = json.load(open(export_file))['messages'] +# Text-only messages carry an empty "file" and are not download targets. +media = [m for m in msgs if m.get('file')] + +dialog = str(json.load(open(export_file)).get('id', '')).lstrip('-') +if not dialog.isdigit(): + ids = {n.split('_')[0] for n in sizes if '_' in n} + dialog = ids.pop() if len(ids) == 1 else '' +if not dialog: + sys.exit('cannot determine dialog id') + +absent, empty, tiny = [], [], [] +for m in media: + name = f"{dialog}_{m['id']}_{m['file']}" + if name not in sizes: + absent.append(m['id']) + elif sizes[name] == 0: + empty.append(m['id']) + elif sizes[name] < 1024: + tiny.append((m['id'], sizes[name], name)) + +todo = sorted(absent + empty) +print(f"messages in export : {len(msgs)}") +print(f" text-only (skip) : {len(msgs) - len(media)}") +print(f" media expected : {len(media)}") +print(f"present and intact : {len(media) - len(todo)}") +print(f" absent : {len(absent)}") +print(f" zero-byte : {len(empty)}") +if tiny: + print(f" under 1KiB (check, not retried): {len(tiny)}") + for i, s, n in tiny[:5]: + print(f" id {i} {s} B {n}") + +if todo: + with open('missing-ids.txt', 'w') as fh: + fh.write('\n'.join(map(str, todo)) + '\n') + print(f"\nneeds another pass : {len(todo)} (ids {min(todo)}–{max(todo)})") + print("written to missing-ids.txt") + sys.exit(1) + +if os.path.exists('missing-ids.txt'): + os.remove('missing-ids.txt') +print("\nCOMPLETE: every media message is present and non-empty.") +PY