mirror of
https://github.com/tiennm99/telegram-exporter.git
synced 2026-10-11 03:13:49 +00:00
tiennm99
6fb77ce633
Backends that finish an upload as a server-side task, pikpak among them, leave it pending long enough that rclone abandons the transfer once --low-level-retries polls run out. Default RCLONE_TRANSFERS to 2 to keep that task queue short and RCLONE_LOW_LEVEL_RETRIES to 20 to wait a slow commit out. Both use the :=default form, so an exported value still wins. Shorten the --min-age default to 45s: excluding '*.tmp' is what actually keeps an unfinished download from being uploaded, so the age guard only needs to cover a rename racing a sweep, and two minutes of dwell made staging larger than it had to be. Sweep every 120s rather than 300s in the driver, which shrinks the backlog each sweep has to move, and raise the pass limit to 30 to suit chats that need more rounds than 20 to converge.
114 lines
4.5 KiB
Bash
Executable File
114 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Drive run.sh repeatedly until every media file in a chat is on the remote.
|
|
#
|
|
# Each pass verifies what is already there, narrows the export JSON to just the
|
|
# ids still needed, and runs the pipeline on that subset. It stops when the
|
|
# verifier reports complete, when a pass makes no progress (the remaining media
|
|
# is genuinely unavailable on Telegram's side), or when the remote runs low on
|
|
# space.
|
|
#
|
|
# Usage: ./export-until-complete.sh -r REMOTE:PATH -c CHAT [options]
|
|
# -r REMOTE:PATH rclone destination (required)
|
|
# -c CHAT chat id/username, used for the initial metadata export
|
|
# -f FILE export JSON (default export-<chat>.json)
|
|
# -d DIR staging directory (default ./staging)
|
|
# -i SECONDS rclone sweep interval (default 120)
|
|
# -p N maximum passes (default 30)
|
|
# -m SIZE cap the staging directory at SIZE, e.g. 40G (default: no cap)
|
|
# -q GIB stop if remote free space falls below this (default 5)
|
|
|
|
set -euo pipefail
|
|
|
|
remote='' chat='' export_file='' staging='./staging' interval=120 max_staging=''
|
|
max_passes=30 min_free_gib=5
|
|
|
|
while getopts ':r:c:f:d:i:p:q:m:h' o; do case $o in
|
|
r) remote=$OPTARG ;; c) chat=$OPTARG ;; f) export_file=$OPTARG ;;
|
|
d) staging=$OPTARG ;; i) interval=$OPTARG ;; p) max_passes=$OPTARG ;;
|
|
q) min_free_gib=$OPTARG ;; m) max_staging=$OPTARG ;;
|
|
h) sed -n '2,19p' "$0"; exit 0 ;;
|
|
*) echo "usage: $0 -r REMOTE:PATH -c CHAT [-f FILE] [-i SECS] [-p N] [-q GIB] [-m SIZE]" >&2; exit 2 ;;
|
|
esac; done
|
|
|
|
log() { printf '\n=== %s [driver] %s ===\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"; }
|
|
|
|
[[ -n $remote ]] || { echo "-r REMOTE:PATH is required" >&2; exit 2; }
|
|
[[ -n $chat || -n $export_file ]] || { echo "-c CHAT or -f FILE is required" >&2; exit 2; }
|
|
[[ -n $export_file ]] || export_file="export-${chat}.json"
|
|
|
|
# Stop the whole loop on Ctrl-C rather than rolling into the next pass. run.sh
|
|
# installs its own handlers, so a pass shuts down cleanly before we exit.
|
|
interrupted=0
|
|
trap 'interrupted=1' INT TERM
|
|
|
|
# tdl's progress bar is ANSI redraws: great on a terminal, unreadable in a log.
|
|
# Show it when stdout is a TTY, suppress it when output is redirected.
|
|
tdl_quiet=()
|
|
[[ -t 1 ]] || tdl_quiet=(--disable-progress-ps)
|
|
|
|
# The metadata export must exist before the first verify has anything to compare.
|
|
if [[ ! -f $export_file ]]; then
|
|
[[ -n $chat ]] || { echo "$export_file missing and no -c CHAT to create it" >&2; exit 2; }
|
|
log "exporting $chat metadata to $export_file"
|
|
tdl chat export -c "$chat" --all --with-content -o "$export_file"
|
|
fi
|
|
|
|
free_gib() {
|
|
rclone about "${remote%%:*}:" --json 2>/dev/null \
|
|
| python3 -c 'import json,sys; print(int(json.load(sys.stdin).get("free",0))//2**30)' 2>/dev/null \
|
|
|| echo 999999 # backends without quota reporting must not block the run
|
|
}
|
|
|
|
prev_todo=-1
|
|
for ((pass = 1; pass <= max_passes; pass++)); do
|
|
((interrupted)) && { log 'interrupted; stopping'; exit 130; }
|
|
|
|
log "pass $pass/$max_passes: verifying"
|
|
if ./verify-export.sh -f "$export_file" -r "$remote" -d "$staging"; then
|
|
log "complete after $((pass - 1)) download pass(es)"
|
|
exit 0
|
|
fi
|
|
|
|
todo=$(wc -l < missing-ids.txt)
|
|
if ((todo == prev_todo)); then
|
|
log "no progress in the last pass; $todo file(s) look permanently unavailable"
|
|
log 'ids left in missing-ids.txt'
|
|
exit 1
|
|
fi
|
|
prev_todo=$todo
|
|
|
|
free=$(free_gib)
|
|
if ((free < min_free_gib)); then
|
|
log "remote has only ${free} GiB free (limit ${min_free_gib}); stopping before it fills"
|
|
exit 3
|
|
fi
|
|
log "$todo file(s) to fetch; ${free} GiB free on remote"
|
|
|
|
# Narrow the full export to the outstanding ids, preserving its top-level shape
|
|
# so tdl reads it exactly like the original.
|
|
python3 - "$export_file" <<'PY'
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
want = {int(l) for l in open('missing-ids.txt') if l.strip()}
|
|
d['messages'] = [m for m in d['messages'] if m['id'] in want]
|
|
json.dump(d, open('gap.json', 'w'))
|
|
print(f"gap.json: {len(d['messages'])} messages")
|
|
PY
|
|
|
|
rc=0
|
|
./run.sh -r "$remote" -f gap.json -d "$staging" -i "$interval" \
|
|
${max_staging:+-m "$max_staging"} \
|
|
-- --group=false -t 4 -l 2 ${tdl_quiet[@]+"${tdl_quiet[@]}"} || rc=$?
|
|
log "pass $pass finished (run.sh exit $rc)"
|
|
|
|
case $rc in
|
|
0|1) ;; # done or partial: verify decides
|
|
3) log 'rclone failure (remote full or unreachable); stopping'; exit 3 ;;
|
|
130|143) log 'run interrupted; stopping'; exit 130 ;;
|
|
esac
|
|
done
|
|
|
|
log "hit the $max_passes-pass limit; run again to continue"
|
|
exit 1
|