mirror of
https://github.com/tiennm99/telegram-exporter.git
synced 2026-10-11 03:13:49 +00:00
feat: add a tdl-to-webdav rolling pipeline script
tdl has no remote destination, so exporting a group larger than local disk needs a staging dir plus a concurrent uploader. The script runs tdl into a small staging dir while rclone moves finished files out to WebDAV, so local disk only holds the files in flight plus one sync interval of throughput. Partial downloads are excluded by name rather than by age: tdl writes <name>.tmp and renames on completion, so a download stalled by a flood wait stops touching its .tmp and would otherwise age past --min-age and be uploaded half-written, destroying resume for that file. --delete-empty-src-dirs runs only in the final sweep, because removing a directory under a running tdl makes it fail to create its next file. tdl is stopped on exit, interrupt or termination so no download is orphaned, the unrestricted sweep runs only after tdl exits 0, and five consecutive rclone failures abort the run before staging fills the disk.
This commit is contained in:
tiennm99 committed
1 parent
ec0584e34e
commit
7d20f5667c
1 file changed
+170
Executable
+170
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Rolling pipeline: tdl downloads Telegram media into a small staging directory
|
||||
# while rclone concurrently moves finished files to a WebDAV remote. Local disk
|
||||
# only ever holds the files in flight plus one sync interval of throughput, so a
|
||||
# group larger than the local disk can still be exported.
|
||||
#
|
||||
# See README.md ("Special case: exporting to WebDAV") for the background.
|
||||
#
|
||||
# Exit codes: 0 ok, 2 usage error, 3 rclone failure, 130/143 interrupted,
|
||||
# anything else is tdl's own exit code.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
readonly PROG=${0##*/}
|
||||
|
||||
# tdl downloads to '<name>.tmp' and renames only on completion, so an unfinished
|
||||
# file is always identifiable by extension. Never move one: a download stalled
|
||||
# by a flood wait stops touching its .tmp, which then ages past --min-age and
|
||||
# would be uploaded half-written, destroying tdl's resume point for that file.
|
||||
readonly TEMP_GLOB='*.tmp'
|
||||
|
||||
# Defaults
|
||||
export_file='export.json'
|
||||
chat=''
|
||||
staging='./staging'
|
||||
remote=''
|
||||
interval=60
|
||||
min_age='2m'
|
||||
max_sync_failures=5
|
||||
|
||||
usage() {
|
||||
cat <<USAGE
|
||||
Usage: $PROG -r REMOTE:PATH [options] [-- extra tdl dl args...]
|
||||
|
||||
Required:
|
||||
-r REMOTE:PATH rclone destination, e.g. tg-webdav:tg-export
|
||||
|
||||
Options:
|
||||
-f FILE tdl export JSON (default: $export_file)
|
||||
-c CHAT chat to export when FILE does not exist, e.g. @mygroup
|
||||
-d DIR staging directory (default: $staging)
|
||||
-i SECONDS seconds between rclone sweeps (default: $interval)
|
||||
-a AGE rclone --min-age, a second guard against moving files still
|
||||
being written (default: $min_age)
|
||||
-h this help
|
||||
|
||||
Everything after -- is appended to the 'tdl dl' command, e.g.
|
||||
$PROG -r tg-webdav:tg-export -- -t 4 -l 1
|
||||
USAGE
|
||||
}
|
||||
|
||||
log() { printf '%s [%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$PROG" "$*" >&2; }
|
||||
die() { log "error: $*"; exit 2; } # usage or precondition
|
||||
fail() { log "error: $*"; exit 3; } # rclone / pipeline failure
|
||||
|
||||
while getopts ':f:c:d:r:i:a:h' opt; do
|
||||
case $opt in
|
||||
f) export_file=$OPTARG ;;
|
||||
c) chat=$OPTARG ;;
|
||||
d) staging=$OPTARG ;;
|
||||
r) remote=$OPTARG ;;
|
||||
i) interval=$OPTARG ;;
|
||||
a) min_age=$OPTARG ;;
|
||||
h) usage; exit 0 ;;
|
||||
:) die "option -$OPTARG requires an argument" ;;
|
||||
?) die "unknown option -$OPTARG (try -h)" ;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
tdl_extra=("$@")
|
||||
|
||||
[[ -n $remote ]] || { usage >&2; die "-r REMOTE:PATH is required"; }
|
||||
[[ $remote == *:* ]] || die "remote '$remote' is not in rclone REMOTE:PATH form"
|
||||
[[ $interval =~ ^[0-9]+$ && $interval -gt 0 ]] || die "-i must be a positive integer"
|
||||
[[ $min_age =~ ^[0-9]+(\.[0-9]+)?(ms|s|m|h|d|w|M|y)?$ ]] \
|
||||
|| die "-a must be an rclone duration, e.g. 2m"
|
||||
|
||||
for tool in tdl rclone; do
|
||||
command -v "$tool" >/dev/null || die "$tool is not installed or not on PATH"
|
||||
done
|
||||
|
||||
rclone lsd "${remote%%:*}:" >/dev/null 2>&1 \
|
||||
|| die "rclone cannot reach remote '${remote%%:*}:' — check 'rclone config'"
|
||||
|
||||
# The export JSON only lists messages; it is cheap to keep and required for both
|
||||
# legs to stay resumable, so never regenerate it when it already exists.
|
||||
if [[ ! -f $export_file ]]; then
|
||||
[[ -n $chat ]] || die "$export_file not found; pass -c CHAT to export it first"
|
||||
log "exporting $chat metadata to $export_file"
|
||||
tdl chat export -c "$chat" --all --with-content -o "$export_file"
|
||||
fi
|
||||
|
||||
mkdir -p "$staging"
|
||||
|
||||
tdl_pid=''
|
||||
tdl_rc=0
|
||||
sweep_ok=0
|
||||
|
||||
# Move whatever is finished. Partial .tmp files are always excluded. $1 is an
|
||||
# optional --min-age guard; $2 enables --delete-empty-src-dirs, which is safe
|
||||
# only once tdl has stopped — rclone removing a directory between tdl's
|
||||
# MkdirAll and Create makes tdl fail, and it can take the staging root too,
|
||||
# hence the mkdir afterwards.
|
||||
sweep() {
|
||||
local rc=0
|
||||
local args=(--exclude "$TEMP_GLOB")
|
||||
if [[ -n ${1:-} ]]; then args+=(--min-age "$1"); fi
|
||||
if ((${2:-0})); then args+=(--delete-empty-src-dirs); fi
|
||||
rclone move "$staging" "$remote" "${args[@]}" || rc=$?
|
||||
mkdir -p "$staging"
|
||||
return $rc
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ -n $tdl_pid ]] && kill -0 "$tdl_pid" 2>/dev/null; then
|
||||
log "stopping tdl (pid $tdl_pid)"
|
||||
kill -TERM "$tdl_pid" 2>/dev/null || true
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||
kill -0 "$tdl_pid" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
kill -KILL "$tdl_pid" 2>/dev/null || true
|
||||
fi
|
||||
if ((sweep_ok)); then
|
||||
log 'sweeping completed files before exit'
|
||||
sweep "$min_age" || log 'warning: final safety sweep failed; staging kept'
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'log "interrupted (SIGINT)"; exit 130' INT
|
||||
trap 'log "terminated (SIGTERM)"; exit 143' TERM
|
||||
|
||||
log "downloading into $staging, moving to $remote every ${interval}s"
|
||||
tdl dl -f "$export_file" -d "$staging" \
|
||||
--takeout --group --skip-same --continue ${tdl_extra[@]+"${tdl_extra[@]}"} &
|
||||
tdl_pid=$!
|
||||
sweep_ok=1
|
||||
|
||||
failures=0
|
||||
while kill -0 "$tdl_pid" 2>/dev/null; do
|
||||
# sleep as a job so signals are handled without waiting out the interval
|
||||
sleep "$interval" &
|
||||
wait $! 2>/dev/null || true
|
||||
|
||||
if sweep "$min_age"; then
|
||||
failures=0
|
||||
else
|
||||
failures=$((failures + 1))
|
||||
log "warning: rclone sweep failed ($failures/$max_sync_failures)"
|
||||
if ((failures >= max_sync_failures)); then
|
||||
sweep_ok=0
|
||||
fail "rclone failed $failures times in a row; stopping before staging fills the disk"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
wait "$tdl_pid" || tdl_rc=$?
|
||||
tdl_pid=''
|
||||
sweep_ok=0
|
||||
|
||||
if ((tdl_rc != 0)); then
|
||||
log "tdl exited $tdl_rc; staging kept at $staging — re-run to resume"
|
||||
sweep "$min_age" || log 'warning: sweep after failure did not complete'
|
||||
exit "$tdl_rc"
|
||||
fi
|
||||
|
||||
log 'tdl finished; final sweep'
|
||||
sweep '' 1 || fail "final sweep failed; files remain in $staging"
|
||||
log "done — everything moved to $remote"
|
||||
Reference in new issue
Block a user