diff --git a/tg-export-webdav.sh b/tg-export-webdav.sh new file mode 100755 index 0000000..1c4c302 --- /dev/null +++ b/tg-export-webdav.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# +# Rolling pipeline: tdl downloads Telegram media into a small staging directory +# while rclone concurrently moves finished files to a WebDAV remote. Local disk +# only ever holds the files in flight plus one sync interval of throughput, so a +# group larger than the local disk can still be exported. +# +# See README.md ("Special case: exporting to WebDAV") for the background. +# +# Exit codes: 0 ok, 2 usage error, 3 rclone failure, 130/143 interrupted, +# anything else is tdl's own exit code. + +set -euo pipefail + +readonly PROG=${0##*/} + +# tdl downloads to '.tmp' and renames only on completion, so an unfinished +# file is always identifiable by extension. Never move one: a download stalled +# by a flood wait stops touching its .tmp, which then ages past --min-age and +# would be uploaded half-written, destroying tdl's resume point for that file. +readonly TEMP_GLOB='*.tmp' + +# Defaults +export_file='export.json' +chat='' +staging='./staging' +remote='' +interval=60 +min_age='2m' +max_sync_failures=5 + +usage() { + cat <&2; } +die() { log "error: $*"; exit 2; } # usage or precondition +fail() { log "error: $*"; exit 3; } # rclone / pipeline failure + +while getopts ':f:c:d:r:i:a:h' opt; do + case $opt in + f) export_file=$OPTARG ;; + c) chat=$OPTARG ;; + d) staging=$OPTARG ;; + r) remote=$OPTARG ;; + i) interval=$OPTARG ;; + a) min_age=$OPTARG ;; + h) usage; exit 0 ;; + :) die "option -$OPTARG requires an argument" ;; + ?) die "unknown option -$OPTARG (try -h)" ;; + esac +done +shift $((OPTIND - 1)) +tdl_extra=("$@") + +[[ -n $remote ]] || { usage >&2; die "-r REMOTE:PATH is required"; } +[[ $remote == *:* ]] || die "remote '$remote' is not in rclone REMOTE:PATH form" +[[ $interval =~ ^[0-9]+$ && $interval -gt 0 ]] || die "-i must be a positive integer" +[[ $min_age =~ ^[0-9]+(\.[0-9]+)?(ms|s|m|h|d|w|M|y)?$ ]] \ + || die "-a must be an rclone duration, e.g. 2m" + +for tool in tdl rclone; do + command -v "$tool" >/dev/null || die "$tool is not installed or not on PATH" +done + +rclone lsd "${remote%%:*}:" >/dev/null 2>&1 \ + || die "rclone cannot reach remote '${remote%%:*}:' — check 'rclone config'" + +# The export JSON only lists messages; it is cheap to keep and required for both +# legs to stay resumable, so never regenerate it when it already exists. +if [[ ! -f $export_file ]]; then + [[ -n $chat ]] || die "$export_file not found; pass -c CHAT to export it first" + log "exporting $chat metadata to $export_file" + tdl chat export -c "$chat" --all --with-content -o "$export_file" +fi + +mkdir -p "$staging" + +tdl_pid='' +tdl_rc=0 +sweep_ok=0 + +# Move whatever is finished. Partial .tmp files are always excluded. $1 is an +# optional --min-age guard; $2 enables --delete-empty-src-dirs, which is safe +# only once tdl has stopped — rclone removing a directory between tdl's +# MkdirAll and Create makes tdl fail, and it can take the staging root too, +# hence the mkdir afterwards. +sweep() { + local rc=0 + local args=(--exclude "$TEMP_GLOB") + if [[ -n ${1:-} ]]; then args+=(--min-age "$1"); fi + if ((${2:-0})); then args+=(--delete-empty-src-dirs); fi + rclone move "$staging" "$remote" "${args[@]}" || rc=$? + mkdir -p "$staging" + return $rc +} + +cleanup() { + if [[ -n $tdl_pid ]] && kill -0 "$tdl_pid" 2>/dev/null; then + log "stopping tdl (pid $tdl_pid)" + kill -TERM "$tdl_pid" 2>/dev/null || true + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "$tdl_pid" 2>/dev/null || break + sleep 1 + done + kill -KILL "$tdl_pid" 2>/dev/null || true + fi + if ((sweep_ok)); then + log 'sweeping completed files before exit' + sweep "$min_age" || log 'warning: final safety sweep failed; staging kept' + fi +} +trap cleanup EXIT +trap 'log "interrupted (SIGINT)"; exit 130' INT +trap 'log "terminated (SIGTERM)"; exit 143' TERM + +log "downloading into $staging, moving to $remote every ${interval}s" +tdl dl -f "$export_file" -d "$staging" \ + --takeout --group --skip-same --continue ${tdl_extra[@]+"${tdl_extra[@]}"} & +tdl_pid=$! +sweep_ok=1 + +failures=0 +while kill -0 "$tdl_pid" 2>/dev/null; do + # sleep as a job so signals are handled without waiting out the interval + sleep "$interval" & + wait $! 2>/dev/null || true + + if sweep "$min_age"; then + failures=0 + else + failures=$((failures + 1)) + log "warning: rclone sweep failed ($failures/$max_sync_failures)" + if ((failures >= max_sync_failures)); then + sweep_ok=0 + fail "rclone failed $failures times in a row; stopping before staging fills the disk" + fi + fi +done + +wait "$tdl_pid" || tdl_rc=$? +tdl_pid='' +sweep_ok=0 + +if ((tdl_rc != 0)); then + log "tdl exited $tdl_rc; staging kept at $staging — re-run to resume" + sweep "$min_age" || log 'warning: sweep after failure did not complete' + exit "$tdl_rc" +fi + +log 'tdl finished; final sweep' +sweep '' 1 || fail "final sweep failed; files remain in $staging" +log "done — everything moved to $remote"