From 7d20f5667cd248b0ec2303130b05d210529839b7 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Tue, 25 Aug 2026 15:48:46 +0700 Subject: [PATCH] feat: add a tdl-to-webdav rolling pipeline script tdl has no remote destination, so exporting a group larger than local disk needs a staging dir plus a concurrent uploader. The script runs tdl into a small staging dir while rclone moves finished files out to WebDAV, so local disk only holds the files in flight plus one sync interval of throughput. Partial downloads are excluded by name rather than by age: tdl writes .tmp and renames on completion, so a download stalled by a flood wait stops touching its .tmp and would otherwise age past --min-age and be uploaded half-written, destroying resume for that file. --delete-empty-src-dirs runs only in the final sweep, because removing a directory under a running tdl makes it fail to create its next file. tdl is stopped on exit, interrupt or termination so no download is orphaned, the unrestricted sweep runs only after tdl exits 0, and five consecutive rclone failures abort the run before staging fills the disk. --- tg-export-webdav.sh | 170 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100755 tg-export-webdav.sh diff --git a/tg-export-webdav.sh b/tg-export-webdav.sh new file mode 100755 index 0000000..1c4c302 --- /dev/null +++ b/tg-export-webdav.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# +# Rolling pipeline: tdl downloads Telegram media into a small staging directory +# while rclone concurrently moves finished files to a WebDAV remote. Local disk +# only ever holds the files in flight plus one sync interval of throughput, so a +# group larger than the local disk can still be exported. +# +# See README.md ("Special case: exporting to WebDAV") for the background. +# +# Exit codes: 0 ok, 2 usage error, 3 rclone failure, 130/143 interrupted, +# anything else is tdl's own exit code. + +set -euo pipefail + +readonly PROG=${0##*/} + +# tdl downloads to '.tmp' and renames only on completion, so an unfinished +# file is always identifiable by extension. Never move one: a download stalled +# by a flood wait stops touching its .tmp, which then ages past --min-age and +# would be uploaded half-written, destroying tdl's resume point for that file. +readonly TEMP_GLOB='*.tmp' + +# Defaults +export_file='export.json' +chat='' +staging='./staging' +remote='' +interval=60 +min_age='2m' +max_sync_failures=5 + +usage() { + cat <&2; } +die() { log "error: $*"; exit 2; } # usage or precondition +fail() { log "error: $*"; exit 3; } # rclone / pipeline failure + +while getopts ':f:c:d:r:i:a:h' opt; do + case $opt in + f) export_file=$OPTARG ;; + c) chat=$OPTARG ;; + d) staging=$OPTARG ;; + r) remote=$OPTARG ;; + i) interval=$OPTARG ;; + a) min_age=$OPTARG ;; + h) usage; exit 0 ;; + :) die "option -$OPTARG requires an argument" ;; + ?) die "unknown option -$OPTARG (try -h)" ;; + esac +done +shift $((OPTIND - 1)) +tdl_extra=("$@") + +[[ -n $remote ]] || { usage >&2; die "-r REMOTE:PATH is required"; } +[[ $remote == *:* ]] || die "remote '$remote' is not in rclone REMOTE:PATH form" +[[ $interval =~ ^[0-9]+$ && $interval -gt 0 ]] || die "-i must be a positive integer" +[[ $min_age =~ ^[0-9]+(\.[0-9]+)?(ms|s|m|h|d|w|M|y)?$ ]] \ + || die "-a must be an rclone duration, e.g. 2m" + +for tool in tdl rclone; do + command -v "$tool" >/dev/null || die "$tool is not installed or not on PATH" +done + +rclone lsd "${remote%%:*}:" >/dev/null 2>&1 \ + || die "rclone cannot reach remote '${remote%%:*}:' — check 'rclone config'" + +# The export JSON only lists messages; it is cheap to keep and required for both +# legs to stay resumable, so never regenerate it when it already exists. +if [[ ! -f $export_file ]]; then + [[ -n $chat ]] || die "$export_file not found; pass -c CHAT to export it first" + log "exporting $chat metadata to $export_file" + tdl chat export -c "$chat" --all --with-content -o "$export_file" +fi + +mkdir -p "$staging" + +tdl_pid='' +tdl_rc=0 +sweep_ok=0 + +# Move whatever is finished. Partial .tmp files are always excluded. $1 is an +# optional --min-age guard; $2 enables --delete-empty-src-dirs, which is safe +# only once tdl has stopped — rclone removing a directory between tdl's +# MkdirAll and Create makes tdl fail, and it can take the staging root too, +# hence the mkdir afterwards. +sweep() { + local rc=0 + local args=(--exclude "$TEMP_GLOB") + if [[ -n ${1:-} ]]; then args+=(--min-age "$1"); fi + if ((${2:-0})); then args+=(--delete-empty-src-dirs); fi + rclone move "$staging" "$remote" "${args[@]}" || rc=$? + mkdir -p "$staging" + return $rc +} + +cleanup() { + if [[ -n $tdl_pid ]] && kill -0 "$tdl_pid" 2>/dev/null; then + log "stopping tdl (pid $tdl_pid)" + kill -TERM "$tdl_pid" 2>/dev/null || true + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "$tdl_pid" 2>/dev/null || break + sleep 1 + done + kill -KILL "$tdl_pid" 2>/dev/null || true + fi + if ((sweep_ok)); then + log 'sweeping completed files before exit' + sweep "$min_age" || log 'warning: final safety sweep failed; staging kept' + fi +} +trap cleanup EXIT +trap 'log "interrupted (SIGINT)"; exit 130' INT +trap 'log "terminated (SIGTERM)"; exit 143' TERM + +log "downloading into $staging, moving to $remote every ${interval}s" +tdl dl -f "$export_file" -d "$staging" \ + --takeout --group --skip-same --continue ${tdl_extra[@]+"${tdl_extra[@]}"} & +tdl_pid=$! +sweep_ok=1 + +failures=0 +while kill -0 "$tdl_pid" 2>/dev/null; do + # sleep as a job so signals are handled without waiting out the interval + sleep "$interval" & + wait $! 2>/dev/null || true + + if sweep "$min_age"; then + failures=0 + else + failures=$((failures + 1)) + log "warning: rclone sweep failed ($failures/$max_sync_failures)" + if ((failures >= max_sync_failures)); then + sweep_ok=0 + fail "rclone failed $failures times in a row; stopping before staging fills the disk" + fi + fi +done + +wait "$tdl_pid" || tdl_rc=$? +tdl_pid='' +sweep_ok=0 + +if ((tdl_rc != 0)); then + log "tdl exited $tdl_rc; staging kept at $staging — re-run to resume" + sweep "$min_age" || log 'warning: sweep after failure did not complete' + exit "$tdl_rc" +fi + +log 'tdl finished; final sweep' +sweep '' 1 || fail "final sweep failed; files remain in $staging" +log "done — everything moved to $remote"