mirror of
https://github.com/tiennm99/telegram-exporter.git
synced 2026-10-12 18:16:49 +00:00
tiennm99
70394b9763
The verifier reconstructs {DialogID}_{MessageID}_{FileName} from the export
and looks it up in the remote listing. A file stored under any other name is
not the file the export asked for, so it counts as absent however close the
name looks.
That leaves a stale copy behind when the re-download lands beside it, so
absent files whose message id is already present on the remote are now listed
separately with both names, ready to be removed.
Remote paths are reduced to their basename first, so a grouped subdirectory
layout matches the same way a flat one does.
115 lines
4.5 KiB
Bash
Executable File
115 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
#
|
||
# Verify a tdl/rclone export is complete and every file is intact.
|
||
#
|
||
# Rebuilds the exact filename tdl produces for each media message in the export
|
||
# JSON ({DialogID}_{MessageID}_{FileName}) and checks it exists on the remote or
|
||
# in staging. The export is Telegram's own record of the name, so the match is
|
||
# exact: a file stored under any other name is not the file the export asked
|
||
# for and counts as absent, however close the name looks. Zero-byte files count
|
||
# as missing too: rclone overwrites a size-mismatched destination, so re-running
|
||
# repairs them. Files under 1 KiB are reported for inspection but trusted, since
|
||
# some real media is genuinely that small.
|
||
#
|
||
# A remote file whose id matches but whose name does not is a stale copy from an
|
||
# earlier download; it is listed separately so it can be deleted, because the
|
||
# re-download lands beside it rather than replacing it.
|
||
#
|
||
# Writes every id needing another attempt to missing-ids.txt.
|
||
# Exit 0 = complete, 1 = incomplete, 2 = usage error.
|
||
#
|
||
# Usage: ./verify-export.sh -f export-<chat>.json -r REMOTE:PATH [-d STAGING]
|
||
|
||
set -euo pipefail
|
||
|
||
export_file='' remote='' staging='./staging'
|
||
while getopts ':f:r:d:h' o; do case $o in
|
||
f) export_file=$OPTARG ;; r) remote=$OPTARG ;; d) staging=$OPTARG ;;
|
||
h) sed -n '2,16p' "$0"; exit 0 ;;
|
||
*) echo "usage: $0 -f FILE -r REMOTE:PATH [-d STAGING]" >&2; exit 2 ;;
|
||
esac; done
|
||
|
||
[[ -f $export_file ]] || { echo "no such export file: $export_file" >&2; exit 2; }
|
||
[[ -n $remote ]] || { echo "-r REMOTE:PATH is required" >&2; exit 2; }
|
||
|
||
listing=$(mktemp); trap 'rm -f "$listing"' EXIT
|
||
# lsl gives sizes as well as names, so truncated uploads are detectable.
|
||
rclone lsl "$remote" > "$listing"
|
||
|
||
python3 - "$export_file" "$listing" "$staging" <<'PY'
|
||
import json, os, re, sys
|
||
export_file, listing, staging = sys.argv[1], sys.argv[2], sys.argv[3]
|
||
|
||
sizes = {}
|
||
for line in open(listing):
|
||
m = re.match(r'^\s*(\d+)\s+\S+\s+\S+\s+(.*)$', line.rstrip('\n'))
|
||
if m:
|
||
sizes[os.path.basename(m.group(2))] = int(m.group(1))
|
||
if os.path.isdir(staging):
|
||
for f in os.listdir(staging):
|
||
if not f.endswith('.tmp'):
|
||
sizes.setdefault(f, os.path.getsize(os.path.join(staging, f)))
|
||
|
||
msgs = json.load(open(export_file))['messages']
|
||
# Text-only messages carry an empty "file" and are not download targets.
|
||
media = [m for m in msgs if m.get('file')]
|
||
|
||
dialog = str(json.load(open(export_file)).get('id', '')).lstrip('-')
|
||
if not dialog.isdigit():
|
||
ids = {n.split('_')[0] for n in sizes if '_' in n}
|
||
dialog = ids.pop() if len(ids) == 1 else ''
|
||
if not dialog:
|
||
sys.exit('cannot determine dialog id')
|
||
|
||
# Names already stored for each message id, used only to tell an absent file
|
||
# apart from one sitting there under the wrong name.
|
||
stored = {}
|
||
for name in sizes:
|
||
parts = name.split('_', 2)
|
||
if len(parts) == 3 and parts[0] == dialog and parts[1].isdigit():
|
||
stored.setdefault(int(parts[1]), []).append(name)
|
||
|
||
absent, empty, tiny, misnamed = [], [], [], []
|
||
for m in media:
|
||
name = f"{dialog}_{m['id']}_{m['file']}"
|
||
if name not in sizes:
|
||
absent.append(m['id'])
|
||
for other in stored.get(m['id'], []):
|
||
misnamed.append((m['id'], name, other, sizes[other]))
|
||
elif sizes[name] == 0:
|
||
empty.append(m['id'])
|
||
elif sizes[name] < 1024:
|
||
tiny.append((m['id'], sizes[name], name))
|
||
|
||
todo = sorted(absent + empty)
|
||
print(f"messages in export : {len(msgs)}")
|
||
print(f" text-only (skip) : {len(msgs) - len(media)}")
|
||
print(f" media expected : {len(media)}")
|
||
print(f"present and intact : {len(media) - len(todo)}")
|
||
print(f" absent : {len(absent)}")
|
||
print(f" zero-byte : {len(empty)}")
|
||
if tiny:
|
||
print(f" under 1KiB (check, not retried): {len(tiny)}")
|
||
for i, s, n in tiny[:5]:
|
||
print(f" id {i} {s} B {n}")
|
||
if misnamed:
|
||
print(f"\nstored under a different name : {len(misnamed)}")
|
||
print(" counted as absent and fetched again; delete the stale copies so the")
|
||
print(" re-download does not leave two files for the same message:")
|
||
for i, want, got, size in misnamed:
|
||
print(f" id {i} {size} B")
|
||
print(f" export: {want}")
|
||
print(f" remote: {got}")
|
||
|
||
if todo:
|
||
with open('missing-ids.txt', 'w') as fh:
|
||
fh.write('\n'.join(map(str, todo)) + '\n')
|
||
print(f"\nneeds another pass : {len(todo)} (ids {min(todo)}–{max(todo)})")
|
||
print("written to missing-ids.txt")
|
||
sys.exit(1)
|
||
|
||
if os.path.exists('missing-ids.txt'):
|
||
os.remove('missing-ids.txt')
|
||
print("\nCOMPLETE: every media message is present and non-empty.")
|
||
PY
|