diff --git a/.env.example b/.env.example index a1e05a7..9ebb237 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1 @@ -PHONE=+1234567890 -API_ID=123456789 -API_HASH=0123456789abcdef0123456789abcdef -GROUP_IDS=1234567890,9876543210 REDIS_URL=rediss://default:@: -REDIS_PREFIX=telegram-export diff --git a/.gitignore b/.gitignore index e226bf7..ae03633 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.session *.csv +plans/ diff --git a/README.md b/README.md index 5f6ca86..dcbec03 100644 --- a/README.md +++ b/README.md @@ -18,14 +18,22 @@ pip install -r requirements.txt 3. Create a new Telegram app at [https://my.telegram.org](https://my.telegram.org) and get the `api_id` and `api_hash`. 4. Create a free Redis database (e.g. [Upstash](https://upstash.com)) and copy its `rediss://` connection URL. -5. Copy `.env.example` to `.env` and fill in `API_ID`, `API_HASH`, `PHONE`, `GROUP_IDS`, and `REDIS_URL`. -6. Crawl configured groups: +5. Copy `.env.example` to `.env` and fill in `REDIS_URL`. +6. Store Telegram config in Redis: + +```bash +python configure.py --api-id --phone --groups , +``` + +`configure.py` prompts for `api_hash` so it does not land in shell history. + +7. Crawl configured groups: ```bash python crawl.py ``` -The first run asks for the Telegram login code once, then stores the session in Redis. Any later run — on any device pointed at the same Redis — reuses that session and **does not** prompt again. +The first crawl asks for the Telegram login code once, then stores the session in Redis. Any later run — on any device pointed at the same Redis — reuses the Redis config and session, and **does not** prompt again. ## Compare two crawls @@ -47,19 +55,19 @@ The output lists members added in `time2` and removed since `time1`. | Variable | Description | |----------|-------------| -| `PHONE` | Telegram account phone (e.g. `+1234567890`) | -| `API_ID` / `API_HASH` | From [my.telegram.org](https://my.telegram.org) | -| `GROUP_IDS` | Comma-separated group IDs to export | | `REDIS_URL` | Redis connection string (`rediss://default:@:`) | -| `REDIS_PREFIX` | Key namespace (default `telegram-export`); isolates keys when sharing Redis with other projects | + +Telegram `api_id`, `api_hash`, `phone`, and `group_ids` are stored in Redis by `configure.py`. ## How data is stored -All data lives in Redis under `REDIS_PREFIX`. No key references another, so deleting any key can never corrupt another's state: +All data lives in Redis under the `telegram-export` prefix. No key references another, so deleting any key can never corrupt another's state: ``` -:session: -> StringSession string (login) -:run::: -> one group's export as JSON: +telegram-export:config -> JSON config: + { api_id, api_hash, phone, group_ids } +telegram-export:session -> StringSession string (login) +telegram-export:run:: -> one group's export as JSON: { group_id, title, time, members: [{ id, username, first_name, last_name }] } ``` diff --git a/common.py b/common.py index a2cc8ef..30d5cb4 100644 --- a/common.py +++ b/common.py @@ -2,7 +2,7 @@ Each group's members are stored per export as one self-contained JSON key: - :run::: + :run:: -> {group_id, title, time, members:[{id,username,first_name,last_name}]} All groups in a single run share one timestamp. No key references another, so @@ -13,7 +13,7 @@ single atomic SET; listing scans the run keys. import json from datetime import datetime -from config import key, phone, redis_client +from config import key, redis_client def new_run_time(): @@ -39,7 +39,7 @@ def save_group_export(group_id, title, members, run_time): 'members': [member_dict(m) for m in members], } redis_client.set( - key('run', phone, run_time, str(group_id)), + key('run', run_time, str(group_id)), json.dumps(record, ensure_ascii=False), ) @@ -50,7 +50,7 @@ def list_exports(): Tolerates keys deleted mid-scan and corrupt/non-JSON values (skips them). """ records = [] - for export_key in redis_client.scan_iter(match=key('run', phone, '*')): + for export_key in redis_client.scan_iter(match=key('run', '*')): raw = redis_client.get(export_key) if raw is None: # deleted between scan and get continue diff --git a/compare.py b/compare.py index 00cae25..d0b2e2b 100644 --- a/compare.py +++ b/compare.py @@ -1,8 +1,6 @@ import argparse import sys -from common import diff_group_members, get_group_export, latest_two_group_exports - def parse_args(): parser = argparse.ArgumentParser( @@ -23,6 +21,8 @@ def parse_args(): def main(): args = parse_args() + from common import diff_group_members, get_group_export, latest_two_group_exports + if args.times: before_time, after_time = args.times before_record = get_group_export(args.group_id, before_time) @@ -39,12 +39,12 @@ def main(): print(f'need at least 2 exports for group {args.group_id}', file=sys.stderr) return 1 - print_summary(args.group_id, before_record, after_record) + added, removed = diff_group_members(before_record, after_record) + print_summary(args.group_id, before_record, after_record, added, removed) return 0 -def print_summary(group_id, before_record, after_record): - added, removed = diff_group_members(before_record, after_record) +def print_summary(group_id, before_record, after_record, added, removed): title = after_record.get('title') or before_record.get('title') or '' before_members = before_record.get('members', []) after_members = after_record.get('members', []) diff --git a/config.py b/config.py index d0fec37..5712150 100644 --- a/config.py +++ b/config.py @@ -1,27 +1,60 @@ import os +import json import redis from dotenv import load_dotenv load_dotenv() -phone = os.getenv('PHONE') -api_id = os.getenv('API_ID') -api_hash = os.getenv('API_HASH') -group_ids_str = os.getenv('GROUP_IDS', '') -group_ids = [int(id.strip()) for id in group_ids_str.split(',') if id.strip()] if group_ids_str else [] - # Shared Redis holds the Telegram session + export history so the tool runs on -# any device from just .env. REDIS_PREFIX namespaces every key so this project -# never collides with others sharing the same Redis instance. +# any device from just .env. The prefix is fixed to keep local setup minimal. redis_url = os.getenv('REDIS_URL') if not redis_url: raise SystemExit('REDIS_URL not set in .env (e.g. rediss://default:@:)') -redis_prefix = os.getenv('REDIS_PREFIX', 'telegram-export') +redis_prefix = 'telegram-export' redis_client = redis.from_url(redis_url, decode_responses=True) def key(*parts): """Build a namespaced Redis key. Single source of the prefix (DRY).""" return ':'.join([redis_prefix, *parts]) + + +def load_app_config(): + """Load Telegram crawl config from Redis.""" + raw = redis_client.get(key('config')) + if not raw: + raise SystemExit( + 'Telegram config not found in Redis. Run configure.py first.' + ) + try: + config = json.loads(raw) + except ValueError as exc: + raise SystemExit('Telegram config in Redis is not valid JSON.') from exc + + missing = [ + name for name in ('api_id', 'api_hash', 'phone', 'group_ids') + if not config.get(name) + ] + if missing: + raise SystemExit(f'Telegram config missing: {", ".join(missing)}') + + return { + 'api_id': int(config['api_id']), + 'api_hash': str(config['api_hash']), + 'phone': str(config['phone']), + 'group_ids': [int(group_id) for group_id in config['group_ids']], + } + + +def save_app_config(api_id, api_hash, phone, group_ids): + """Save Telegram crawl config to Redis.""" + record = { + 'api_id': int(api_id), + 'api_hash': str(api_hash), + 'phone': str(phone), + 'group_ids': [int(group_id) for group_id in group_ids], + } + redis_client.set(key('config'), json.dumps(record, ensure_ascii=False)) + return record diff --git a/configure.py b/configure.py new file mode 100644 index 0000000..292431e --- /dev/null +++ b/configure.py @@ -0,0 +1,42 @@ +import argparse +import getpass + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Store Telegram crawl config in Redis.', + ) + parser.add_argument('--api-id', required=True, type=int, help='Telegram API ID') + parser.add_argument('--api-hash', help='Telegram API hash') + parser.add_argument('--phone', required=True, help='Telegram account phone') + parser.add_argument( + '--groups', + required=True, + help='Comma-separated Telegram group IDs to crawl', + ) + return parser.parse_args() + + +def parse_group_ids(value): + group_ids = [part.strip() for part in value.split(',') if part.strip()] + if not group_ids: + raise SystemExit('at least one group id is required') + return [int(group_id) for group_id in group_ids] + + +def main(): + args = parse_args() + group_ids = parse_group_ids(args.groups) + api_hash = args.api_hash or getpass.getpass('API hash: ') + if not api_hash: + raise SystemExit('api hash is required') + + from config import save_app_config + + config = save_app_config(args.api_id, api_hash, args.phone, group_ids) + print(f'saved Redis config for {config["phone"]}: {len(group_ids)} group(s).') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/crawl.py b/crawl.py index c850caa..f7f0609 100644 --- a/crawl.py +++ b/crawl.py @@ -6,38 +6,51 @@ from telethon.sync import TelegramClient from telethon.tl.types import Chat, Channel from common import new_run_time, save_group_export -from config import api_hash, api_id, group_ids, phone +from config import load_app_config from session_store import load_session, save_session -# Session loads from Redis: a saved string means no re-login on any device. -client = TelegramClient(StringSession(load_session()), api_id, api_hash) -client.connect() -if not client.is_user_authorized(): - client.send_code_request(phone) - try: - client.sign_in(code=input('Enter code: ')) - except SessionPasswordNeededError: - client.sign_in(password=getpass.getpass()) +def main(): + app_config = load_app_config() + api_hash = app_config['api_hash'] + api_id = app_config['api_id'] + group_ids = app_config['group_ids'] + phone = app_config['phone'] -client.start(phone) -save_session(client.session.save()) + # Session loads from Redis: a saved string means no re-login on any device. + client = TelegramClient(StringSession(load_session()), api_id, api_hash) -run_time = new_run_time() -saved = 0 -for group_id in group_ids: - try: - entity = client.get_entity(group_id) - if not isinstance(entity, (Chat, Channel)): - print(f'skip {group_id} because it is not a group.') - continue - print(f'exporting {entity.title} (ID: {group_id})') - members = client.get_participants(entity) - members.sort(key=lambda x: x.id) - save_group_export(group_id, entity.title, members, run_time) - saved += 1 - print(f'export {entity.title} done.') - except Exception as e: - print(f'error accessing group {group_id}: {e}') + client.connect() + if not client.is_user_authorized(): + client.send_code_request(phone) + try: + client.sign_in(code=input('Enter code: ')) + except SessionPasswordNeededError: + client.sign_in(password=getpass.getpass()) -print(f'saved run {run_time}: {saved} group(s) to Redis.') + client.start(phone) + save_session(client.session.save()) + + run_time = new_run_time() + saved = 0 + for group_id in group_ids: + try: + entity = client.get_entity(group_id) + if not isinstance(entity, (Chat, Channel)): + print(f'skip {group_id} because it is not a group.') + continue + print(f'exporting {entity.title} (ID: {group_id})') + members = client.get_participants(entity) + members.sort(key=lambda x: x.id) + save_group_export(group_id, entity.title, members, run_time) + saved += 1 + print(f'export {entity.title} done.') + except Exception as e: + print(f'error accessing group {group_id}: {e}') + + print(f'saved run {run_time}: {saved} group(s) to Redis.') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/migrate_local_to_redis.py b/migrate_local_to_redis.py deleted file mode 100644 index 0ddfdc7..0000000 --- a/migrate_local_to_redis.py +++ /dev/null @@ -1,86 +0,0 @@ -"""One-off migration: import old local CSV export folders into Redis. - -Old layout : '/.csv' -New key : :run::: - -> {group_id, title, time, members:[{id,username,first_name,last_name}]} - -Idempotent: re-running overwrites the same keys. CSV filenames carry sanitized -titles (special chars stripped), so we map by exact filename stem to the real -title + group_id. Empty CSV fields are normalized to None to match live exports. -""" - -import csv -import glob -import json -import os -import sys -from datetime import datetime - -# Vietnamese group titles need UTF-8 stdout (Windows console defaults to cp1252). -sys.stdout.reconfigure(encoding='utf-8') - -from config import key, phone, redis_client - -# Sanitized CSV filename stem -> (group_id, real title) -GROUP_MAP = { - 'ZingPlay Game Studios': (-1001480682135, 'ZingPlay Game Studios'), - 'ZPS HCM': (-230962353, 'ZPS HCM'), - 'ZPS HCM - Xin nghỉ (phépđi trễ)': (-1001660824205, 'ZPS HCM - Xin nghỉ (phép/đi trễ)'), -} - -FOLDER_GLOB = '20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]-[0-9][0-9]-[0-9][0-9]' - - -def _norm(value): - """CSV stores missing fields as ''; live exports use None. Normalize to None.""" - return value if value else None - - -def parse_members(csv_path): - members = [] - with open(csv_path, encoding='UTF-8', newline='') as f: - reader = csv.DictReader(f) - for row in reader: - members.append({ - 'id': int(row['id']), - 'username': _norm(row['username']), - 'first_name': _norm(row['first_name']), - 'last_name': _norm(row['last_name']), - }) - return members - - -def main(): - folders = sorted(glob.glob(FOLDER_GLOB)) - keys_written = 0 - members_total = 0 - skipped = [] - - for folder in folders: - run_time = datetime.strptime(folder, '%Y-%m-%d %H-%M-%S').strftime('%Y%m%d%H%M%S') - for csv_path in sorted(glob.glob(os.path.join(folder, '*.csv'))): - stem = os.path.splitext(os.path.basename(csv_path))[0] - if stem not in GROUP_MAP: - skipped.append(csv_path) - continue - group_id, title = GROUP_MAP[stem] - members = parse_members(csv_path) - record = {'group_id': group_id, 'title': title, 'time': run_time, 'members': members} - redis_client.set( - key('run', phone, run_time, str(group_id)), - json.dumps(record, ensure_ascii=False), - ) - keys_written += 1 - members_total += len(members) - print(f'{run_time} {group_id} {title}: {len(members)} members') - - print(f'\nmigrated {keys_written} key(s) from {len(folders)} folder(s); ' - f'{members_total} member rows total.') - if skipped: - print(f'skipped {len(skipped)} unmapped CSV(s):') - for path in skipped: - print(f' {path}') - - -if __name__ == '__main__': - main() diff --git a/plans/260609-compare-group-export-diffs/plan.md b/plans/260609-compare-group-export-diffs/plan.md deleted file mode 100644 index 0c8a7ce..0000000 --- a/plans/260609-compare-group-export-diffs/plan.md +++ /dev/null @@ -1,33 +0,0 @@ -# Compare Group Export Diffs - -## Context - -- `common.py` stores and lists Redis export records. -- `crawl.py` exports groups and should not be reused for comparison because it logs into Telegram. -- `README.md` documents current usage and Redis record shape. - -## Requirements - -- Add command that receives `group_id`, optional `time1`, optional `time2`. -- If both times missing, compare latest 2 exports for that group. -- If one time missing, fail with clear usage error. -- Print members added and removed between two crawls. -- Keep Redis schema unchanged. - -## Implementation - -1. Add focused helpers in `common.py` to filter exports by group/time and compare members by Telegram `id`. -2. Add `compare.py` CLI with argparse. -3. Update `README.md` with command examples. -4. Run Python compile check. - -## Success Criteria - -- `python compare.py ` compares latest two times. -- `python compare.py ` compares explicit times. -- Missing export records produce readable errors. -- Existing export flow remains unchanged. - -## Unresolved Questions - -None. diff --git a/session_store.py b/session_store.py index 8cb3014..9c1f20e 100644 --- a/session_store.py +++ b/session_store.py @@ -4,9 +4,9 @@ The session string grants full account access; it lives only in Redis (behind TLS/auth), never on local disk. Deleting the key simply forces a clean re-login. """ -from config import key, phone, redis_client +from config import key, redis_client -_SESSION_KEY = key('session', phone) +_SESSION_KEY = key('session') def load_session():