From 2c820a07a365afd3c4987da35c9a6a979a921563 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Thu, 6 Aug 2026 20:18:18 +0700 Subject: [PATCH] feat(crawl): use configured group ids from redis --- README.md | 23 ++++++++--------------- crawl.py | 21 +++------------------ tests/test_crawl.py | 36 +++--------------------------------- 3 files changed, 14 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index be6c73a..5528359 100644 --- a/README.md +++ b/README.md @@ -25,26 +25,19 @@ pip install -r requirements.txt python configure.py ``` -`configure.py` prompts for `api_id`, `api_hash`, phone, and group IDs. The stored -group IDs remain available to commands that use configured defaults, such as -`compare.py`. +`configure.py` prompts for `api_id`, `api_hash`, phone, and group IDs. All +commands read the stored group IDs from this Redis config. -7. Crawl one or more groups by ID: +7. Crawl the configured groups: ```bash -python crawl.py [ ...] +python crawl.py ``` -For example: - -```bash -python crawl.py -1001234567890 -1009876543210 -``` - -Only the group IDs supplied to this command are crawled; the stored group list -is not used. The first crawl asks for the Telegram login code once, then stores -the session in Redis. Any later run — on any device pointed at the same Redis — -reuses the Redis config and session, and **does not** prompt again. +Every group ID stored in the Redis config is crawled. To change the list, run +`configure.py` again. The first crawl asks for the Telegram login code once, +then stores the session in Redis. Any later run — on any device pointed at the +same Redis — reuses the Redis config and session, and **does not** prompt again. ## Compare two crawls diff --git a/crawl.py b/crawl.py index 51750ac..16b7638 100644 --- a/crawl.py +++ b/crawl.py @@ -1,4 +1,3 @@ -import argparse import getpass from telethon.errors import SessionPasswordNeededError @@ -11,26 +10,12 @@ from config import load_app_config from session_store import load_session, save_session -def parse_args(argv=None): - parser = argparse.ArgumentParser( - description='Export members from one or more Telegram groups.', - ) - parser.add_argument( - 'group_ids', - nargs='+', - type=int, - metavar='group_id', - help='Telegram group ID to crawl (provide one or more)', - ) - return parser.parse_args(argv) - - -def main(argv=None): - args = parse_args(argv) +def main(): app_config = load_app_config() api_hash = app_config['api_hash'] api_id = app_config['api_id'] phone = app_config['phone'] + group_ids = app_config['group_ids'] # Session loads from Redis: a saved string means no re-login on any device. client = TelegramClient(StringSession(load_session()), api_id, api_hash) @@ -47,7 +32,7 @@ def main(argv=None): run_time = new_run_time() saved = 0 - for group_id in args.group_ids: + for group_id in group_ids: try: entity = client.get_entity(group_id) if not isinstance(entity, (Chat, Channel)): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 438b9ad..11a2802 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,42 +18,13 @@ class FakeMember: self.id = member_id -class CrawlArgumentsTest(unittest.TestCase): - def test_accepts_multiple_group_ids(self): - args = crawl.parse_args(['-100123', '-100456']) - - self.assertEqual(args.group_ids, [-100123, -100456]) - - def test_requires_at_least_one_group_id(self): - with ( - patch.object(crawl, 'load_app_config') as load_app_config, - patch.object(crawl, 'TelegramClient') as telegram_client, - self.assertRaises(SystemExit), - ): - crawl.main([]) - - load_app_config.assert_not_called() - telegram_client.assert_not_called() - - def test_rejects_non_integer_group_id(self): - with ( - patch.object(crawl, 'load_app_config') as load_app_config, - patch.object(crawl, 'TelegramClient') as telegram_client, - self.assertRaises(SystemExit), - ): - crawl.main(['not-a-group']) - - load_app_config.assert_not_called() - telegram_client.assert_not_called() - - class CrawlMainTest(unittest.TestCase): @patch.object(crawl, 'save_group_export') @patch.object(crawl, 'save_session') @patch.object(crawl, 'load_session', return_value='stored-session') @patch.object(crawl, 'load_app_config') @patch.object(crawl, 'TelegramClient') - def test_crawls_supplied_ids_instead_of_configured_ids( + def test_crawls_configured_group_ids( self, telegram_client, load_app_config, @@ -65,7 +36,7 @@ class CrawlMainTest(unittest.TestCase): 'api_id': 123, 'api_hash': 'hash', 'phone': '+15551234567', - 'group_ids': [999], + 'group_ids': [101, 202], } client = MagicMock() telegram_client.return_value = client @@ -83,11 +54,10 @@ class CrawlMainTest(unittest.TestCase): patch.object(crawl, 'StringSession', return_value='session'), patch.object(crawl, 'new_run_time', return_value='20260724120000'), ): - result = crawl.main(['101', '202']) + result = crawl.main() self.assertEqual(result, 0) self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)]) - self.assertNotIn(call(999), client.get_entity.call_args_list) save_session.assert_called_once_with('updated-session') self.assertEqual(save_group_export.call_count, 2) first_members = save_group_export.call_args_list[0].args[2]