feat(crawl): accept group ids as arguments for one-off crawls

Command-line group ids replace the configured Redis list for that run only; with no arguments the configured list is used as before.
This commit is contained in:
tiennm99 committed 2026-10-06 13:27:57 +07:00
1 parent 3344e6b2d1
commit 35f61f4e97
3 files changed
+95 -5

No files matched your search

+11 -2
View File
@@ -35,8 +35,17 @@ python crawl.py
``` ```
Every group ID stored in the Redis config is crawled. To change the list, run Every group ID stored in the Redis config is crawled. To change the list, run
`configure.py` again. The first crawl asks for the Telegram login code once, `configure.py` again.
then stores the session in Redis. Any later run — on any device pointed at the
To crawl other groups just once, pass their IDs; they replace the configured
list for that run only:
```bash
python crawl.py -1001234567890 -1009876543210
```
The first crawl asks for the Telegram login code once, then stores the session
in Redis. Any later run — on any device pointed at the
same Redis — reuses the Redis config and session, and **does not** prompt again. same Redis — reuses the Redis config and session, and **does not** prompt again.
## Compare two crawls ## Compare two crawls
+19 -2
View File
@@ -1,3 +1,4 @@
import argparse
import getpass import getpass
from telethon.errors import SessionPasswordNeededError from telethon.errors import SessionPasswordNeededError
@@ -10,12 +11,28 @@ from config import load_app_config
from session_store import load_session, save_session from session_store import load_session, save_session
def main(): def parse_args(argv=None):
parser = argparse.ArgumentParser(
description='Export members from Telegram groups.',
)
parser.add_argument(
'group_ids',
nargs='*',
type=int,
metavar='group_id',
help='Telegram group ID to crawl; defaults to the configured group IDs',
)
return parser.parse_args(argv)
def main(argv=None):
args = parse_args(argv)
app_config = load_app_config() app_config = load_app_config()
api_hash = app_config['api_hash'] api_hash = app_config['api_hash']
api_id = app_config['api_id'] api_id = app_config['api_id']
phone = app_config['phone'] phone = app_config['phone']
group_ids = app_config['group_ids'] # Command-line IDs override the configured list for one-off crawls.
group_ids = args.group_ids or app_config['group_ids']
# Session loads from Redis: a saved string means no re-login on any device. # Session loads from Redis: a saved string means no re-login on any device.
client = TelegramClient(StringSession(load_session()), api_id, api_hash) client = TelegramClient(StringSession(load_session()), api_id, api_hash)
+65 -1
View File
@@ -18,6 +18,29 @@ class FakeMember:
self.id = member_id self.id = member_id
class CrawlArgumentsTest(unittest.TestCase):
def test_accepts_multiple_group_ids(self):
args = crawl.parse_args(['-100123', '-100456'])
self.assertEqual(args.group_ids, [-100123, -100456])
def test_allows_no_group_ids(self):
args = crawl.parse_args([])
self.assertEqual(args.group_ids, [])
def test_rejects_non_integer_group_id(self):
with (
patch.object(crawl, 'load_app_config') as load_app_config,
patch.object(crawl, 'TelegramClient') as telegram_client,
self.assertRaises(SystemExit),
):
crawl.main(['not-a-group'])
load_app_config.assert_not_called()
telegram_client.assert_not_called()
class CrawlMainTest(unittest.TestCase): class CrawlMainTest(unittest.TestCase):
@patch.object(crawl, 'save_group_export') @patch.object(crawl, 'save_group_export')
@patch.object(crawl, 'save_session') @patch.object(crawl, 'save_session')
@@ -54,7 +77,7 @@ class CrawlMainTest(unittest.TestCase):
patch.object(crawl, 'StringSession', return_value='session'), patch.object(crawl, 'StringSession', return_value='session'),
patch.object(crawl, 'new_run_time', return_value='20260724120000'), patch.object(crawl, 'new_run_time', return_value='20260724120000'),
): ):
result = crawl.main() result = crawl.main([])
self.assertEqual(result, 0) self.assertEqual(result, 0)
self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)]) self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)])
@@ -63,6 +86,47 @@ class CrawlMainTest(unittest.TestCase):
first_members = save_group_export.call_args_list[0].args[2] first_members = save_group_export.call_args_list[0].args[2]
self.assertEqual([member.id for member in first_members], [1, 2]) self.assertEqual([member.id for member in first_members], [1, 2])
@patch.object(crawl, 'save_group_export')
@patch.object(crawl, 'save_session')
@patch.object(crawl, 'load_session', return_value='stored-session')
@patch.object(crawl, 'load_app_config')
@patch.object(crawl, 'TelegramClient')
def test_supplied_ids_override_configured_ids(
self,
telegram_client,
load_app_config,
load_session,
save_session,
save_group_export,
):
load_app_config.return_value = {
'api_id': 123,
'api_hash': 'hash',
'phone': '+15551234567',
'group_ids': [999],
}
client = MagicMock()
telegram_client.return_value = client
client.is_user_authorized.return_value = True
client.session.save.return_value = 'updated-session'
client.get_entity.side_effect = [FakeChat(101), FakeChat(202)]
client.get_participants.side_effect = [
[FakeMember(2), FakeMember(1)],
[FakeMember(4), FakeMember(3)],
]
with (
patch.object(crawl, 'Chat', FakeChat),
patch.object(crawl, 'Channel', FakeChat),
patch.object(crawl, 'StringSession', return_value='session'),
patch.object(crawl, 'new_run_time', return_value='20260724120000'),
):
result = crawl.main(['101', '202'])
self.assertEqual(result, 0)
self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)])
self.assertNotIn(call(999), client.get_entity.call_args_list)
if __name__ == '__main__': if __name__ == '__main__':
unittest.main() unittest.main()