mirror of
https://github.com/tiennm99/export-telegram-group-members.git
synced 2026-10-11 03:13:26 +00:00
feat(crawl): accept group ids as arguments for one-off crawls
Command-line group ids replace the configured Redis list for that run only; with no arguments the configured list is used as before.
This commit is contained in:
1 parent
3344e6b2d1
commit
35f61f4e97
3 files changed
+95
-5
No files matched your search
@@ -35,8 +35,17 @@ python crawl.py
|
|||||||
```
|
```
|
||||||
|
|
||||||
Every group ID stored in the Redis config is crawled. To change the list, run
|
Every group ID stored in the Redis config is crawled. To change the list, run
|
||||||
`configure.py` again. The first crawl asks for the Telegram login code once,
|
`configure.py` again.
|
||||||
then stores the session in Redis. Any later run — on any device pointed at the
|
|
||||||
|
To crawl other groups just once, pass their IDs; they replace the configured
|
||||||
|
list for that run only:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python crawl.py -1001234567890 -1009876543210
|
||||||
|
```
|
||||||
|
|
||||||
|
The first crawl asks for the Telegram login code once, then stores the session
|
||||||
|
in Redis. Any later run — on any device pointed at the
|
||||||
same Redis — reuses the Redis config and session, and **does not** prompt again.
|
same Redis — reuses the Redis config and session, and **does not** prompt again.
|
||||||
|
|
||||||
## Compare two crawls
|
## Compare two crawls
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import argparse
|
||||||
import getpass
|
import getpass
|
||||||
|
|
||||||
from telethon.errors import SessionPasswordNeededError
|
from telethon.errors import SessionPasswordNeededError
|
||||||
@@ -10,12 +11,28 @@ from config import load_app_config
|
|||||||
from session_store import load_session, save_session
|
from session_store import load_session, save_session
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def parse_args(argv=None):
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Export members from Telegram groups.',
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
'group_ids',
|
||||||
|
nargs='*',
|
||||||
|
type=int,
|
||||||
|
metavar='group_id',
|
||||||
|
help='Telegram group ID to crawl; defaults to the configured group IDs',
|
||||||
|
)
|
||||||
|
return parser.parse_args(argv)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
args = parse_args(argv)
|
||||||
app_config = load_app_config()
|
app_config = load_app_config()
|
||||||
api_hash = app_config['api_hash']
|
api_hash = app_config['api_hash']
|
||||||
api_id = app_config['api_id']
|
api_id = app_config['api_id']
|
||||||
phone = app_config['phone']
|
phone = app_config['phone']
|
||||||
group_ids = app_config['group_ids']
|
# Command-line IDs override the configured list for one-off crawls.
|
||||||
|
group_ids = args.group_ids or app_config['group_ids']
|
||||||
|
|
||||||
# Session loads from Redis: a saved string means no re-login on any device.
|
# Session loads from Redis: a saved string means no re-login on any device.
|
||||||
client = TelegramClient(StringSession(load_session()), api_id, api_hash)
|
client = TelegramClient(StringSession(load_session()), api_id, api_hash)
|
||||||
|
|||||||
+65
-1
@@ -18,6 +18,29 @@ class FakeMember:
|
|||||||
self.id = member_id
|
self.id = member_id
|
||||||
|
|
||||||
|
|
||||||
|
class CrawlArgumentsTest(unittest.TestCase):
|
||||||
|
def test_accepts_multiple_group_ids(self):
|
||||||
|
args = crawl.parse_args(['-100123', '-100456'])
|
||||||
|
|
||||||
|
self.assertEqual(args.group_ids, [-100123, -100456])
|
||||||
|
|
||||||
|
def test_allows_no_group_ids(self):
|
||||||
|
args = crawl.parse_args([])
|
||||||
|
|
||||||
|
self.assertEqual(args.group_ids, [])
|
||||||
|
|
||||||
|
def test_rejects_non_integer_group_id(self):
|
||||||
|
with (
|
||||||
|
patch.object(crawl, 'load_app_config') as load_app_config,
|
||||||
|
patch.object(crawl, 'TelegramClient') as telegram_client,
|
||||||
|
self.assertRaises(SystemExit),
|
||||||
|
):
|
||||||
|
crawl.main(['not-a-group'])
|
||||||
|
|
||||||
|
load_app_config.assert_not_called()
|
||||||
|
telegram_client.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
class CrawlMainTest(unittest.TestCase):
|
class CrawlMainTest(unittest.TestCase):
|
||||||
@patch.object(crawl, 'save_group_export')
|
@patch.object(crawl, 'save_group_export')
|
||||||
@patch.object(crawl, 'save_session')
|
@patch.object(crawl, 'save_session')
|
||||||
@@ -54,7 +77,7 @@ class CrawlMainTest(unittest.TestCase):
|
|||||||
patch.object(crawl, 'StringSession', return_value='session'),
|
patch.object(crawl, 'StringSession', return_value='session'),
|
||||||
patch.object(crawl, 'new_run_time', return_value='20260724120000'),
|
patch.object(crawl, 'new_run_time', return_value='20260724120000'),
|
||||||
):
|
):
|
||||||
result = crawl.main()
|
result = crawl.main([])
|
||||||
|
|
||||||
self.assertEqual(result, 0)
|
self.assertEqual(result, 0)
|
||||||
self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)])
|
self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)])
|
||||||
@@ -63,6 +86,47 @@ class CrawlMainTest(unittest.TestCase):
|
|||||||
first_members = save_group_export.call_args_list[0].args[2]
|
first_members = save_group_export.call_args_list[0].args[2]
|
||||||
self.assertEqual([member.id for member in first_members], [1, 2])
|
self.assertEqual([member.id for member in first_members], [1, 2])
|
||||||
|
|
||||||
|
@patch.object(crawl, 'save_group_export')
|
||||||
|
@patch.object(crawl, 'save_session')
|
||||||
|
@patch.object(crawl, 'load_session', return_value='stored-session')
|
||||||
|
@patch.object(crawl, 'load_app_config')
|
||||||
|
@patch.object(crawl, 'TelegramClient')
|
||||||
|
def test_supplied_ids_override_configured_ids(
|
||||||
|
self,
|
||||||
|
telegram_client,
|
||||||
|
load_app_config,
|
||||||
|
load_session,
|
||||||
|
save_session,
|
||||||
|
save_group_export,
|
||||||
|
):
|
||||||
|
load_app_config.return_value = {
|
||||||
|
'api_id': 123,
|
||||||
|
'api_hash': 'hash',
|
||||||
|
'phone': '+15551234567',
|
||||||
|
'group_ids': [999],
|
||||||
|
}
|
||||||
|
client = MagicMock()
|
||||||
|
telegram_client.return_value = client
|
||||||
|
client.is_user_authorized.return_value = True
|
||||||
|
client.session.save.return_value = 'updated-session'
|
||||||
|
client.get_entity.side_effect = [FakeChat(101), FakeChat(202)]
|
||||||
|
client.get_participants.side_effect = [
|
||||||
|
[FakeMember(2), FakeMember(1)],
|
||||||
|
[FakeMember(4), FakeMember(3)],
|
||||||
|
]
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(crawl, 'Chat', FakeChat),
|
||||||
|
patch.object(crawl, 'Channel', FakeChat),
|
||||||
|
patch.object(crawl, 'StringSession', return_value='session'),
|
||||||
|
patch.object(crawl, 'new_run_time', return_value='20260724120000'),
|
||||||
|
):
|
||||||
|
result = crawl.main(['101', '202'])
|
||||||
|
|
||||||
|
self.assertEqual(result, 0)
|
||||||
|
self.assertEqual(client.get_entity.call_args_list, [call(101), call(202)])
|
||||||
|
self.assertNotIn(call(999), client.get_entity.call_args_list)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
unittest.main()
|
unittest.main()
|
||||||
Reference in new issue
Block a user