From 80c1bc3f95d984a904bac1e02faee9a24e2a94ae Mon Sep 17 00:00:00 2001 From: Ville Ranki Date: Mon, 20 Jan 2020 23:54:10 +0200 Subject: [PATCH] Fixed slow polling service issue, some minor tuning. Fixes #16. --- README.md | 6 ++++-- modules/bot.py | 12 +++++++----- modules/common/pollingservice.py | 16 +++++++++------- modules/ig.py | 9 ++++++--- modules/twitter.py | 10 +++++++--- 5 files changed, 33 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6175a34..6771fef 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Support room: #hemppa:hacklab.fi - https://matrix.to/#/#hemppa:hacklab.fi Bot management commands. +* !bot status - print bot status information * !bot version - print version and uptime of the bot * !bot quit - quit the bot process (Must be done as bot owner) * !bot reload - reload all bot modules (Must be done as bot owner) @@ -140,18 +141,19 @@ Prefix with selected service, for example "!ig add accountname" or "!twitter lis * list - List accounts in room * poll - Poll for new items (Must be done as bot owner) * clear - Clear all accounts from this room (Must be done as room admin) +* debug - Show some debug information for accounts in room #### Instagram Polls instagram account(s). Uses instagram scraper library -without any authentication or api key. +without any authentication or api key. See: https://github.com/realsirjoe/instagram-scraper/ #### Twitter Polls twitter account(s). Uses twitter scraper library -without any authentication or api key. +without any authentication or api key. See: https://github.com/taspinar/twitterscraper/tree/master/twitterscraper diff --git a/modules/bot.py b/modules/bot.py index ca109ef..df9fff6 100644 --- a/modules/bot.py +++ b/modules/bot.py @@ -14,14 +14,16 @@ class MatrixModule: print(f'{event.sender} commanded bot to quit, so quitting..') bot.bot_task.cancel() elif args[1]=='version': - uptme = datetime.now() - self.starttime - await bot.send_text(room, f'Hemppa version {bot.version} - Uptime {uptme} - https://github.com/vranki/hemppa') + await bot.send_text(room, f'Hemppa version {bot.version} - https://github.com/vranki/hemppa') elif args[1]=='reload': bot.must_be_admin(room, event) await bot.send_text(room, f'Reloading modules..') bot.stop() bot.reload_modules() bot.start() + elif args[1]=='status': + uptime = datetime.now() - self.starttime + await bot.send_text(room, f'Uptime {uptime} - system time is {datetime.now()} - loaded {len(bot.modules)} modules.') elif args[1]=='stats': roomcount = len(bot.client.rooms) usercount = 0 @@ -39,10 +41,10 @@ class MatrixModule: homeservers = sorted(homeservers.items(), key=lambda kv: (kv[1], kv[0]), reverse=True) - if len(homeservers) > 5: - homeservers = homeservers[0:5] + if len(homeservers) > 10: + homeservers = homeservers[0:10] - await bot.send_text(room, f'I\'m seeing {usercount} users in {roomcount} rooms. Top 5 homeservers: {homeservers}') + await bot.send_text(room, f'I\'m seeing {usercount} users in {roomcount} rooms. Top ten homeservers: {homeservers}') else: await bot.send_text(room, 'Unknown command, sorry.') diff --git a/modules/common/pollingservice.py b/modules/common/pollingservice.py index 733ee18..78e26a3 100644 --- a/modules/common/pollingservice.py +++ b/modules/common/pollingservice.py @@ -4,10 +4,11 @@ from datetime import datetime, timedelta from random import randrange class PollingService: - known_ids = set() - account_rooms = dict() # Roomid -> [account, account..] - next_poll_time = dict() # Roomid -> datetime, None = not polled yet - service_name = "Service" + def __init__(self): + self.known_ids = set() + self.account_rooms = dict() # Roomid -> [account, account..] + self.next_poll_time = dict() # Roomid -> datetime, None = not polled yet + self.service_name = "Service" async def matrix_poll(self, bot, pollcount): if len(self.account_rooms): @@ -17,8 +18,9 @@ class PollingService: now = datetime.now() for roomid in self.account_rooms: send_messages = True + # First poll if not self.next_poll_time.get(roomid, None): - self.next_poll_time[roomid] = now + self.next_poll_time[roomid] = now + timedelta(hours=-1) send_messages = False if now >= self.next_poll_time.get(roomid): accounts = self.account_rooms[roomid] @@ -44,10 +46,10 @@ class PollingService: if args[1] == 'list': await bot.send_text(room, f'{self.service_name} accounts in this room: {self.account_rooms.get(room.room_id) or []}') if args[1] == 'debug': - await bot.send_text(room, f'{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}') + await bot.send_text(room, f"{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}\n" \ + f"Next poll in this room at {self.next_poll_time.get(room.room_id)} - in {self.next_poll_time.get(room.room_id) - datetime.now()}") elif args[1] == 'poll': bot.must_be_owner(event) - print('forced polling') for roomid in self.account_rooms: self.next_poll_time[roomid] = datetime.now() await self.poll_all_accounts(bot) diff --git a/modules/ig.py b/modules/ig.py index 5554dda..4609161 100644 --- a/modules/ig.py +++ b/modules/ig.py @@ -8,13 +8,16 @@ from igramscraper.exception.instagram_not_found_exception import \ InstagramNotFoundException from igramscraper.instagram import Instagram -class MatrixModule(PollingService): - instagram = Instagram() - service_name = 'Instagram' +class MatrixModule(PollingService): + def __init__(self): + super().__init__() + self.instagram = Instagram() + self.service_name = 'Instagram' async def poll_implementation(self, bot, account, roomid, send_messages): try: medias = self.instagram.get_medias(account, 5) + print(f'Polling instagram account {account} for room {roomid} - got {len(medias)} posts.') for media in medias: if send_messages: if media.identifier not in self.known_ids: diff --git a/modules/twitter.py b/modules/twitter.py index 41f71ca..d302d6f 100644 --- a/modules/twitter.py +++ b/modules/twitter.py @@ -4,14 +4,18 @@ from modules.common.pollingservice import PollingService # https://github.com/taspinar/twitterscraper/tree/master/twitterscraper class MatrixModule(PollingService): - service_name = 'Twitter' + def __init__(self): + super().__init__() + self.service_name = 'Twitter' async def poll_implementation(self, bot, account, roomid, send_messages): try: - for tweet in query_tweets_from_user(account, limit=1): + tweets = query_tweets_from_user(account, limit=1) + print(f'Polling twitter account {account} - got {len(tweets)} tweets') + for tweet in tweets: if tweet.tweet_id not in self.known_ids: if send_messages: - await bot.send_html(bot.get_room_by_id(roomid), f'Twitter {account}: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}') + await bot.send_html(bot.get_room_by_id(roomid), f'Twitter {account}: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}') self.known_ids.add(tweet.tweet_id) except Exception: print('Polling twitter account failed:')