jarno
/
hemppa
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed slow polling service issue, some minor tuning. Fixes #16.

This commit is contained in:
Ville Ranki 2020-01-20 23:54:10 +02:00
parent 9665b60da3
commit 80c1bc3f95
5 changed files with 33 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -19,6 +19,7 @@ Support room: #hemppa:hacklab.fi - https://matrix.to/#/#hemppa:hacklab.fi
Bot management commands.
* !bot status - print bot status information
* !bot version - print version and uptime of the bot
* !bot quit - quit the bot process (Must be done as bot owner)
* !bot reload - reload all bot modules (Must be done as bot owner)
@ -140,18 +141,19 @@ Prefix with selected service, for example "!ig add accountname" or "!twitter lis
* list - List accounts in room
* poll - Poll for new items (Must be done as bot owner)
* clear - Clear all accounts from this room (Must be done as room admin)
* debug - Show some debug information for accounts in room
#### Instagram
Polls instagram account(s). Uses instagram scraper library
without any authentication or api key.
without any authentication or api key.
See: https://github.com/realsirjoe/instagram-scraper/
#### Twitter
Polls twitter account(s). Uses twitter scraper library
without any authentication or api key.
without any authentication or api key.
See: https://github.com/taspinar/twitterscraper/tree/master/twitterscraper

12
modules/bot.py
View File

@ -14,14 +14,16 @@ class MatrixModule:
print(f'{event.sender} commanded bot to quit, so quitting..')
bot.bot_task.cancel()
elif args[1]=='version':
uptme = datetime.now() - self.starttime
await bot.send_text(room, f'Hemppa version {bot.version} - Uptime {uptme} - https://github.com/vranki/hemppa')
await bot.send_text(room, f'Hemppa version {bot.version} - https://github.com/vranki/hemppa')
elif args[1]=='reload':
bot.must_be_admin(room, event)
await bot.send_text(room, f'Reloading modules..')
bot.stop()
bot.reload_modules()
bot.start()
elif args[1]=='status':
uptime = datetime.now() - self.starttime
await bot.send_text(room, f'Uptime {uptime} - system time is {datetime.now()} - loaded {len(bot.modules)} modules.')
elif args[1]=='stats':
roomcount = len(bot.client.rooms)
usercount = 0
@ -39,10 +41,10 @@ class MatrixModule:
homeservers = sorted(homeservers.items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
if len(homeservers) > 5:
homeservers = homeservers[0:5]
if len(homeservers) > 10:
homeservers = homeservers[0:10]
await bot.send_text(room, f'I\'m seeing {usercount} users in {roomcount} rooms. Top 5 homeservers: {homeservers}')
await bot.send_text(room, f'I\'m seeing {usercount} users in {roomcount} rooms. Top ten homeservers: {homeservers}')
else:
await bot.send_text(room, 'Unknown command, sorry.')

16
modules/common/pollingservice.py
View File

@ -4,10 +4,11 @@ from datetime import datetime, timedelta
from random import randrange
class PollingService:
known_ids = set()
account_rooms = dict() # Roomid -> [account, account..]
next_poll_time = dict() # Roomid -> datetime, None = not polled yet
service_name = "Service"
def __init__(self):
self.known_ids = set()
self.account_rooms = dict() # Roomid -> [account, account..]
self.next_poll_time = dict() # Roomid -> datetime, None = not polled yet
self.service_name = "Service"
async def matrix_poll(self, bot, pollcount):
if len(self.account_rooms):
@ -17,8 +18,9 @@ class PollingService:
now = datetime.now()
for roomid in self.account_rooms:
send_messages = True
# First poll
if not self.next_poll_time.get(roomid, None):
self.next_poll_time[roomid] = now
self.next_poll_time[roomid] = now + timedelta(hours=-1)
send_messages = False
if now >= self.next_poll_time.get(roomid):
accounts = self.account_rooms[roomid]
@ -44,10 +46,10 @@ class PollingService:
if args[1] == 'list':
await bot.send_text(room, f'{self.service_name} accounts in this room: {self.account_rooms.get(room.room_id) or []}')
if args[1] == 'debug':
await bot.send_text(room, f'{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}')
await bot.send_text(room, f"{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}\n" \
f"Next poll in this room at {self.next_poll_time.get(room.room_id)} - in {self.next_poll_time.get(room.room_id) - datetime.now()}")
elif args[1] == 'poll':
bot.must_be_owner(event)
print('forced polling')
for roomid in self.account_rooms:
self.next_poll_time[roomid] = datetime.now()
await self.poll_all_accounts(bot)

9
modules/ig.py
View File

@ -8,13 +8,16 @@ from igramscraper.exception.instagram_not_found_exception import \
InstagramNotFoundException
from igramscraper.instagram import Instagram
class MatrixModule(PollingService):
instagram = Instagram()
service_name = 'Instagram'
class MatrixModule(PollingService):
def __init__(self):
super().__init__()
self.instagram = Instagram()
self.service_name = 'Instagram'
async def poll_implementation(self, bot, account, roomid, send_messages):
try:
medias = self.instagram.get_medias(account, 5)
print(f'Polling instagram account {account} for room {roomid} - got {len(medias)} posts.')
for media in medias:
if send_messages:
if media.identifier not in self.known_ids:

10
modules/twitter.py
View File

@ -4,14 +4,18 @@ from modules.common.pollingservice import PollingService
# https://github.com/taspinar/twitterscraper/tree/master/twitterscraper
class MatrixModule(PollingService):
service_name = 'Twitter'
def __init__(self):
super().__init__()
self.service_name = 'Twitter'
async def poll_implementation(self, bot, account, roomid, send_messages):
try:
for tweet in query_tweets_from_user(account, limit=1):
tweets = query_tweets_from_user(account, limit=1)
print(f'Polling twitter account {account} - got {len(tweets)} tweets')
for tweet in tweets:
if tweet.tweet_id not in self.known_ids:
if send_messages:
await bot.send_html(bot.get_room_by_id(roomid), f'Twitter <a href="https://twitter.com{tweet.tweet_url}">{account}</a>: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}')
await bot.send_html(bot.get_room_by_id(roomid), f'<a href="https://twitter.com{tweet.tweet_url}">Twitter {account}</a>: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}')
self.known_ids.add(tweet.tweet_id)
except Exception:
print('Polling twitter account failed:')