Fixed slow polling service issue, some minor tuning. Fixes #16.
This commit is contained in:
parent
9665b60da3
commit
80c1bc3f95
|
@ -19,6 +19,7 @@ Support room: #hemppa:hacklab.fi - https://matrix.to/#/#hemppa:hacklab.fi
|
||||||
|
|
||||||
Bot management commands.
|
Bot management commands.
|
||||||
|
|
||||||
|
* !bot status - print bot status information
|
||||||
* !bot version - print version and uptime of the bot
|
* !bot version - print version and uptime of the bot
|
||||||
* !bot quit - quit the bot process (Must be done as bot owner)
|
* !bot quit - quit the bot process (Must be done as bot owner)
|
||||||
* !bot reload - reload all bot modules (Must be done as bot owner)
|
* !bot reload - reload all bot modules (Must be done as bot owner)
|
||||||
|
@ -140,18 +141,19 @@ Prefix with selected service, for example "!ig add accountname" or "!twitter lis
|
||||||
* list - List accounts in room
|
* list - List accounts in room
|
||||||
* poll - Poll for new items (Must be done as bot owner)
|
* poll - Poll for new items (Must be done as bot owner)
|
||||||
* clear - Clear all accounts from this room (Must be done as room admin)
|
* clear - Clear all accounts from this room (Must be done as room admin)
|
||||||
|
* debug - Show some debug information for accounts in room
|
||||||
|
|
||||||
#### Instagram
|
#### Instagram
|
||||||
|
|
||||||
Polls instagram account(s). Uses instagram scraper library
|
Polls instagram account(s). Uses instagram scraper library
|
||||||
without any authentication or api key.
|
without any authentication or api key.
|
||||||
|
|
||||||
See: https://github.com/realsirjoe/instagram-scraper/
|
See: https://github.com/realsirjoe/instagram-scraper/
|
||||||
|
|
||||||
#### Twitter
|
#### Twitter
|
||||||
|
|
||||||
Polls twitter account(s). Uses twitter scraper library
|
Polls twitter account(s). Uses twitter scraper library
|
||||||
without any authentication or api key.
|
without any authentication or api key.
|
||||||
|
|
||||||
See: https://github.com/taspinar/twitterscraper/tree/master/twitterscraper
|
See: https://github.com/taspinar/twitterscraper/tree/master/twitterscraper
|
||||||
|
|
||||||
|
|
|
@ -14,14 +14,16 @@ class MatrixModule:
|
||||||
print(f'{event.sender} commanded bot to quit, so quitting..')
|
print(f'{event.sender} commanded bot to quit, so quitting..')
|
||||||
bot.bot_task.cancel()
|
bot.bot_task.cancel()
|
||||||
elif args[1]=='version':
|
elif args[1]=='version':
|
||||||
uptme = datetime.now() - self.starttime
|
await bot.send_text(room, f'Hemppa version {bot.version} - https://github.com/vranki/hemppa')
|
||||||
await bot.send_text(room, f'Hemppa version {bot.version} - Uptime {uptme} - https://github.com/vranki/hemppa')
|
|
||||||
elif args[1]=='reload':
|
elif args[1]=='reload':
|
||||||
bot.must_be_admin(room, event)
|
bot.must_be_admin(room, event)
|
||||||
await bot.send_text(room, f'Reloading modules..')
|
await bot.send_text(room, f'Reloading modules..')
|
||||||
bot.stop()
|
bot.stop()
|
||||||
bot.reload_modules()
|
bot.reload_modules()
|
||||||
bot.start()
|
bot.start()
|
||||||
|
elif args[1]=='status':
|
||||||
|
uptime = datetime.now() - self.starttime
|
||||||
|
await bot.send_text(room, f'Uptime {uptime} - system time is {datetime.now()} - loaded {len(bot.modules)} modules.')
|
||||||
elif args[1]=='stats':
|
elif args[1]=='stats':
|
||||||
roomcount = len(bot.client.rooms)
|
roomcount = len(bot.client.rooms)
|
||||||
usercount = 0
|
usercount = 0
|
||||||
|
@ -39,10 +41,10 @@ class MatrixModule:
|
||||||
|
|
||||||
homeservers = sorted(homeservers.items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
homeservers = sorted(homeservers.items(), key=lambda kv: (kv[1], kv[0]), reverse=True)
|
||||||
|
|
||||||
if len(homeservers) > 5:
|
if len(homeservers) > 10:
|
||||||
homeservers = homeservers[0:5]
|
homeservers = homeservers[0:10]
|
||||||
|
|
||||||
await bot.send_text(room, f'I\'m seeing {usercount} users in {roomcount} rooms. Top 5 homeservers: {homeservers}')
|
await bot.send_text(room, f'I\'m seeing {usercount} users in {roomcount} rooms. Top ten homeservers: {homeservers}')
|
||||||
else:
|
else:
|
||||||
await bot.send_text(room, 'Unknown command, sorry.')
|
await bot.send_text(room, 'Unknown command, sorry.')
|
||||||
|
|
||||||
|
|
|
@ -4,10 +4,11 @@ from datetime import datetime, timedelta
|
||||||
from random import randrange
|
from random import randrange
|
||||||
|
|
||||||
class PollingService:
|
class PollingService:
|
||||||
known_ids = set()
|
def __init__(self):
|
||||||
account_rooms = dict() # Roomid -> [account, account..]
|
self.known_ids = set()
|
||||||
next_poll_time = dict() # Roomid -> datetime, None = not polled yet
|
self.account_rooms = dict() # Roomid -> [account, account..]
|
||||||
service_name = "Service"
|
self.next_poll_time = dict() # Roomid -> datetime, None = not polled yet
|
||||||
|
self.service_name = "Service"
|
||||||
|
|
||||||
async def matrix_poll(self, bot, pollcount):
|
async def matrix_poll(self, bot, pollcount):
|
||||||
if len(self.account_rooms):
|
if len(self.account_rooms):
|
||||||
|
@ -17,8 +18,9 @@ class PollingService:
|
||||||
now = datetime.now()
|
now = datetime.now()
|
||||||
for roomid in self.account_rooms:
|
for roomid in self.account_rooms:
|
||||||
send_messages = True
|
send_messages = True
|
||||||
|
# First poll
|
||||||
if not self.next_poll_time.get(roomid, None):
|
if not self.next_poll_time.get(roomid, None):
|
||||||
self.next_poll_time[roomid] = now
|
self.next_poll_time[roomid] = now + timedelta(hours=-1)
|
||||||
send_messages = False
|
send_messages = False
|
||||||
if now >= self.next_poll_time.get(roomid):
|
if now >= self.next_poll_time.get(roomid):
|
||||||
accounts = self.account_rooms[roomid]
|
accounts = self.account_rooms[roomid]
|
||||||
|
@ -44,10 +46,10 @@ class PollingService:
|
||||||
if args[1] == 'list':
|
if args[1] == 'list':
|
||||||
await bot.send_text(room, f'{self.service_name} accounts in this room: {self.account_rooms.get(room.room_id) or []}')
|
await bot.send_text(room, f'{self.service_name} accounts in this room: {self.account_rooms.get(room.room_id) or []}')
|
||||||
if args[1] == 'debug':
|
if args[1] == 'debug':
|
||||||
await bot.send_text(room, f'{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}')
|
await bot.send_text(room, f"{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}\n" \
|
||||||
|
f"Next poll in this room at {self.next_poll_time.get(room.room_id)} - in {self.next_poll_time.get(room.room_id) - datetime.now()}")
|
||||||
elif args[1] == 'poll':
|
elif args[1] == 'poll':
|
||||||
bot.must_be_owner(event)
|
bot.must_be_owner(event)
|
||||||
print('forced polling')
|
|
||||||
for roomid in self.account_rooms:
|
for roomid in self.account_rooms:
|
||||||
self.next_poll_time[roomid] = datetime.now()
|
self.next_poll_time[roomid] = datetime.now()
|
||||||
await self.poll_all_accounts(bot)
|
await self.poll_all_accounts(bot)
|
||||||
|
|
|
@ -8,13 +8,16 @@ from igramscraper.exception.instagram_not_found_exception import \
|
||||||
InstagramNotFoundException
|
InstagramNotFoundException
|
||||||
from igramscraper.instagram import Instagram
|
from igramscraper.instagram import Instagram
|
||||||
|
|
||||||
class MatrixModule(PollingService):
|
class MatrixModule(PollingService):
|
||||||
instagram = Instagram()
|
def __init__(self):
|
||||||
service_name = 'Instagram'
|
super().__init__()
|
||||||
|
self.instagram = Instagram()
|
||||||
|
self.service_name = 'Instagram'
|
||||||
|
|
||||||
async def poll_implementation(self, bot, account, roomid, send_messages):
|
async def poll_implementation(self, bot, account, roomid, send_messages):
|
||||||
try:
|
try:
|
||||||
medias = self.instagram.get_medias(account, 5)
|
medias = self.instagram.get_medias(account, 5)
|
||||||
|
print(f'Polling instagram account {account} for room {roomid} - got {len(medias)} posts.')
|
||||||
for media in medias:
|
for media in medias:
|
||||||
if send_messages:
|
if send_messages:
|
||||||
if media.identifier not in self.known_ids:
|
if media.identifier not in self.known_ids:
|
||||||
|
|
|
@ -4,14 +4,18 @@ from modules.common.pollingservice import PollingService
|
||||||
# https://github.com/taspinar/twitterscraper/tree/master/twitterscraper
|
# https://github.com/taspinar/twitterscraper/tree/master/twitterscraper
|
||||||
|
|
||||||
class MatrixModule(PollingService):
|
class MatrixModule(PollingService):
|
||||||
service_name = 'Twitter'
|
def __init__(self):
|
||||||
|
super().__init__()
|
||||||
|
self.service_name = 'Twitter'
|
||||||
|
|
||||||
async def poll_implementation(self, bot, account, roomid, send_messages):
|
async def poll_implementation(self, bot, account, roomid, send_messages):
|
||||||
try:
|
try:
|
||||||
for tweet in query_tweets_from_user(account, limit=1):
|
tweets = query_tweets_from_user(account, limit=1)
|
||||||
|
print(f'Polling twitter account {account} - got {len(tweets)} tweets')
|
||||||
|
for tweet in tweets:
|
||||||
if tweet.tweet_id not in self.known_ids:
|
if tweet.tweet_id not in self.known_ids:
|
||||||
if send_messages:
|
if send_messages:
|
||||||
await bot.send_html(bot.get_room_by_id(roomid), f'Twitter <a href="https://twitter.com{tweet.tweet_url}">{account}</a>: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}')
|
await bot.send_html(bot.get_room_by_id(roomid), f'<a href="https://twitter.com{tweet.tweet_url}">Twitter {account}</a>: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}')
|
||||||
self.known_ids.add(tweet.tweet_id)
|
self.known_ids.add(tweet.tweet_id)
|
||||||
except Exception:
|
except Exception:
|
||||||
print('Polling twitter account failed:')
|
print('Polling twitter account failed:')
|
||||||
|
|
Loading…
Reference in New Issue