diff --git a/Pipfile b/Pipfile index 0f5460c..0f3d815 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ google-auth-httplib2 = "*" google-auth-oauthlib = "*" requests = "*" igramscraper = "*" +twitterscraper = "*" [dev-packages] pylint = "*" diff --git a/README.md b/README.md index 79ef86f..6175a34 100644 --- a/README.md +++ b/README.md @@ -126,20 +126,34 @@ Example: * !loc Tampere -### Instagram +### Slow polling services -Polls instagram account(s) and posts new items to the room. Uses instagram scraper library -without any authentication or api key so it polls only randomly every 30 to 60 minutes to keep traffic at minimum. - -See: https://github.com/realsirjoe/instagram-scraper/ +These have the same usage - you can add one or more accounts to a room and bot polls the accounts. +New posts are sent to room. Polls only randomly every 30 to 60 minutes to keep traffic at minimum. Commands: -* !ig add [accountname] - Add instagram account to this room (Must be done as room admin) -* !ig del [accountname] - Delete instagram account from room (Must be done as room admin) -* !ig list - List accounts in room -* !ig poll - Poll for new items (Must be done as bot owner) -* !ig clear - Clear all ig accounts from this room (Must be done as room admin) +Prefix with selected service, for example "!ig add accountname" or "!twitter list" + +* add [accountname] - Add account to this room (Must be done as room admin) +* del [accountname] - Delete account from room (Must be done as room admin) +* list - List accounts in room +* poll - Poll for new items (Must be done as bot owner) +* clear - Clear all accounts from this room (Must be done as room admin) + +#### Instagram + +Polls instagram account(s). Uses instagram scraper library +without any authentication or api key. + +See: https://github.com/realsirjoe/instagram-scraper/ + +#### Twitter + +Polls twitter account(s). Uses twitter scraper library +without any authentication or api key. + +See: https://github.com/taspinar/twitterscraper/tree/master/twitterscraper ## Bot setup diff --git a/bot.py b/bot.py index a370cef..7f1b146 100755 --- a/bot.py +++ b/bot.py @@ -27,7 +27,7 @@ class CommandRequiresOwner(Exception): class Bot: appid = 'org.vranki.hemppa' - version = '1.1' + version = '1.2' client = None join_on_invite = False modules = dict() @@ -107,6 +107,9 @@ class Bot: body = event.body if len(body) == 0: return + if body[0] != '!': + return + command = body.split().pop(0) # Strip away non-alphanumeric characters, including leading ! for security @@ -150,9 +153,12 @@ class Bot: traceback.print_exc(file=sys.stderr) return None - def reload_module(self, modulename): - print('Reloading', modulename) - self.modules[modulename] = self.load_module(modulename) + def reload_modules(self): + for modulename in bot.modules: + print('Reloading', modulename, '..') + self.modules[modulename] = self.load_module(modulename) + + self.load_settings(self.get_account_data()) def get_modules(self): modulefiles = glob.glob('./modules/*.py') diff --git a/modules/bot.py b/modules/bot.py index 078f022..ca109ef 100644 --- a/modules/bot.py +++ b/modules/bot.py @@ -20,8 +20,7 @@ class MatrixModule: bot.must_be_admin(room, event) await bot.send_text(room, f'Reloading modules..') bot.stop() - for modulename in bot.modules: - bot.reload_module(modulename) + bot.reload_modules() bot.start() elif args[1]=='stats': roomcount = len(bot.client.rooms) diff --git a/modules/common/pollingservice.py b/modules/common/pollingservice.py new file mode 100644 index 0000000..733ee18 --- /dev/null +++ b/modules/common/pollingservice.py @@ -0,0 +1,101 @@ +import traceback +import sys +from datetime import datetime, timedelta +from random import randrange + +class PollingService: + known_ids = set() + account_rooms = dict() # Roomid -> [account, account..] + next_poll_time = dict() # Roomid -> datetime, None = not polled yet + service_name = "Service" + + async def matrix_poll(self, bot, pollcount): + if len(self.account_rooms): + await self.poll_all_accounts(bot) + + async def poll_all_accounts(self, bot): + now = datetime.now() + for roomid in self.account_rooms: + send_messages = True + if not self.next_poll_time.get(roomid, None): + self.next_poll_time[roomid] = now + send_messages = False + if now >= self.next_poll_time.get(roomid): + accounts = self.account_rooms[roomid] + for account in accounts: + await self.poll_account(bot, account, roomid, send_messages) + + self.first_run = False + + async def poll_implementation(self, bot, account, roomid, send_messages): + pass + + async def poll_account(self, bot, account, roomid, send_messages): + polldelay = timedelta(minutes=30 + randrange(30)) + self.next_poll_time[roomid] = datetime.now() + polldelay + + await self.poll_implementation(bot, account, roomid, send_messages) + + + async def matrix_message(self, bot, room, event): + args = event.body.split() + + if len(args) == 2: + if args[1] == 'list': + await bot.send_text(room, f'{self.service_name} accounts in this room: {self.account_rooms.get(room.room_id) or []}') + if args[1] == 'debug': + await bot.send_text(room, f'{self.service_name} accounts: {self.account_rooms.get(room.room_id) or []} - known ids: {self.known_ids}') + elif args[1] == 'poll': + bot.must_be_owner(event) + print('forced polling') + for roomid in self.account_rooms: + self.next_poll_time[roomid] = datetime.now() + await self.poll_all_accounts(bot) + elif args[1] == 'clear': + bot.must_be_admin(room, event) + self.account_rooms[room.room_id] = [] + bot.save_settings() + await bot.send_text(room, f'Cleared all {self.service_name} accounts from this room') + if len(args) == 3: + if args[1] == 'add': + bot.must_be_admin(room, event) + + account = args[2] + print(f'Adding {self.service_name} account {account} to room id {room.room_id}') + + if self.account_rooms.get(room.room_id): + if account not in self.account_rooms[room.room_id]: + self.account_rooms[room.room_id].append(account) + else: + await bot.send_text(room, 'This account already added in this room!') + return + else: + self.account_rooms[room.room_id] = [account] + bot.save_settings() + await bot.send_text(room, f'Added {self.service_name} account {account} to this room.') + + elif args[1] == 'del': + bot.must_be_admin(room, event) + + account = args[2] + print( + f'Removing {self.service_name} account {account} from room id {room.room_id}') + + if self.account_rooms.get(room.room_id): + self.account_rooms[room.room_id].remove(account) + + print( + f'{self.service_name} accounts now for this room {self.account_rooms.get(room.room_id)}') + + bot.save_settings() + await bot.send_text(room, f'Removed {self.service_name} account from this room') + + def get_settings(self): + return {'account_rooms': self.account_rooms} + + def set_settings(self, data): + if data.get('account_rooms'): + self.account_rooms = data['account_rooms'] + + def help(self): + return(f'{self.service_name} polling') diff --git a/modules/ig.py b/modules/ig.py index 2ddc3ab..5554dda 100644 --- a/modules/ig.py +++ b/modules/ig.py @@ -2,38 +2,17 @@ import traceback import sys from datetime import datetime, timedelta from random import randrange +from modules.common.pollingservice import PollingService from igramscraper.exception.instagram_not_found_exception import \ InstagramNotFoundException from igramscraper.instagram import Instagram - -class MatrixModule: +class MatrixModule(PollingService): instagram = Instagram() + service_name = 'Instagram' - known_ids = set() - account_rooms = dict() # Roomid -> [account, account..] - next_poll_time = dict() # Roomid -> datetime, None = not polled yet - - async def matrix_poll(self, bot, pollcount): - if len(self.account_rooms): - await self.poll_all_accounts(bot) - - async def poll_all_accounts(self, bot): - now = datetime.now() - for roomid in self.account_rooms: - send_messages = True - if not self.next_poll_time.get(roomid, None): - self.next_poll_time[roomid] = now - send_messages = False - if now >= self.next_poll_time.get(roomid): - accounts = self.account_rooms[roomid] - for account in accounts: - await self.poll_account(bot, account, roomid, send_messages) - - self.first_run = False - - async def poll_account(self, bot, account, roomid, send_messages): + async def poll_implementation(self, bot, account, roomid, send_messages): try: medias = self.instagram.get_medias(account, 5) for media in medias: @@ -53,72 +32,3 @@ class MatrixModule: polldelay = timedelta(minutes=30 + randrange(30)) self.next_poll_time[roomid] = datetime.now() + polldelay - - async def matrix_message(self, bot, room, event): - args = event.body.split() - - if len(args) == 2: - if args[1] == 'list': - await bot.send_text(room, f'Instagram accounts in this room: {self.account_rooms.get(room.room_id) or []}') - elif args[1] == 'poll': - bot.must_be_owner(event) - for roomid in self.account_rooms: - self.next_poll_time[roomid] = datetime.now() - await self.poll_all_accounts(bot) - elif args[1] == 'clear': - bot.must_be_admin(room, event) - self.account_rooms[room.room_id] = [] - bot.save_settings() - await bot.send_text(room, 'Cleared all instagram accounts from this room') - if len(args) == 3: - if args[1] == 'add': - bot.must_be_admin(room, event) - - account = args[2] - print(f'Adding account {account} to room id {room.room_id}') - - if self.account_rooms.get(room.room_id): - if account not in self.account_rooms[room.room_id]: - self.account_rooms[room.room_id].append(account) - else: - await bot.send_text(room, 'This instagram account already added in this room!') - return - else: - self.account_rooms[room.room_id] = [account] - - print( - f'Accounts now for this room {self.account_rooms.get(room.room_id)}') - - try: - await self.poll_account(bot, account, room.room_id, False) - bot.save_settings() - await bot.send_text(room, 'Added new instagram account to this room') - except InstagramNotFoundException: - await bot.send_text(room, 'Account doesn\'t seem to exist') - self.account_rooms[room.room_id].remove(account) - - elif args[1] == 'del': - bot.must_be_admin(room, event) - - account = args[2] - print( - f'Removing account {account} from room id {room.room_id}') - - if self.account_rooms.get(room.room_id): - self.account_rooms[room.room_id].remove(account) - - print( - f'Accounts now for this room {self.account_rooms.get(room.room_id)}') - - bot.save_settings() - await bot.send_text(room, 'Removed instagram account from this room') - - def get_settings(self): - return {'account_rooms': self.account_rooms} - - def set_settings(self, data): - if data.get('account_rooms'): - self.account_rooms = data['account_rooms'] - - def help(self): - return('Instagram polling') diff --git a/modules/twitter.py b/modules/twitter.py new file mode 100644 index 0000000..a21c4f7 --- /dev/null +++ b/modules/twitter.py @@ -0,0 +1,13 @@ +from twitterscraper import query_tweets_from_user +from modules.common.pollingservice import PollingService + +# https://github.com/taspinar/twitterscraper/tree/master/twitterscraper + +class MatrixModule(PollingService): + service_name = 'Twitter' + + async def poll_implementation(self, bot, account, roomid, send_messages): + for tweet in query_tweets_from_user("twitter", limit=1): + if tweet.tweet_id not in self.known_ids: + await bot.send_html(bot.get_room_by_id(roomid), f'Twitter {account}: {tweet.text}', f'Twitter {account}: {tweet.text} - https://twitter.com{tweet.tweet_url}') + self.known_ids.add(tweet.tweet_id)