diff --git a/slixfeed/__main__.py b/slixfeed/__main__.py index c50b185..b68cfd6 100644 --- a/slixfeed/__main__.py +++ b/slixfeed/__main__.py @@ -13,11 +13,17 @@ FIXME TODO +0) from slixfeed.FILENAME import XYZ + See project feed2toot + 1) SQL prepared statements. 2) Machine Learning for scrapping Title, Link, Summary and Timstamp. -3) Support MUC. +3) Set MUC subject + Feeds which entries are to be set as groupchat subject. + Perhaps not, as it would require to check every feed for this setting. + Maybe a separate bot. 4) Support categories. @@ -36,7 +42,9 @@ TODO 11) Download and upload/send article (xHTML, xHTMLZ, Markdown, MHTML, TXT). Use Readability. -12) Fetch summary from URL, instead of storing summary. +12) Fetch summary from URL, instead of storing summary, or + Store 5 upcoming summaries. + This would help making the database files smaller. 13) Support protocol Gopher https://github.com/michael-lazar/pygopherd @@ -60,22 +68,22 @@ from argparse import ArgumentParser from getpass import getpass import logging -from datetime import date -import time +# from datetime import date +# import time # from eliot import start_action, to_file # # to_file(open("slixfeed.log", "w")) # # with start_action(action_type="set_date()", jid=jid): # # with start_action(action_type="message()", msg=msg): -#import irchandler -import xmpphandler -#import matrixhandler +#import slixfeed.irchandler +from xmpphandler import Slixfeed +#import slixfeed.matrixhandler if __name__ == '__main__': # Setup the command line arguments. - parser = ArgumentParser(description=xmpphandler.Slixfeed.__doc__) + parser = ArgumentParser(description=Slixfeed.__doc__) # Output verbosity options. parser.add_argument( @@ -109,7 +117,7 @@ if __name__ == '__main__': # Setup the Slixfeed and register plugins. Note that while plugins may # have interdependencies, the order in which you register them does # not matter. - xmpp = xmpphandler.Slixfeed(args.jid, args.password) + xmpp = Slixfeed(args.jid, args.password) xmpp.register_plugin('xep_0004') # Data Forms xmpp.register_plugin('xep_0030') # Service Discovery xmpp.register_plugin('xep_0045') # Multi-User Chat diff --git a/slixfeed/confighandler.py b/slixfeed/confighandler.py index 01a4f3a..2dc9a73 100644 --- a/slixfeed/confighandler.py +++ b/slixfeed/confighandler.py @@ -10,8 +10,9 @@ TODO """ + import os -import filehandler +from filehandler import get_default_confdir from random import randrange @@ -64,7 +65,7 @@ def get_list(): Dictionary of pathnames. """ paths = [] - cfg_dir = filehandler.get_default_confdir() + cfg_dir = get_default_confdir() if not os.path.isdir(cfg_dir): os.mkdir(cfg_dir) cfg_file = os.path.join(cfg_dir, r"url_paths.txt") diff --git a/slixfeed/datahandler.py b/slixfeed/datahandler.py index c3470f2..944e268 100644 --- a/slixfeed/datahandler.py +++ b/slixfeed/datahandler.py @@ -14,25 +14,32 @@ TODO """ -import aiohttp -import asyncio -import feedparser - -import sqlitehandler -import confighandler -import datetimehandler -import listhandler - +from aiohttp import ClientError +from aiohttp import ClientSession +from aiohttp import ClientTimeout +from asyncio import TimeoutError from asyncio.exceptions import IncompleteReadError -from http.client import IncompleteRead -from urllib import error from bs4 import BeautifulSoup +from feedparser import parse +from http.client import IncompleteRead +from lxml import html +from datetimehandler import now +from datetimehandler import rfc2822_to_iso8601 +from confighandler import get_list +from listhandler import is_listed +from sqlitehandler import add_entry_and_set_date +from sqlitehandler import insert_feed +from sqlitehandler import check_entry_exist +from sqlitehandler import check_feed_exist +from sqlitehandler import get_feeds_url +from sqlitehandler import remove_nonexistent_entries +from sqlitehandler import update_source_status +from sqlitehandler import update_source_validity +from urllib import error # from xml.etree.ElementTree import ElementTree, ParseError from urllib.parse import urljoin from urllib.parse import urlsplit from urllib.parse import urlunsplit -from lxml import html - # NOTE Why (if res[0]) and (if res[1] == 200)? async def download_updates(db_file, url=None): @@ -49,7 +56,7 @@ async def download_updates(db_file, url=None): if url: urls = [url] # Valid [url] and [url,] and (url,) else: - urls = await sqlitehandler.get_feeds_url(db_file) + urls = await get_feeds_url(db_file) for url in urls: # print(os.path.basename(db_file), url[0]) source = url[0] @@ -60,14 +67,14 @@ async def download_updates(db_file, url=None): # urls.next() # next(urls) continue - await sqlitehandler.update_source_status( + await update_source_status( db_file, res[1], source ) if res[0]: try: - feed = feedparser.parse(res[0]) + feed = parse(res[0]) if feed.bozo: # bozo = ( # "WARNING: Bozo detected for feed: {}\n" @@ -78,7 +85,7 @@ async def download_updates(db_file, url=None): valid = 0 else: valid = 1 - await sqlitehandler.update_source_validity( + await update_source_validity( db_file, source, valid) @@ -102,8 +109,8 @@ async def download_updates(db_file, url=None): # NOTE Need to correct the SQL statement to do so entries = feed.entries # length = len(entries) - # await sqlitehandler.remove_entry(db_file, source, length) - await sqlitehandler.remove_nonexistent_entries( + # await remove_entry(db_file, source, length) + await remove_nonexistent_entries( db_file, feed, source @@ -113,14 +120,14 @@ async def download_updates(db_file, url=None): # TODO Pass date too for comparion check if entry.has_key("published"): date = entry.published - date = await datetimehandler.rfc2822_to_iso8601(date) + date = await rfc2822_to_iso8601(date) elif entry.has_key("updated"): date = entry.updated - date = await datetimehandler.rfc2822_to_iso8601(date) + date = await rfc2822_to_iso8601(date) else: # TODO Just set date = "*** No date ***" # date = await datetime.now().isoformat() - date = await datetimehandler.now() + date = await now() # NOTE Would seconds result in better database performance # date = datetime.datetime(date) # date = (date-datetime.datetime(1970,1,1)).total_seconds() @@ -140,7 +147,7 @@ async def download_updates(db_file, url=None): eid = entry.id else: eid = link - exist = await sqlitehandler.check_entry_exist( + exist = await check_entry_exist( db_file, source, eid=eid, @@ -172,13 +179,13 @@ async def download_updates(db_file, url=None): summary, pathname ) - allow_list = await listhandler.is_listed( + allow_list = await is_listed( db_file, "filter-allow", string ) if not allow_list: - reject_list = await listhandler.is_listed( + reject_list = await is_listed( db_file, "filter-deny", string @@ -202,19 +209,19 @@ async def download_updates(db_file, url=None): read_status ) if isinstance(date, int): - print("date is int") + print("PROBLEM: date is int") print(date) - breakpoint() + # breakpoint() print(source) print(date) - await sqlitehandler.add_entry_and_set_date( + await add_entry_and_set_date( db_file, source, entry ) - # print(await datetimehandler.current_time(), entry, title) + # print(await current_time(), entry, title) # else: - # print(await datetimehandler.current_time(), exist, title) + # print(await current_time(), exist, title) # NOTE Why (if result[0]) and (if result[1] == 200)? @@ -237,7 +244,7 @@ async def view_feed(url): result = await download_feed(url) if result[0]: try: - feed = feedparser.parse(result[0]) + feed = parse(result[0]) if feed.bozo: # msg = ( # ">{}\n" @@ -258,7 +265,7 @@ async def view_feed(url): "> {}\n" "Error: {}" ).format(url, e) - breakpoint() + # breakpoint() if result[1] == 200: title = await get_title(url, result[0]) entries = feed.entries @@ -278,10 +285,10 @@ async def view_feed(url): link = "*** No link ***" if entry.has_key("published"): date = entry.published - date = await datetimehandler.rfc2822_to_iso8601(date) + date = await rfc2822_to_iso8601(date) elif entry.has_key("updated"): date = entry.updated - date = await datetimehandler.rfc2822_to_iso8601(date) + date = await rfc2822_to_iso8601(date) else: date = "*** No date ***" msg += ( @@ -313,7 +320,7 @@ async def view_entry(url, num): result = await download_feed(url) if result[0]: try: - feed = feedparser.parse(result[0]) + feed = parse(result[0]) if feed.bozo: # msg = ( # ">{}\n" @@ -334,9 +341,9 @@ async def view_entry(url, num): "> {}\n" "Error: {}" ).format(url, e) - breakpoint() + # breakpoint() if result[1] == 200: - feed = feedparser.parse(result[0]) + feed = parse(result[0]) title = await get_title(url, result[0]) entries = feed.entries num = int(num) - 1 @@ -347,10 +354,10 @@ async def view_entry(url, num): title = "*** No title ***" if entry.has_key("published"): date = entry.published - date = await datetimehandler.rfc2822_to_iso8601(date) + date = await rfc2822_to_iso8601(date) elif entry.has_key("updated"): date = entry.updated - date = await datetimehandler.rfc2822_to_iso8601(date) + date = await rfc2822_to_iso8601(date) else: date = "*** No date ***" if entry.has_key("summary"): @@ -407,9 +414,9 @@ async def add_feed_no_check(db_file, data): url = data[0] title = data[1] url = await trim_url(url) - exist = await sqlitehandler.check_feed_exist(db_file, url) + exist = await check_feed_exist(db_file, url) if not exist: - msg = await sqlitehandler.add_feed(db_file, url, title) + msg = await insert_feed(db_file, url, title) await download_updates(db_file, [url]) else: ix = exist[0] @@ -440,21 +447,21 @@ async def add_feed(db_file, url): """ msg = None url = await trim_url(url) - exist = await sqlitehandler.check_feed_exist(db_file, url) + exist = await check_feed_exist(db_file, url) if not exist: res = await download_feed(url) if res[0]: - feed = feedparser.parse(res[0]) + feed = parse(res[0]) title = await get_title(url, feed) if feed.bozo: bozo = ( "Bozo detected. Failed to load: {}." ).format(url) print(bozo) - msg = await probe_page(add_feed, url, res[0], db_file) + msg = await probe_page(add_feed, url, res[0], db_file=db_file) else: status = res[1] - msg = await sqlitehandler.add_feed( + msg = await insert_feed( db_file, url, title, @@ -507,6 +514,7 @@ async def probe_page(callback, url, doc, num=None, db_file=None): elif isinstance(msg, list): url = msg[0] if db_file: + print("if db_file", db_file) return await callback(db_file, url) elif num: return await callback(url, num) @@ -528,9 +536,9 @@ async def download_feed(url): msg: list or str Document or error message. """ - timeout = aiohttp.ClientTimeout(total=10) - async with aiohttp.ClientSession() as session: - # async with aiohttp.ClientSession(trust_env=True) as session: + timeout = ClientTimeout(total=10) + async with ClientSession() as session: + # async with ClientSession(trust_env=True) as session: try: async with session.get(url, timeout=timeout) as response: status = response.status @@ -558,13 +566,13 @@ async def download_feed(url): False, "HTTP Error: " + str(status) ] - except aiohttp.ClientError as e: + except ClientError as e: # print('Error', str(e)) msg = [ False, "Error: " + str(e) ] - except asyncio.TimeoutError as e: + except TimeoutError as e: # print('Timeout:', str(e)) msg = [ False, @@ -771,7 +779,7 @@ async def feed_mode_request(url, tree): """ feeds = {} parted_url = urlsplit(url) - paths = confighandler.get_list() + paths = get_list() for path in paths: address = urlunsplit([ parted_url.scheme, @@ -782,10 +790,10 @@ async def feed_mode_request(url, tree): ]) res = await download_feed(address) if res[1] == 200: - # print(feedparser.parse(res[0])["feed"]["title"]) - # feeds[address] = feedparser.parse(res[0])["feed"]["title"] + # print(parse(res[0])["feed"]["title"]) + # feeds[address] = parse(res[0])["feed"]["title"] try: - title = feedparser.parse(res[0])["feed"]["title"] + title = parse(res[0])["feed"]["title"] except: title = '*** No Title ***' feeds[address] = title @@ -806,7 +814,7 @@ async def feed_mode_request(url, tree): res = await download_feed(address) if res[1] == 200: try: - feeds[address] = feedparser.parse(res[0]) + feeds[address] = parse(res[0]) # print(feeds) except: continue @@ -871,7 +879,7 @@ async def feed_mode_scan(url, tree): feeds = {} # paths = [] # TODO Test - paths = confighandler.get_list() + paths = get_list() for path in paths: # xpath_query = "//*[@*[contains(.,'{}')]]".format(path) xpath_query = "//a[contains(@href,'{}')]".format(path) @@ -908,7 +916,7 @@ async def feed_mode_scan(url, tree): res = await download_feed(address) if res[1] == 200: try: - feeds[address] = feedparser.parse(res[0]) + feeds[address] = parse(res[0]) # print(feeds) except: continue @@ -992,7 +1000,7 @@ async def feed_mode_auto_discovery(url, tree): # # The following code requires more bandwidth. # res = await download_feed(feed) # if res[0]: - # disco = feedparser.parse(res[0]) + # disco = parse(res[0]) # title = disco["feed"]["title"] # msg += "{} \n {} \n\n".format(title, feed) feed_name = feed.xpath('@title')[0] diff --git a/slixfeed/filehandler.py b/slixfeed/filehandler.py index d6cf666..4d12c10 100644 --- a/slixfeed/filehandler.py +++ b/slixfeed/filehandler.py @@ -4,7 +4,7 @@ import os import sys -import sqlitehandler +from sqlitehandler import create_tables def get_default_dbdir(): """ @@ -97,8 +97,8 @@ async def initdb(jid, callback, message=None): if not os.path.isdir(db_dir): os.mkdir(db_dir) db_file = os.path.join(db_dir, r"{}.db".format(jid)) - sqlitehandler.create_tables(db_file) - # await sqlitehandler.set_default_values(db_file) + create_tables(db_file) + # await set_default_values(db_file) if message: return await callback(db_file, message) else: diff --git a/slixfeed/listhandler.py b/slixfeed/listhandler.py index 462f3fa..11dcab7 100644 --- a/slixfeed/listhandler.py +++ b/slixfeed/listhandler.py @@ -15,7 +15,7 @@ TODO """ -import sqlitehandler +from sqlitehandler import get_settings_value async def add_to_list(newwords, keywords): @@ -98,7 +98,7 @@ async def is_listed(db_file, key, string): """ # async def reject(db_file, string): # async def is_blacklisted(db_file, string): - list = await sqlitehandler.get_settings_value( + list = await get_settings_value( db_file, key ) @@ -119,7 +119,7 @@ async def is_listed(db_file, key, string): This code was tested at module datahandler reject = 0 - blacklist = await sqlitehandler.get_settings_value( + blacklist = await get_settings_value( db_file, "filter-deny" ) diff --git a/slixfeed/sqlitehandler.py b/slixfeed/sqlitehandler.py index 1148e6d..5fed7c8 100644 --- a/slixfeed/sqlitehandler.py +++ b/slixfeed/sqlitehandler.py @@ -15,17 +15,18 @@ TODO """ -import sqlite3 -import asyncio - +from asyncio import Lock from bs4 import BeautifulSoup from datetime import date +# from slixfeed.confighandler import get_value_default +import confighandler as confighandler +# from slixfeed.datahandler import join_url +import datahandler as datahandler +from datetimehandler import current_time +from datetimehandler import rfc2822_to_iso8601 +from sqlite3 import connect from sqlite3 import Error -import confighandler -import datahandler -import datetimehandler - # from eliot import start_action, to_file # # with start_action(action_type="list_feeds()", db=db_file): # # with start_action(action_type="last_entries()", num=num): @@ -35,7 +36,7 @@ import datetimehandler # # with start_action(action_type="check_entry()", link=link): # aiosqlite -DBLOCK = asyncio.Lock() +DBLOCK = Lock() CURSORS = {} @@ -56,7 +57,7 @@ def create_connection(db_file): """ conn = None try: - conn = sqlite3.connect(db_file) + conn = connect(db_file) return conn except Error as e: print(e) @@ -155,9 +156,9 @@ def get_cursor(db_file): return CURSORS[db_file] -async def add_feed(db_file, url, title=None, status=None): +async def insert_feed(db_file, url, title=None, status=None): """ - Add a new feed into the feeds table. + Insert a new feed into the feeds table. Parameters ---------- @@ -879,11 +880,13 @@ async def add_entry(cur, entry): try: cur.execute(sql, entry) except: + print(await current_time(), "COROUTINE OBJECT NOW") print(entry[6]) print(type(entry[6])) print(entry) print(type(entry)) - breakpoint() + print(await current_time(), "COROUTINE OBJECT NOW") + # breakpoint() # NOTE See remove_nonexistent_entries @@ -992,7 +995,7 @@ async def remove_nonexistent_entries(db_file, feed, source): # print("compare11:", title, link, time) # print("compare22:", item[1], item[2], item[4]) # print("============") - time = await datetimehandler.rfc2822_to_iso8601(entry.published) + time = await rfc2822_to_iso8601(entry.published) if (item[1] == title and item[2] == link and item[4] == time): @@ -1401,8 +1404,8 @@ async def check_entry_exist(db_file, source, eid=None, "timestamp": date }).fetchone() except: - print(await datetimehandler.current_time(), "ERROR DATE: source =", source) - print(await datetimehandler.current_time(), "ERROR DATE: date =", date) + print(await current_time(), "ERROR DATE: source =", source) + print(await current_time(), "ERROR DATE: date =", date) else: sql = ( "SELECT id " @@ -1419,7 +1422,7 @@ async def check_entry_exist(db_file, source, eid=None, else: None except: - print(await datetimehandler.current_time(), "ERROR DATE: result =", source) + print(await current_time(), "ERROR DATE: result =", source) async def set_settings_value(db_file, key_value): diff --git a/slixfeed/taskhandler.py b/slixfeed/taskhandler.py index 35786b8..0cfe828 100644 --- a/slixfeed/taskhandler.py +++ b/slixfeed/taskhandler.py @@ -44,11 +44,16 @@ import logging import os import slixmpp -import datahandler -import datetimehandler -import filehandler -import sqlitehandler -import xmpphandler +from datahandler import download_updates +from datetimehandler import current_time +from filehandler import initdb +from filehandler import get_default_dbdir +from sqlitehandler import get_entry_unread +from sqlitehandler import get_settings_value +from sqlitehandler import get_number_of_items +from sqlitehandler import get_number_of_entries_unread +# from xmpphandler import Slixfeed +import xmpphandler as xmpphandler main_task = [] jid_tasker = {} @@ -118,7 +123,7 @@ Pass a list (or dict) of tasks to start NOTE -Consider callback e.g. xmpphandler.Slixfeed.send_status. +Consider callback e.g. Slixfeed.send_status. Or taskhandler for each protocol or specific taskhandler function. """ @@ -131,12 +136,12 @@ async def task_jid(self, jid): jid : str Jabber ID. """ - enabled = await filehandler.initdb( + enabled = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "enabled" ) - # print(await datetimehandler.current_time(), "enabled", enabled, jid) + # print(await current_time(), "enabled", enabled, jid) if enabled: # NOTE Perhaps we want to utilize super with keyword # arguments in order to know what tasks to initiate. @@ -177,7 +182,7 @@ async def task_jid(self, jid): async def send_update(self, jid, num=None): - # print(await datetimehandler.current_time(), jid, "def send_update") + # print(await current_time(), jid, "def send_update") """ Send news items as messages. @@ -190,14 +195,14 @@ async def send_update(self, jid, num=None): """ # print("Starting send_update()") # print(jid) - new = await filehandler.initdb( + new = await initdb( jid, - sqlitehandler.get_entry_unread, + get_entry_unread, num ) if new: # TODO Add while loop to assure delivery. - # print(await datetimehandler.current_time(), ">>> ACT send_message",jid) + # print(await current_time(), ">>> ACT send_message",jid) if await xmpphandler.Slixfeed.is_muc(self, jid): chat_type = "groupchat" else: @@ -216,9 +221,9 @@ async def send_update(self, jid, num=None): send_update, "interval" ) - # interval = await filehandler.initdb( + # interval = await initdb( # jid, - # sqlitehandler.get_settings_value, + # get_settings_value, # "interval" # ) # task_manager[jid]["interval"] = loop.call_at( @@ -227,8 +232,8 @@ async def send_update(self, jid, num=None): # send_update(jid) # ) - # print(await datetimehandler.current_time(), "asyncio.get_event_loop().time()") - # print(await datetimehandler.current_time(), asyncio.get_event_loop().time()) + # print(await current_time(), "asyncio.get_event_loop().time()") + # print(await current_time(), asyncio.get_event_loop().time()) # await asyncio.sleep(60 * interval) # loop.call_later( @@ -242,7 +247,7 @@ async def send_update(self, jid, num=None): async def send_status(self, jid): - # print(await datetimehandler.current_time(), jid, "def send_status") + # print(await current_time(), jid, "def send_status") """ Send status message. @@ -251,23 +256,23 @@ async def send_status(self, jid): jid : str Jabber ID. """ - # print(await datetimehandler.current_time(), "> SEND STATUS",jid) + # print(await current_time(), "> SEND STATUS",jid) status_text="🤖️ Slixfeed RSS News Bot" - enabled = await filehandler.initdb( + enabled = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "enabled" ) if not enabled: status_mode = "xa" status_text = "📫️ Send \"Start\" to receive updates" else: - feeds = await filehandler.initdb( + feeds = await initdb( jid, - sqlitehandler.get_number_of_items, + get_number_of_items, "feeds" ) - # print(await datetimehandler.current_time(), jid, "has", feeds, "feeds") + # print(await current_time(), jid, "has", feeds, "feeds") if not feeds: print(">>> not feeds:", feeds, "jid:", jid) status_mode = "available" @@ -275,9 +280,9 @@ async def send_status(self, jid): "📪️ Send a URL from a blog or a news website" ) else: - unread = await filehandler.initdb( + unread = await initdb( jid, - sqlitehandler.get_number_of_entries_unread + get_number_of_entries_unread ) if unread: status_mode = "chat" @@ -295,7 +300,7 @@ async def send_status(self, jid): status_text = "📭️ No news" # breakpoint() - # print(await datetimehandler.current_time(), status_text, "for", jid) + # print(await current_time(), status_text, "for", jid) xmpphandler.Slixfeed.send_presence( self, pshow=status_mode, @@ -332,9 +337,9 @@ async def refresh_task(self, jid, callback, key, val=None): Value. The default is None. """ if not val: - val = await filehandler.initdb( + val = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, key ) # if task_manager[jid][key]: @@ -364,7 +369,7 @@ async def refresh_task(self, jid, callback, key, val=None): # TODO Take this function out of # async def check_updates(jid): - # print(await datetimehandler.current_time(), jid, "def check_updates") + # print(await current_time(), jid, "def check_updates") """ Start calling for update check up. @@ -374,8 +379,8 @@ async def check_updates(jid): Jabber ID. """ while True: - # print(await datetimehandler.current_time(), "> CHCK UPDATE",jid) - await filehandler.initdb(jid, datahandler.download_updates) + # print(await current_time(), "> CHCK UPDATE",jid) + await initdb(jid, download_updates) await asyncio.sleep(60 * 90) # Schedule to call this function again in 90 minutes # loop.call_at( @@ -397,7 +402,7 @@ async def select_file(self): Initiate actions by JID (Jabber ID). """ while True: - db_dir = filehandler.get_default_dbdir() + db_dir = get_default_dbdir() if not os.path.isdir(db_dir): msg = ( "Slixfeed can not work without a database.\n" @@ -406,7 +411,7 @@ async def select_file(self): "Send a feed to the bot by URL:\n" "https://reclaimthenet.org/feed/" ) - # print(await datetimehandler.current_time(), msg) + # print(await current_time(), msg) print(msg) else: os.chdir(db_dir) diff --git a/slixfeed/xmpphandler.py b/slixfeed/xmpphandler.py index fbfd7be..d4383c2 100644 --- a/slixfeed/xmpphandler.py +++ b/slixfeed/xmpphandler.py @@ -51,23 +51,36 @@ NOTE import asyncio import logging -import os +# import os import slixmpp from random import randrange - +from datahandler import add_feed +from datahandler import add_feed_no_check +from datahandler import feed_to_http +from datahandler import view_entry +from datahandler import view_feed +from datetimehandler import current_time +from filehandler import initdb +from listhandler import add_to_list +from listhandler import remove_from_list +from sqlitehandler import get_settings_value +from sqlitehandler import set_settings_value +from sqlitehandler import mark_source_as_read +from sqlitehandler import last_entries +from sqlitehandler import list_feeds +from sqlitehandler import remove_feed +from sqlitehandler import search_feeds +from sqlitehandler import statistics +from sqlitehandler import toggle_status +from taskhandler import clean_tasks_xmpp +from taskhandler import start_tasks_xmpp +from taskhandler import refresh_task +from taskhandler import send_status +from taskhandler import send_update from slixmpp.plugins.xep_0363.http_upload import FileTooBig, HTTPError, UploadServiceNotFound # from slixmpp.plugins.xep_0402 import BookmarkStorage, Conference from slixmpp.plugins.xep_0048.stanza import Bookmarks -# TODO datahandler -> slixfeed.datahandler -# See project feed2toot -import datahandler -import datetimehandler -import filehandler -import listhandler -import sqlitehandler -import taskhandler - import xmltodict import xml.etree.ElementTree as ET from lxml import etree @@ -170,24 +183,24 @@ class Slixfeed(slixmpp.ClientXMPP): # print("def presence_available", presence["from"].bare) if presence["from"].bare not in self.boundjid.bare: jid = presence["from"].bare - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["interval", "status", "check"] ) - await taskhandler.start_tasks_xmpp( + await start_tasks_xmpp( self, jid, ["interval", "status", "check"] ) - # await taskhandler.task_jid(self, jid) - # main_task.extend([asyncio.create_task(taskhandler.task_jid(jid))]) + # await task_jid(self, jid) + # main_task.extend([asyncio.create_task(task_jid(jid))]) # print(main_task) async def stop_tasks(self, presence): if not self.boundjid.bare: jid = presence["from"].bare print(">>> unavailable:", jid) - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["interval", "status", "check"] ) @@ -224,16 +237,16 @@ class Slixfeed(slixmpp.ClientXMPP): async def join_muc(self, inviter, muc_jid): - token = await filehandler.initdb( + token = await initdb( muc_jid, - sqlitehandler.get_settings_value, + get_settings_value, "token" ) if token != "accepted": token = randrange(10000, 99999) - await filehandler.initdb( + await initdb( muc_jid, - sqlitehandler.set_settings_value, + set_settings_value, ["token", token] ) self.send_message( @@ -325,14 +338,14 @@ class Slixfeed(slixmpp.ClientXMPP): async def on_session_end(self, event): - print(await datetimehandler.current_time(), "Session ended. Attempting to reconnect.") + print(await current_time(), "Session ended. Attempting to reconnect.") print(event) logging.warning("Session ended. Attempting to reconnect.") await self.recover_connection(event) async def on_connection_failed(self, event): - print(await datetimehandler.current_time(), "Connection failed. Attempting to reconnect.") + print(await current_time(), "Connection failed. Attempting to reconnect.") print(event) logging.warning("Connection failed. Attempting to reconnect.") await self.recover_connection(event) @@ -343,7 +356,7 @@ class Slixfeed(slixmpp.ClientXMPP): # if self.connection_attempts <= self.max_connection_attempts: # self.reconnect(wait=5.0) # wait a bit before attempting to reconnect # else: - # print(await datetimehandler.current_time(),"Maximum connection attempts exceeded.") + # print(await current_time(),"Maximum connection attempts exceeded.") # logging.error("Maximum connection attempts exceeded.") print("Attempt:", self.connection_attempts) self.reconnect(wait=5.0) @@ -377,10 +390,10 @@ class Slixfeed(slixmpp.ClientXMPP): print(message) if message["type"] in ("chat", "normal"): jid = message["from"].bare - await taskhandler.refresh_task( + await refresh_task( self, jid, - taskhandler.send_status, + send_status, "status", 20 ) @@ -409,11 +422,11 @@ class Slixfeed(slixmpp.ClientXMPP): jid = presence["from"].bare if presence["show"] in ("away", "dnd", "xa"): print(">>> away, dnd, xa:", jid) - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["interval"] ) - await taskhandler.start_tasks_xmpp( + await start_tasks_xmpp( self, jid, ["status", "check"] @@ -450,7 +463,7 @@ class Slixfeed(slixmpp.ClientXMPP): # Deprecated in favour of event "presence_available" # if not main_task: - # await taskhandler.select_file() + # await select_file() async def is_muc(self, jid): @@ -583,15 +596,15 @@ class Slixfeed(slixmpp.ClientXMPP): if (msg['muc']['nick'] == "Slixfeed (RSS News Bot)" or not msg["body"].startswith("!")): return - token = await filehandler.initdb( + token = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "token" ) if token == "accepted": - operator = await filehandler.initdb( + operator = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "masters" ) if operator: @@ -600,17 +613,17 @@ class Slixfeed(slixmpp.ClientXMPP): # # Begin processing new JID # # Deprecated in favour of event "presence_available" - # db_dir = filehandler.get_default_dbdir() + # db_dir = get_default_dbdir() # os.chdir(db_dir) # if jid + ".db" not in os.listdir(): - # await taskhandler.task_jid(jid) + # await task_jid(jid) message = " ".join(msg["body"].split()) if msg["type"] == "groupchat": message = message[1:] message_lowercase = message.lower() - print(await datetimehandler.current_time(), "ACCOUNT: " + str(msg["from"])) - print(await datetimehandler.current_time(), "COMMAND:", message) + print(await current_time(), "ACCOUNT: " + str(msg["from"])) + print(await current_time(), "COMMAND:", message) match message_lowercase: case "help": @@ -639,20 +652,20 @@ class Slixfeed(slixmpp.ClientXMPP): case _ if message_lowercase.startswith("activate"): if msg["type"] == "groupchat": acode = message[9:] - token = await filehandler.initdb( + token = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "token" ) if int(acode) == token: - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, ["masters", nick] ) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, ["token", "accepted"] ) action = "{}, your are in command.".format(nick) @@ -665,31 +678,31 @@ class Slixfeed(slixmpp.ClientXMPP): url = message.split(" ")[0] title = " ".join(message.split(" ")[1:]) if url.startswith("http"): - action = await filehandler.initdb( + action = await initdb( jid, - datahandler.add_feed_no_check, + add_feed_no_check, [url, title] ) - old = await filehandler.initdb( + old = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "old" ) if old: - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["status"] ) - # await taskhandler.send_status(jid) - await taskhandler.start_tasks_xmpp( + # await send_status(jid) + await start_tasks_xmpp( self, jid, ["status"] ) else: - await filehandler.initdb( + await initdb( jid, - sqlitehandler.mark_source_as_read, + mark_source_as_read, url ) else: @@ -698,18 +711,18 @@ class Slixfeed(slixmpp.ClientXMPP): key = "filter-" + message[:5] val = message[7:] if val: - keywords = await filehandler.initdb( + keywords = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, key ) - val = await listhandler.add_to_list( + val = await add_to_list( val, keywords ) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) action = ( @@ -722,18 +735,18 @@ class Slixfeed(slixmpp.ClientXMPP): key = "filter-" + message[:5] val = message[7:] if val: - keywords = await filehandler.initdb( + keywords = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, key ) - val = await listhandler.remove_from_list( + val = await remove_from_list( val, keywords ) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) action = ( @@ -746,18 +759,18 @@ class Slixfeed(slixmpp.ClientXMPP): key = "filter-" + message[:4] val = message[6:] if val: - keywords = await filehandler.initdb( + keywords = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, key ) - val = await listhandler.add_to_list( + val = await add_to_list( val, keywords ) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) action = ( @@ -770,18 +783,18 @@ class Slixfeed(slixmpp.ClientXMPP): key = "filter-" + message[:4] val = message[6:] if val: - keywords = await filehandler.initdb( + keywords = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, key ) - val = await listhandler.remove_from_list( + val = await remove_from_list( val, keywords ) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) action = ( @@ -797,8 +810,8 @@ class Slixfeed(slixmpp.ClientXMPP): message_lowercase.startswith("feed:")): url = message if url.startswith("feed:"): - url = await datahandler.feed_to_http(url) - await taskhandler.clean_tasks_xmpp( + url = await feed_to_http(url) + await clean_tasks_xmpp( jid, ["status"] ) @@ -806,55 +819,55 @@ class Slixfeed(slixmpp.ClientXMPP): "📫️ Processing request to fetch data from {}" ).format(url) process_task_message(self, jid, task) - action = await filehandler.initdb( + action = await initdb( jid, - datahandler.add_feed, + add_feed, url ) - await taskhandler.start_tasks_xmpp( + await start_tasks_xmpp( self, jid, ["status"] ) # action = "> " + message + "\n" + action # FIXME Make the taskhandler to update status message - # await taskhandler.refresh_task( + # await refresh_task( # self, # jid, - # taskhandler.send_status, + # send_status, # "status", # 20 # ) # NOTE This would show the number of new unread entries - old = await filehandler.initdb( + old = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, "old" ) if old: - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["status"] ) - # await taskhandler.send_status(jid) - await taskhandler.start_tasks_xmpp( + # await send_status(jid) + await start_tasks_xmpp( self, jid, ["status"] ) else: - await filehandler.initdb( + await initdb( jid, - sqlitehandler.mark_source_as_read, + mark_source_as_read, url ) case _ if message_lowercase.startswith("feeds"): query = message[6:] if query: if len(query) > 3: - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.search_feeds, + search_feeds, query ) else: @@ -862,9 +875,9 @@ class Slixfeed(slixmpp.ClientXMPP): "Enter at least 4 characters to search" ) else: - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.list_feeds + list_feeds ) case "goodbye": if msg["type"] == "groupchat": @@ -883,17 +896,17 @@ class Slixfeed(slixmpp.ClientXMPP): # action = ( # "Updates will be sent every {} minutes." # ).format(action) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) # NOTE Perhaps this should be replaced # by functions clean and start - await taskhandler.refresh_task( + await refresh_task( self, jid, - taskhandler.send_update, + send_update, key, val ) @@ -909,9 +922,9 @@ class Slixfeed(slixmpp.ClientXMPP): key = message[:6] val = message[7:] if val: - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) if val == 0: @@ -928,18 +941,18 @@ class Slixfeed(slixmpp.ClientXMPP): key = message[:7] val = message[11:] if val: - names = await filehandler.initdb( + names = await initdb( jid, - sqlitehandler.get_settings_value, + get_settings_value, key ) - val = await listhandler.add_to_list( + val = await add_to_list( val, names ) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) action = ( @@ -949,9 +962,9 @@ class Slixfeed(slixmpp.ClientXMPP): else: action = "Missing value." case "new": - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, ["old", 0] ) action = ( @@ -959,34 +972,34 @@ class Slixfeed(slixmpp.ClientXMPP): ) case _ if message_lowercase.startswith("next"): num = message[5:] - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["interval", "status"] ) - await taskhandler.start_tasks_xmpp( + await start_tasks_xmpp( self, jid, ["interval", "status"] ) - # await taskhandler.refresh_task( + # await refresh_task( # self, # jid, - # taskhandler.send_update, + # send_update, # "interval", # num # ) - # await taskhandler.refresh_task( + # await refresh_task( # self, # jid, - # taskhandler.send_status, + # send_status, # "status", # 20 # ) - # await taskhandler.refresh_task(jid, key, val) + # await refresh_task(jid, key, val) case "old": - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, ["old", 1] ) action = ( @@ -999,9 +1012,9 @@ class Slixfeed(slixmpp.ClientXMPP): # action = ( # "Every update will contain {} news items." # ).format(action) - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) action = ( @@ -1019,22 +1032,22 @@ class Slixfeed(slixmpp.ClientXMPP): "📫️ Processing request to fetch data from {}" ).format(url) process_task_message(self, jid, task) - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["status"] ) if url.startswith("feed:"): - url = await datahandler.feed_to_http(url) + url = await feed_to_http(url) match len(data): case 1: if url.startswith("http"): - action = await datahandler.view_feed(url) + action = await view_feed(url) else: action = "Missing URL." case 2: num = data[1] if url.startswith("http"): - action = await datahandler.view_entry(url, num) + action = await view_entry(url, num) else: action = "Missing URL." case _: @@ -1043,7 +1056,7 @@ class Slixfeed(slixmpp.ClientXMPP): "`read URL` or `read URL NUMBER`\n" "URL must not contain white space." ) - await taskhandler.start_tasks_xmpp( + await start_tasks_xmpp( self, jid, ["status"] @@ -1051,9 +1064,9 @@ class Slixfeed(slixmpp.ClientXMPP): case _ if message_lowercase.startswith("recent"): num = message[7:] if num: - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.last_entries, + last_entries, num ) else: @@ -1061,23 +1074,23 @@ class Slixfeed(slixmpp.ClientXMPP): case _ if message_lowercase.startswith("remove"): ix = message[7:] if ix: - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.remove_feed, + remove_feed, ix ) - # await taskhandler.refresh_task( + # await refresh_task( # self, # jid, - # taskhandler.send_status, + # send_status, # "status", # 20 # ) - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["status"] ) - await taskhandler.start_tasks_xmpp( + await start_tasks_xmpp( self, jid, ["status"] @@ -1088,9 +1101,9 @@ class Slixfeed(slixmpp.ClientXMPP): query = message[7:] if query: if len(query) > 1: - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.search_entries, + search_entries, query ) else: @@ -1103,30 +1116,30 @@ class Slixfeed(slixmpp.ClientXMPP): # action = "Updates are enabled." key = "enabled" val = 1 - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) - # asyncio.create_task(taskhandler.task_jid(self, jid)) - await taskhandler.start_tasks_xmpp( + # asyncio.create_task(task_jid(self, jid)) + await start_tasks_xmpp( self, jid, ["interval", "status", "check"] ) action = "Updates are enabled." - # print(await datetimehandler.current_time(), "task_manager[jid]") + # print(await current_time(), "task_manager[jid]") # print(task_manager[jid]) case "stats": - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.statistics + statistics ) case _ if message_lowercase.startswith("status "): ix = message[7:] - action = await filehandler.initdb( + action = await initdb( jid, - sqlitehandler.toggle_status, + toggle_status, ix ) case "stop": @@ -1142,23 +1155,23 @@ class Slixfeed(slixmpp.ClientXMPP): # task_manager[jid]["interval"].cancel() # key = "enabled" # val = 0 - # action = await filehandler.initdb( + # action = await initdb( # jid, - # sqlitehandler.set_settings_value, + # set_settings_value, # [key, val] # ) # except: # action = "Updates are already disabled." # # print("Updates are already disabled. Nothing to do.") - # # await taskhandler.send_status(jid) + # # await send_status(jid) key = "enabled" val = 0 - await filehandler.initdb( + await initdb( jid, - sqlitehandler.set_settings_value, + set_settings_value, [key, val] ) - await taskhandler.clean_tasks_xmpp( + await clean_tasks_xmpp( jid, ["interval", "status"] )