From a3d68f6a8f8824fd5723951d2a7cdfc2ca55384b Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Wed, 3 Jan 2024 10:37:33 +0000 Subject: [PATCH] Add functionality to export feeds to OPML --- slixfeed/__main__.py | 3 ++ slixfeed/config.py | 38 +++++++++++--------- slixfeed/datetime.py | 18 ++++++++-- slixfeed/opml.py | 75 +++++++++++++++++++++++++++++++++++++++ slixfeed/sqlite.py | 6 ++-- slixfeed/task.py | 34 ++++++++---------- slixfeed/xmpp/bookmark.py | 8 ----- slixfeed/xmpp/connect.py | 5 ++- slixfeed/xmpp/muc.py | 4 +-- slixfeed/xmpp/process.py | 69 +++++++++++++++++++++++++++++------ slixfeed/xmpp/upload.py | 35 ++++++++++++++++++ 11 files changed, 233 insertions(+), 62 deletions(-) create mode 100644 slixfeed/opml.py create mode 100644 slixfeed/xmpp/upload.py diff --git a/slixfeed/__main__.py b/slixfeed/__main__.py index 89d2b42..8275b24 100644 --- a/slixfeed/__main__.py +++ b/slixfeed/__main__.py @@ -106,8 +106,11 @@ class Jabber: xmpp.register_plugin('xep_0048') # Bookmarks xmpp.register_plugin('xep_0060') # Publish-Subscribe # xmpp.register_plugin('xep_0065') # SOCKS5 Bytestreams + xmpp.register_plugin('xep_0066') # Out of Band Data + xmpp.register_plugin('xep_0071') # XHTML-IM xmpp.register_plugin('xep_0199', {'keepalive': True}) # XMPP Ping xmpp.register_plugin('xep_0249') # Multi-User Chat + xmpp.register_plugin('xep_0363') # HTTP File Upload xmpp.register_plugin('xep_0402') # PEP Native Bookmarks # proxy_enabled = get_value("accounts", "XMPP Connect", "proxy_enabled") diff --git a/slixfeed/config.py b/slixfeed/config.py index 4bd3016..6986ee8 100644 --- a/slixfeed/config.py +++ b/slixfeed/config.py @@ -5,30 +5,25 @@ TODO -1) Use file settings.csv and pathnames.txt instead: - See get_value_default and get_default_list +1) Website-specific filter (i.e. audiobookbay). -2) Website-specific filter (i.e. audiobookbay). +2) Exclude websites from being subjected to filtering (e.g. metapedia). -3) Exclude websites from filtering (e.g. metapedia). - -4) Filter phrases: +3) Filter phrases: Refer to sqlitehandler.search_entries for implementation. It is expected to be more complex than function search_entries. -5) Copy file from /etc/slixfeed/ or /usr/share/slixfeed/ +4) Copy file from /etc/slixfeed/ or /usr/share/slixfeed/ """ import configparser -# from file import get_default_confdir -import slixfeed.config as config import slixfeed.sqlite as sqlite import os # from random import randrange import sys import yaml - +import logging def get_value(filename, section, keys): """ @@ -48,8 +43,9 @@ def get_value(filename, section, keys): result : list or str A single value as string or multiple values as list. """ + result = None config_res = configparser.RawConfigParser() - config_dir = config.get_default_confdir() + config_dir = get_default_confdir() # if not os.path.isdir(config_dir): # config_dir = '/usr/share/slixfeed/' if not os.path.isdir(config_dir): @@ -63,18 +59,28 @@ def get_value(filename, section, keys): for key in keys: try: value = section_res[key] + logging.debug("Found value {} for key {}".format( + value, key)) except: - print("Missing key:", key) value = '' + logging.error("Missing key:", key) result.extend([value]) elif isinstance(keys, str): key = keys try: result = section_res[key] + logging.debug("Found value {} for key {}".format( + value, key)) except: - print("Missing key:", key) result = '' - return result + # logging.error("Missing key:", key) + if result == None: + logging.error( + "Check configuration file {} for " + "missing key {} under section {}.".format( + filename, section, keys)) + else: + return result # TODO Store config file as an object in runtime, otherwise @@ -96,7 +102,7 @@ def get_value_default(filename, section, key): Value. """ config_res = configparser.RawConfigParser() - config_dir = config.get_default_confdir() + config_dir = get_default_confdir() if not os.path.isdir(config_dir): config_dir = '/usr/share/slixfeed/' config_file = os.path.join(config_dir, filename + ".ini") @@ -120,7 +126,7 @@ def get_list(filename): result : list List of pathnames or keywords. """ - config_dir = config.get_default_confdir() + config_dir = get_default_confdir() if not os.path.isdir(config_dir): config_dir = '/usr/share/slixfeed/' config_file = os.path.join(config_dir, filename) diff --git a/slixfeed/datetime.py b/slixfeed/datetime.py index de18233..eb7cea2 100644 --- a/slixfeed/datetime.py +++ b/slixfeed/datetime.py @@ -15,7 +15,7 @@ def now(): Returns ------- - date : ? + date : ??? ISO 8601 Timestamp. """ date = datetime.now().isoformat() @@ -28,7 +28,7 @@ def current_time(): Returns ------- - date : ? + date : str HH:MM:SS timestamp. """ now = datetime.now() @@ -36,6 +36,20 @@ def current_time(): return time +def timestamp(): + """ + Print time stamp to be used in filename. + + Returns + ------- + formatted_time : str + %Y%m%d-%H%M%S timestamp. + """ + now = datetime.now() + formatted_time = now.strftime("%Y%m%d-%H%M%S") + return formatted_time + + def validate(date): """ Validate date format. diff --git a/slixfeed/opml.py b/slixfeed/opml.py new file mode 100644 index 0000000..527ceb1 --- /dev/null +++ b/slixfeed/opml.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" + +{ + 'bozo': False, + 'bozo_exception': None, + 'feeds': [ + { + 'url': 'https://kurtmckee.org/tag/listparser/feed', + 'title': 'listparser blog', + 'categories': [], + 'tags': [] + }, + { + 'url': 'https://github.com/kurtmckee/listparser/commits/develop.atom', + 'title': 'listparser changelog', + 'categories': [], + 'tags': [] + } + ], + 'lists': [], + 'opportunities': [], + 'meta': { + 'title': 'listparser project feeds', + 'author': { + 'name': 'Kurt McKee', + 'email': 'contactme@kurtmckee.org', + 'url': 'https://kurtmckee.org/' + } + }, + 'version': 'opml2' + } + +""" + +import listparser +import lxml + + +async def import_from_file(db_file, opml_doc): + feeds = listparser.parse(opml_doc)['feeds'] + for feed in feeds: + url = feed['url'] + title = feed['title'] + # categories = feed['categories'] + # tags = feed['tags'] + # await datahandler.add_feed_no_check(db_file, [url, title]) + +from slixfeed.datetime import current_time +import xml.etree.ElementTree as ET + +# NOTE Use OPyML or LXML +async def export_to_file(jid, filename, results): + root = ET.Element("opml") + root.set("version", "1.0") + head = ET.SubElement(root, "head") + ET.SubElement(head, "title").text = "Subscriptions for {}".format(jid) + ET.SubElement(head, "description").text = ( + "Set of feeds exported with Slixfeed.") + ET.SubElement(head, "generator").text = "Slixfeed" + ET.SubElement(head, "urlPublic").text = ( + "https://gitgud.io/sjehuda/slixfeed") + time_stamp = current_time() + ET.SubElement(head, "dateCreated").text = time_stamp + ET.SubElement(head, "dateModified").text = time_stamp + body = ET.SubElement(root, "body") + for result in results: + outline = ET.SubElement(body, "outline") + outline.set("text", result[0]) + outline.set("xmlUrl", result[1]) + # outline.set("type", result[2]) + tree = ET.ElementTree(root) + tree.write(filename) diff --git a/slixfeed/sqlite.py b/slixfeed/sqlite.py index 270e15c..427ef91 100644 --- a/slixfeed/sqlite.py +++ b/slixfeed/sqlite.py @@ -836,9 +836,9 @@ async def add_entry(cur, entry): try: cur.execute(sql, entry) except: - print("") - # print("Unknown error for sqlite.add_entry") - # print(entry) + # None + print("Unknown error for sqlite.add_entry") + print(entry) # print(current_time(), "COROUTINE OBJECT NOW") # for i in entry: # print(type(i)) diff --git a/slixfeed/task.py b/slixfeed/task.py index c66f711..618061e 100644 --- a/slixfeed/task.py +++ b/slixfeed/task.py @@ -86,7 +86,7 @@ await taskhandler.start_tasks( """ async def start_tasks_xmpp(self, jid, tasks): - # print("start_tasks_xmpp", jid, tasks) + logging.debug("Starting tasks {} for JID {}".format(tasks, jid)) task_manager[jid] = {} for task in tasks: # print("task:", task) @@ -115,13 +115,14 @@ async def start_tasks_xmpp(self, jid, tasks): async def clean_tasks_xmpp(jid, tasks): - # print("clean_tasks_xmpp", jid, tasks) + logging.debug("Stopping tasks {} for JID {}".format(tasks, jid)) for task in tasks: # if task_manager[jid][task]: try: task_manager[jid][task].cancel() except: - print("No task", task, "for JID", jid, "(clean_tasks)") + logging.debug( + "No task {} for JID {} (clean_tasks)".format(task, jid)) """ @@ -138,7 +139,6 @@ Consider callback e.g. Slixfeed.send_status. Or taskhandler for each protocol or specific taskhandler function. """ async def task_jid(self, jid): - # print("task_jid", jid) """ JID (Jabber ID) task manager. @@ -149,7 +149,6 @@ async def task_jid(self, jid): """ db_file = get_pathname_to_database(jid) enabled = await get_settings_value(db_file, "enabled") - # print(await current_time(), "enabled", enabled, jid) if enabled: # NOTE Perhaps we want to utilize super with keyword # arguments in order to know what tasks to initiate. @@ -187,8 +186,6 @@ async def task_jid(self, jid): async def send_update(self, jid, num=None): - print("send_update", jid) - # print(await current_time(), jid, "def send_update") """ Send news items as messages. @@ -199,8 +196,7 @@ async def send_update(self, jid, num=None): num : str, optional Number. The default is None. """ - # print("Starting send_update()") - # print(jid) + logging.debug("Sending a news update to JID {}".format(jid)) db_file = get_pathname_to_database(jid) enabled = await get_settings_value(db_file, "enabled") if enabled: @@ -260,8 +256,6 @@ async def send_update(self, jid, num=None): async def send_status(self, jid): - # print("send_status", jid) - # print(await current_time(), jid, "def send_status") """ Send status message. @@ -270,7 +264,7 @@ async def send_status(self, jid): jid : str Jabber ID. """ - # print(await current_time(), "> SEND STATUS",jid) + logging.debug("Sending a status message to JID {}".format(jid)) status_text="🤖️ Slixfeed RSS News Bot" db_file = get_pathname_to_database(jid) enabled = await get_settings_value(db_file, "enabled") @@ -323,7 +317,6 @@ async def send_status(self, jid): async def refresh_task(self, jid, callback, key, val=None): - # print("refresh_task", jid, key) """ Apply new setting at runtime. @@ -336,6 +329,7 @@ async def refresh_task(self, jid, callback, key, val=None): val : str, optional Value. The default is None. """ + logging.debug("Refreshing task {} for JID {}".format(callback, jid)) if not val: db_file = get_pathname_to_database(jid) val = await get_settings_value(db_file, key) @@ -344,7 +338,9 @@ async def refresh_task(self, jid, callback, key, val=None): try: task_manager[jid][key].cancel() except: - print("No task of type", key, "to cancel for JID", jid) + logging.debug( + "No task of type {} to cancel for " + "JID {} (clean_tasks)").format(key, jid) # task_manager[jid][key] = loop.call_at( # loop.time() + 60 * float(val), # loop.create_task, @@ -374,8 +370,6 @@ async def wait_and_run(self, callback, jid, val): # TODO Take this function out of # async def check_updates(jid): - # print("check_updates", jid) - # print(await current_time(), jid, "def check_updates") """ Start calling for update check up. @@ -384,8 +378,8 @@ async def check_updates(jid): jid : str Jabber ID. """ + logging.debug("Scanning for updates for JID {}".format(jid)) while True: - # print(await current_time(), "> CHCK UPDATE",jid) db_file = get_pathname_to_database(jid) await download_updates(db_file) val = get_value_default("settings", "Settings", "check") @@ -399,8 +393,8 @@ async def check_updates(jid): async def start_tasks(self, presence): - # print("def presence_available", presence["from"].bare) jid = presence["from"].bare + logging.debug("Beginning tasks for JID {}".format(jid)) if jid not in self.boundjid.bare: await clean_tasks_xmpp( jid, ["interval", "status", "check"]) @@ -414,7 +408,7 @@ async def start_tasks(self, presence): async def stop_tasks(self, presence): if not self.boundjid.bare: jid = presence["from"].bare - print(">>> unavailable:", jid) + logging.debug("Stopping tasks for JID {}".format(jid)) await clean_tasks_xmpp( jid, ["interval", "status", "check"]) @@ -440,7 +434,7 @@ async def check_readiness(self, presence): jid = presence["from"].bare if presence["show"] in ("away", "dnd", "xa"): - print(">>> away, dnd, xa:", jid) + logging.debug("Stopping updates for JID {}".format(jid)) await clean_tasks_xmpp( jid, ["interval"]) await start_tasks_xmpp( diff --git a/slixfeed/xmpp/bookmark.py b/slixfeed/xmpp/bookmark.py index 12cd5ac..2990576 100644 --- a/slixfeed/xmpp/bookmark.py +++ b/slixfeed/xmpp/bookmark.py @@ -1,14 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -""" - -TODO - -1) Function to list bookmarks - -""" - from slixmpp.plugins.xep_0048.stanza import Bookmarks diff --git a/slixfeed/xmpp/connect.py b/slixfeed/xmpp/connect.py index 831b5b2..5765e9e 100644 --- a/slixfeed/xmpp/connect.py +++ b/slixfeed/xmpp/connect.py @@ -17,7 +17,10 @@ async def recover_connection(self, event, message): # print(current_time(),"Maximum connection attempts exceeded.") # logging.error("Maximum connection attempts exceeded.") print(current_time(), "Attempt number", self.connection_attempts) - seconds = int(get_value("accounts", "XMPP Connect", "reconnect_timeout")) + seconds = get_value("accounts", "XMPP Connect", "reconnect_timeout") + if not seconds: + seconds = 30 + seconds = int(seconds) print(current_time(), "Next attempt within", seconds, "seconds") # NOTE asyncio.sleep doesn't interval as expected # await asyncio.sleep(seconds) diff --git a/slixfeed/xmpp/muc.py b/slixfeed/xmpp/muc.py index 8c3d4ba..3bc8803 100644 --- a/slixfeed/xmpp/muc.py +++ b/slixfeed/xmpp/muc.py @@ -12,7 +12,7 @@ TODO 3) If groupchat error is received, send that error message to inviter. """ - +import logging import slixfeed.xmpp.bookmark as bookmark import slixfeed.xmpp.process as process from slixfeed.datetime import current_time @@ -38,7 +38,7 @@ async def autojoin(self, event): for conference in conferences: if conference["autojoin"]: muc_jid = conference["jid"] - print(current_time(), "Autojoining groupchat", muc_jid) + logging.debug("Autojoin groupchat", muc_jid) self.plugin['xep_0045'].join_muc( muc_jid, self.nick, diff --git a/slixfeed/xmpp/process.py b/slixfeed/xmpp/process.py index fbfa435..fc08dbd 100644 --- a/slixfeed/xmpp/process.py +++ b/slixfeed/xmpp/process.py @@ -25,8 +25,9 @@ from slixfeed.config import ( get_value, get_pathname_to_database, remove_from_list) -from slixfeed.datetime import current_time +from slixfeed.datetime import current_time, timestamp import slixfeed.fetch as fetcher +import slixfeed.opml as opml import slixfeed.sqlite as sqlite import slixfeed.task as task import slixfeed.utility as utility @@ -36,6 +37,7 @@ import slixfeed.xmpp.compose as compose import slixfeed.xmpp.muc as groupchat import slixfeed.xmpp.status as status import slixfeed.xmpp.text as text +import slixfeed.xmpp.upload as upload async def event(self, event): @@ -76,8 +78,6 @@ async def message(self, message): for stanza objects and the Message stanza to see how it may be used. """ - # print("message") - # print(message) if message["type"] in ("chat", "groupchat", "normal"): jid = message["from"].bare if message["type"] == "groupchat": @@ -325,12 +325,46 @@ async def message(self, message): else: response = "Missing keywords." send_reply_message(self, message, response) + case _ if message_lowercase.startswith("export "): + valid = 1 + key = message_text[7:] + data_dir = get_default_dbdir() + if not os.path.isdir(data_dir): + os.mkdir(data_dir) + filename = os.path.join( + data_dir, "opml", "slixfeed_" + timestamp() + "." + key) + db_file = get_pathname_to_database(jid) + results = await sqlite.get_feeds(db_file) + match key: + case "opml": + status_type = "dnd" + status_message = ( + "📂️ Procesing request to export feeds into OPML ...") + send_status_message(self, jid, status_type, status_message) + await opml.export_to_file( + jid, filename, results) + url = await upload.start(self, jid, filename) + response = ( + "Feeds exported successfully to an OPML " + "Outline Syndication.\n{}").format(url) + await task.start_tasks_xmpp(self, jid, ["status"]) + case "html": + response = "Not yet implemented." + case "markdown": + response = "Not yet implemented" + case _: + response = "Unsupported filetype." + valid = 0 + if valid: + # send_oob_reply_message(message, url, response) + send_oob_message(self, jid, url) + send_reply_message(self, message, response) case _ if (message_lowercase.startswith("gemini") or - message_lowercase.startswith("gopher:")): + message_lowercase.startswith("gopher")): response = "Gemini and Gopher are not supported yet." send_reply_message(self, message, response) case _ if (message_lowercase.startswith("http") or - message_lowercase.startswith("feed:")): + message_lowercase.startswith("feed")): url = message_text await task.clean_tasks_xmpp(jid, ["status"]) status_type = "dnd" @@ -713,14 +747,14 @@ async def message(self, message): # if response: message.reply(response).send() if not response: response = "EMPTY MESSAGE - ACTION ONLY" - log_dir = get_default_dbdir() - if not os.path.isdir(log_dir): - os.mkdir(log_dir) + data_dir = get_default_dbdir() + if not os.path.isdir(data_dir): + os.mkdir(data_dir) utility.log_as_markdown( - current_time(), os.path.join(log_dir, jid), + current_time(), os.path.join(data_dir, "logs", jid), jid, message_text) utility.log_as_markdown( - current_time(), os.path.join(log_dir, jid), + current_time(), os.path.join(data_dir, "logs", jid), self.boundjid.bare, response) @@ -735,6 +769,21 @@ def send_reply_message(self, message, response): message.reply(response).send() +def send_oob_reply_message(message, url, response): + reply = message.reply(response) + reply['oob']['url'] = url + reply.send() + + +def send_oob_message(self, jid, url): + html = ( + f'' + f'{url}') + message = self.make_message(mto=jid, mbody=url, mhtml=html) + message['oob']['url'] = url + message.send() + + # def greet(self, jid, chat_type="chat"): # messages = [ # "Greetings!", diff --git a/slixfeed/xmpp/upload.py b/slixfeed/xmpp/upload.py new file mode 100644 index 0000000..ce6eeb7 --- /dev/null +++ b/slixfeed/xmpp/upload.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Based on http_upload.py example from project slixmpp +https://codeberg.org/poezio/slixmpp/src/branch/master/examples/http_upload.py +""" + +import logging +from slixmpp.exceptions import IqTimeout +# import sys + + +async def start(self, jid, filename, domain=None): + logging.info('Uploading file %s...', filename) + try: + upload_file = self['xep_0363'].upload_file + # if self.encrypted and not self['xep_0454']: + # print( + # 'The xep_0454 module isn\'t available. ' + # 'Ensure you have \'cryptography\' ' + # 'from extras_require installed.', + # file=sys.stderr, + # ) + # return + # elif self.encrypted: + # upload_file = self['xep_0454'].upload_file + url = await upload_file( + filename, domain, timeout=10, + ) + except IqTimeout: + raise TimeoutError('Could not send message in time') + logging.info('Upload success!') + logging.info('Sending file to %s', jid) + return url