From bcbbf1ab0465e09ef512b4b81c66f66b6ca529b2 Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Sun, 14 Apr 2024 12:56:45 +0000 Subject: [PATCH] Add functionality for handling with PubSub nodes; Add functionality to submit items from database to PubSub nodes; Change the fashion by which items are sent; Fix minor issues. --- slixfeed/action.py | 447 +++++++--- slixfeed/assets/about.toml | 148 +++- slixfeed/assets/feeds.toml | 48 +- slixfeed/sqlite.py | 124 ++- slixfeed/task.py | 4 +- slixfeed/version.py | 4 +- slixfeed/xmpp/bookmark.py | 2 +- slixfeed/xmpp/chat.py | 1569 ++++++++++++++++++++++++++++++++++++ slixfeed/xmpp/client.py | 849 ++++++++++++++----- slixfeed/xmpp/component.py | 4 +- slixfeed/xmpp/process.py | 1555 ----------------------------------- slixfeed/xmpp/publish.py | 63 +- 12 files changed, 2853 insertions(+), 1964 deletions(-) create mode 100644 slixfeed/xmpp/chat.py delete mode 100644 slixfeed/xmpp/process.py diff --git a/slixfeed/action.py b/slixfeed/action.py index 8717285..cb9552e 100644 --- a/slixfeed/action.py +++ b/slixfeed/action.py @@ -28,6 +28,7 @@ TODO from asyncio.exceptions import IncompleteReadError from bs4 import BeautifulSoup from feedparser import parse +import hashlib from http.client import IncompleteRead import json from slixfeed.log import Logger @@ -39,7 +40,6 @@ import slixfeed.crawl as crawl import slixfeed.dt as dt import slixfeed.fetch as fetch import slixfeed.sqlite as sqlite -import slixfeed.url as uri from slixfeed.url import ( complete_url, join_url, @@ -56,10 +56,11 @@ from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.publish import XmppPubsub from slixfeed.xmpp.upload import XmppUpload from slixfeed.xmpp.utility import get_chat_type +from slixmpp.xmlstream import ET import sys from urllib import error from urllib.parse import parse_qs, urlsplit -import xml.etree.ElementTree as ET +import xml.etree.ElementTree as ETR try: import tomllib @@ -174,10 +175,7 @@ async def xmpp_send_status_message(self, jid): jid_file = jid.replace('/', '_') db_file = config.get_pathname_to_database(jid_file) enabled = Config.get_setting_value(self.settings, jid, 'enabled') - if not enabled: - status_mode = 'xa' - status_text = '📪️ Send "Start" to receive updates' - else: + if enabled: jid_task = self.pending_tasks[jid] if len(jid_task): status_mode = 'dnd' @@ -202,7 +200,9 @@ async def xmpp_send_status_message(self, jid): else: status_mode = 'available' status_text = '📭️ No news' - + else: + status_mode = 'xa' + status_text = '📪️ Send "Start" to receive updates' # breakpoint() # print(await current_time(), status_text, "for", jid) XmppPresence.send(self, jid, status_text, status_type=status_mode) @@ -215,69 +215,267 @@ async def xmpp_send_status_message(self, jid): # ) -async def xmpp_send_pubsub(self, jid_bare, num=None): +async def xmpp_pubsub_send_selected_entry(self, jid_bare, jid_file, node_id, entry_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: jid_bare: {} jid_file: {}'.format(function_name, jid_bare, jid_file)) + # jid_file = jid_bare.replace('/', '_') + db_file = config.get_pathname_to_database(jid_file) + report = {} + if jid_bare == self.boundjid.bare: + node_id = 'urn:xmpp:microblog:0' + node_subtitle = None + node_title = None + else: + feed_id = sqlite.get_feed_id_by_entry_index(db_file, entry_id) + feed_id = feed_id[0] + feed_properties = sqlite.get_feed_properties(db_file, feed_id) + node_id = feed_properties[2] + node_title = feed_properties[3] + node_subtitle = feed_properties[5] + xep = None + iq_create_node = XmppPubsub.create_node( + self, jid_bare, node_id, xep, node_title, node_subtitle) + await XmppIQ.send(self, iq_create_node) + entry = sqlite.get_entry_properties(db_file, entry_id) + print('xmpp_pubsub_send_selected_entry',jid_bare) + print(node_id) + entry_dict = pack_entry_into_dict(db_file, entry) + node_item = create_rfc4287_entry(entry_dict) + entry_url = entry_dict['link'] + item_id = hash_url_to_md5(entry_url) + iq_create_entry = XmppPubsub.create_entry( + self, jid_bare, node_id, item_id, node_item) + await XmppIQ.send(self, iq_create_entry) + await sqlite.mark_as_read(db_file, entry_id) + report = entry_url + return report + + +async def xmpp_pubsub_send_unread_items(self, jid_bare): function_name = sys._getframe().f_code.co_name logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare)) jid_file = jid_bare.replace('/', '_') db_file = config.get_pathname_to_database(jid_file) - enabled = Config.get_setting_value(self.settings, jid_bare, 'enabled') - if enabled: - if num: counter = 0 - report = {} - subscriptions = sqlite.get_active_feeds_url(db_file) - for url in subscriptions: - url = url[0] - if jid_bare == self.boundjid.bare: - node = 'urn:xmpp:microblog:0' - feed_title = None - feed_subtitle = None - else: - feed_id = sqlite.get_feed_id(db_file, url) - feed_id = feed_id[0] - feed_title = sqlite.get_feed_title(db_file, feed_id) - feed_title = feed_title[0] - feed_subtitle = sqlite.get_feed_subtitle(db_file, feed_id) - feed_subtitle = feed_subtitle[0] - node = sqlite.get_feed_identifier(db_file, feed_id) - node = node[0] - xep = None - iq_create_node = XmppPubsub.create_node( - self, jid_bare, node, xep, feed_title, feed_subtitle) - await XmppIQ.send(self, iq_create_node) - entries = sqlite.get_unread_entries_of_feed(db_file, feed_id) - feed_properties = sqlite.get_feed_properties(db_file, feed_id) - feed_version = feed_properties[2] - print('xmpp_send_pubsub',jid_bare) - print(node) - # if num and counter < num: - report[url] = len(entries) - for entry in entries: - feed_entry = {'authors' : entry[3], - 'content' : entry[6], - 'content_type' : entry[7], - 'contact' : entry[4], - 'contributors' : entry[5], - 'summary' : entry[8], - 'summary_type' : entry[9], - 'enclosures' : entry[13], - 'language' : entry[10], - 'link' : entry[2], - 'links' : entry[11], - 'published' : entry[15], - 'tags' : entry[12], - 'title' : entry[1], - 'updated' : entry[16]} - iq_create_entry = XmppPubsub.create_entry( - self, jid_bare, node, feed_entry, feed_version) - await XmppIQ.send(self, iq_create_entry) - ix = entry[0] - await sqlite.mark_as_read(db_file, ix) - # counter += 1 - # if num and counter > num: break - return report + report = {} + subscriptions = sqlite.get_active_feeds_url(db_file) + for url in subscriptions: + url = url[0] + if jid_bare == self.boundjid.bare: + node_id = 'urn:xmpp:microblog:0' + node_subtitle = None + node_title = None + else: + # feed_id = sqlite.get_feed_id(db_file, url) + # feed_id = feed_id[0] + # feed_properties = sqlite.get_feed_properties(db_file, feed_id) + # node_id = feed_properties[2] + # node_title = feed_properties[3] + # node_subtitle = feed_properties[5] + feed_id = sqlite.get_feed_id(db_file, url) + feed_id = feed_id[0] + node_id = sqlite.get_feed_identifier(db_file, feed_id) + node_id = node_id[0] + node_title = sqlite.get_feed_title(db_file, feed_id) + node_title = node_title[0] + node_subtitle = sqlite.get_feed_subtitle(db_file, feed_id) + node_subtitle = node_subtitle[0] + xep = None + iq_create_node = XmppPubsub.create_node( + self, jid_bare, node_id, xep, node_title, node_subtitle) + await XmppIQ.send(self, iq_create_node) + entries = sqlite.get_unread_entries_of_feed(db_file, feed_id) + print('xmpp_pubsub_send_unread_items',jid_bare) + print(node_id) + report[url] = len(entries) + for entry in entries: + feed_entry = pack_entry_into_dict(db_file, entry) + node_entry = create_rfc4287_entry(feed_entry) + entry_url = feed_entry['link'] + item_id = hash_url_to_md5(entry_url) + iq_create_entry = XmppPubsub.create_entry( + self, jid_bare, node_id, item_id, node_entry) + await XmppIQ.send(self, iq_create_entry) + ix = entry[0] + await sqlite.mark_as_read(db_file, ix) + return report -async def xmpp_send_message(self, jid, num=None): +def pack_entry_into_dict(db_file, entry): + entry_id = entry[0] + authors = sqlite.get_authors_by_entry_id(db_file, entry_id) + entry_authors = [] + for author in authors: + entry_author = { + 'name': author[2], + 'email': author[3], + 'url': author[4]} + entry_authors.extend([entry_author]) + + contributors = sqlite.get_contributors_by_entry_id(db_file, entry_id) + entry_contributors = [] + for contributor in contributors: + entry_contributor = { + 'name': contributor[2], + 'email': contributor[3], + 'url': contributor[4]} + entry_contributors.extend([entry_contributor]) + + links = sqlite.get_links_by_entry_id(db_file, entry_id) + entry_links = [] + for link in links: + entry_link = { + 'url': link[2], + 'type': link[3], + 'rel': link[4], + 'size': link[5]} + entry_links.extend([entry_link]) + + + tags = sqlite.get_tags_by_entry_id(db_file, entry_id) + entry_tags = [] + for tag in tags: + entry_tag = { + 'term': tag[2], + 'scheme': tag[3], + 'label': tag[4]} + entry_tags.extend([entry_tag]) + + contents = sqlite.get_contents_by_entry_id(db_file, entry_id) + entry_contents = [] + for content in contents: + entry_content = { + 'text': content[2], + 'type': content[3], + 'base': content[4], + 'lang': content[5]} + entry_contents.extend([entry_content]) + + feed_entry = { + 'authors' : entry_authors, + 'category' : entry[10], + 'comments' : entry[12], + 'contents' : entry_contents, + 'contributors' : entry_contributors, + 'summary_base' : entry[9], + 'summary_lang' : entry[7], + 'summary_text' : entry[6], + 'summary_type' : entry[8], + 'enclosures' : entry[13], + 'href' : entry[11], + 'link' : entry[3], + 'links' : entry_links, + 'published' : entry[14], + 'rating' : entry[13], + 'tags' : entry_tags, + 'title' : entry[4], + 'title_type' : entry[3], + 'updated' : entry[15]} + return feed_entry + + +# NOTE Warning: Entry might not have a link +# TODO Handle situation error +def hash_url_to_md5(url): + url_encoded = url.encode() + url_hashed = hashlib.md5(url_encoded) + url_digest = url_hashed.hexdigest() + return url_digest + + +def create_rfc4287_entry(feed_entry): + node_entry = ET.Element('entry') + node_entry.set('xmlns', 'http://www.w3.org/2005/Atom') + + # Title + title = ET.SubElement(node_entry, 'title') + if feed_entry['title']: + if feed_entry['title_type']: title.set('type', feed_entry['title_type']) + title.text = feed_entry['title'] + elif feed_entry['summary_text']: + if feed_entry['summary_type']: title.set('type', feed_entry['summary_type']) + title.text = feed_entry['summary_text'] + # if feed_entry['summary_base']: title.set('base', feed_entry['summary_base']) + # if feed_entry['summary_lang']: title.set('lang', feed_entry['summary_lang']) + else: + title.text = feed_entry['published'] + + # Some feeds have identical content for contents and summary + # So if content is present, do not add summary + if feed_entry['contents']: + # Content + for feed_entry_content in feed_entry['contents']: + content = ET.SubElement(node_entry, 'content') + # if feed_entry_content['base']: content.set('base', feed_entry_content['base']) + if feed_entry_content['lang']: content.set('lang', feed_entry_content['lang']) + if feed_entry_content['type']: content.set('type', feed_entry_content['type']) + content.text = feed_entry_content['text'] + else: + # Summary + summary = ET.SubElement(node_entry, 'summary') # TODO Try 'content' + # if feed_entry['summary_base']: summary.set('base', feed_entry['summary_base']) + # TODO Check realization of "lang" + if feed_entry['summary_type']: summary.set('type', feed_entry['summary_type']) + if feed_entry['summary_lang']: summary.set('lang', feed_entry['summary_lang']) + summary.text = feed_entry['summary_text'] + + # Authors + for feed_entry_author in feed_entry['authors']: + author = ET.SubElement(node_entry, 'author') + name = ET.SubElement(author, 'name') + name.text = feed_entry_author['name'] + if feed_entry_author['url']: + uri = ET.SubElement(author, 'uri') + uri.text = feed_entry_author['url'] + if feed_entry_author['email']: + email = ET.SubElement(author, 'email') + email.text = feed_entry_author['email'] + + # Contributors + for feed_entry_contributor in feed_entry['contributors']: + contributor = ET.SubElement(node_entry, 'author') + name = ET.SubElement(contributor, 'name') + name.text = feed_entry_contributor['name'] + if feed_entry_contributor['url']: + uri = ET.SubElement(contributor, 'uri') + uri.text = feed_entry_contributor['url'] + if feed_entry_contributor['email']: + email = ET.SubElement(contributor, 'email') + email.text = feed_entry_contributor['email'] + + # Category + category = ET.SubElement(node_entry, "category") + category.set('category', feed_entry['category']) + + # Tags + for feed_entry_tag in feed_entry['tags']: + tag = ET.SubElement(node_entry, 'category') + tag.set('term', feed_entry_tag['term']) + + # Link + link = ET.SubElement(node_entry, "link") + link.set('href', feed_entry['link']) + + # Links + for feed_entry_link in feed_entry['links']: + link = ET.SubElement(node_entry, "link") + link.set('href', feed_entry_link['url']) + link.set('type', feed_entry_link['type']) + link.set('rel', feed_entry_link['rel']) + + # Date updated + if feed_entry['updated']: + updated = ET.SubElement(node_entry, 'updated') + updated.text = feed_entry['updated'] + + # Date published + if feed_entry['published']: + published = ET.SubElement(node_entry, 'published') + published.text = feed_entry['published'] + + return node_entry + + +async def xmpp_chat_send_unread_items(self, jid, num=None): """ Send news items as messages. @@ -292,56 +490,54 @@ async def xmpp_send_message(self, jid, num=None): logger.debug('{}: jid: {} num: {}'.format(function_name, jid, num)) jid_file = jid.replace('/', '_') db_file = config.get_pathname_to_database(jid_file) - enabled = Config.get_setting_value(self.settings, jid, 'enabled') - if enabled: - show_media = Config.get_setting_value(self.settings, jid, 'media') - if not num: - num = Config.get_setting_value(self.settings, jid, 'quantum') - else: - num = int(num) - results = sqlite.get_unread_entries(db_file, num) - news_digest = '' - media = None - chat_type = await get_chat_type(self, jid) - for result in results: - ix = result[0] - title_e = result[1] - url = result[2] - summary = result[3] - feed_id = result[4] - date = result[5] - enclosure = sqlite.get_enclosure_by_entry_id(db_file, ix) - if enclosure: enclosure = enclosure[0] - title_f = sqlite.get_feed_title(db_file, feed_id) - title_f = title_f[0] - news_digest += await list_unread_entries(self, result, title_f, jid) - # print(db_file) - # print(result[0]) - # breakpoint() - await sqlite.mark_as_read(db_file, ix) + show_media = Config.get_setting_value(self.settings, jid, 'media') + if not num: + num = Config.get_setting_value(self.settings, jid, 'quantum') + else: + num = int(num) + results = sqlite.get_unread_entries(db_file, num) + news_digest = '' + media = None + chat_type = await get_chat_type(self, jid) + for result in results: + ix = result[0] + title_e = result[1] + url = result[2] + summary = result[3] + feed_id = result[4] + date = result[5] + enclosure = sqlite.get_enclosure_by_entry_id(db_file, ix) + if enclosure: enclosure = enclosure[0] + title_f = sqlite.get_feed_title(db_file, feed_id) + title_f = title_f[0] + news_digest += await list_unread_entries(self, result, title_f, jid) + # print(db_file) + # print(result[0]) + # breakpoint() + await sqlite.mark_as_read(db_file, ix) - # Find media - # if url.startswith("magnet:"): - # media = action.get_magnet(url) - # elif enclosure.startswith("magnet:"): - # media = action.get_magnet(enclosure) - # elif enclosure: - if show_media: - if enclosure: - media = enclosure - else: - media = await extract_image_from_html(url) - - if media and news_digest: - # Send textual message - XmppMessage.send(self, jid, news_digest, chat_type) - news_digest = '' - # Send media - XmppMessage.send_oob(self, jid, media, chat_type) - media = None - - if news_digest: + # Find media + # if url.startswith("magnet:"): + # media = action.get_magnet(url) + # elif enclosure.startswith("magnet:"): + # media = action.get_magnet(enclosure) + # elif enclosure: + if show_media: + if enclosure: + media = enclosure + else: + media = await extract_image_from_html(url) + + if media and news_digest: + # Send textual message XmppMessage.send(self, jid, news_digest, chat_type) + news_digest = '' + # Send media + XmppMessage.send_oob(self, jid, media, chat_type) + media = None + + if news_digest: + XmppMessage.send(self, jid, news_digest, chat_type) # TODO Add while loop to assure delivery. # print(await current_time(), ">>> ACT send_message",jid) # NOTE Do we need "if statement"? See NOTE at is_muc. @@ -807,25 +1003,25 @@ def export_to_opml(jid, filename, results): function_name = sys._getframe().f_code.co_name logger.debug('{} jid: {} filename: {}' .format(function_name, jid, filename)) - root = ET.Element("opml") + root = ETR.Element("opml") root.set("version", "1.0") - head = ET.SubElement(root, "head") - ET.SubElement(head, "title").text = "{}".format(jid) - ET.SubElement(head, "description").text = ( + head = ETR.SubElement(root, "head") + ETR.SubElement(head, "title").text = "{}".format(jid) + ETR.SubElement(head, "description").text = ( "Set of subscriptions exported by Slixfeed") - ET.SubElement(head, "generator").text = "Slixfeed" - ET.SubElement(head, "urlPublic").text = ( + ETR.SubElement(head, "generator").text = "Slixfeed" + ETR.SubElement(head, "urlPublic").text = ( "https://gitgud.io/sjehuda/slixfeed") time_stamp = dt.current_time() - ET.SubElement(head, "dateCreated").text = time_stamp - ET.SubElement(head, "dateModified").text = time_stamp - body = ET.SubElement(root, "body") + ETR.SubElement(head, "dateCreated").text = time_stamp + ETR.SubElement(head, "dateModified").text = time_stamp + body = ETR.SubElement(root, "body") for result in results: - outline = ET.SubElement(body, "outline") + outline = ETR.SubElement(body, "outline") outline.set("text", result[1]) outline.set("xmlUrl", result[2]) # outline.set("type", result[2]) - tree = ET.ElementTree(root) + tree = ETR.ElementTree(root) tree.write(filename) @@ -835,7 +1031,7 @@ async def import_opml(db_file, result): .format(function_name, db_file)) if not result['error']: document = result['content'] - root = ET.fromstring(document) + root = ETR.fromstring(document) before = sqlite.get_number_of_items(db_file, 'feeds_properties') feeds = [] for child in root.findall(".//outline"): @@ -1789,6 +1985,9 @@ def generate_txt(text, filename): with open(filename, 'w') as file: file.write(text) + +# This works too +# ''.join(xml.etree.ElementTree.fromstring(text).itertext()) def remove_html_tags(data): function_name = sys._getframe().f_code.co_name logger.debug('{}'.format(function_name)) diff --git a/slixfeed/assets/about.toml b/slixfeed/assets/about.toml index bce3023..dfda852 100644 --- a/slixfeed/assets/about.toml +++ b/slixfeed/assets/about.toml @@ -4,7 +4,7 @@ subtitle = "Slixfeed, slixmpp and more" [[about]] name = "Slixfeed" -about = "XMPP news bot" +desc = "XMPP news bot" info = [""" Slixfeed is a news broker bot for syndicated news which aims to be \ an easy to use and fully-featured news aggregating bot. @@ -34,7 +34,7 @@ url = "https://gitgud.io/sjehuda/slixfeed" [[about]] name = "slixmpp" -about = "Slixmpp XMPP Library" +desc = "XMPP library" info = [""" Slixmpp is an MIT licensed XMPP library for Python 3.7+. It is a fork of \ SleekXMPP. @@ -47,7 +47,7 @@ url = "https://codeberg.org/poezio/slixmpp" [[about]] name = "SleekXMPP" -about = "SleekXMPP XMPP Library" +desc = "XMPP library" info = [""" SleekXMPP is an MIT licensed XMPP library for Python 2.6/3.1+, and is \ featured in examples in the book XMPP: The Definitive Guide by Kevin Smith, \ @@ -57,18 +57,22 @@ url = "https://codeberg.org/fritzy/SleekXMPP" [[about]] name = "XMPP" -about = "Previously known as Jabber" +desc = "Messaging protocol (also known as Jabber)" info = [""" XMPP is the Extensible Messaging and Presence Protocol, a set of open \ technologies for instant messaging, presence, multi-party chat, voice and \ -video calls, collaboration, lightweight middleware, content syndication, and \ -generalized routing of XML data. +video calls, collaboration, lightweight middleware, content syndication, \ +and generalized routing of XML data. + +XMPP was originally developed in the Jabber open-source community to \ +provide an open, decentralized alternative to the closed instant messaging \ +services at that time. """] url = "https://xmpp.org/about" [[about]] name = "RSS Task Force" -about = "Swiss Organization" +desc = "Swiss organization" info = [""" The RSS Task Force (previously known as The Syndication Society) is an \ international organization headquartered in Switzerland. @@ -117,17 +121,17 @@ title = "Contributors" subtitle = "People who have contributed to Slixfeed" [[contributors]] -name = "grym from #python" +name = "grym from IRC channel #python" role = "Contributor" info = [""" -Correcting code structure to be better prepared for packaging 18c93083. +Correcting code structure to be better prepared for packaging (18c93083). """] [[contributors]] name = "Guus der Kinderen" role = "XMPP server administrator" info = [""" -Providing OpenFire server for testing various of features. +Providing an Openfire server for testing various of features. XEP-0060: Publish-Subscribe XEP-0114: Jabber Component Protocol @@ -139,7 +143,7 @@ url = "http://goodbytes.im" name = "Simone (roughnecks) Canaletti" role = "XMPP server administrator" info = [""" -Providing Prosody server and Movim instance for testing PubSub. +Providing a Prosody server and a Movim instance for testing PubSub. XEP-0472: Pubsub Social Feed """] @@ -616,7 +620,7 @@ All your data belongs to us. """] [[clients]] -title = "Recommended Clients" +title = "Clients" subtitle = """ As a chat bot, Slixfeed works with any XMPP messenger, yet we have deemed it \ appropriate to list the software that work best with Slixfeed, namely those \ @@ -625,8 +629,14 @@ that provide support for XEP-0050: Ad-Hoc Commands. [[clients]] name = "Cheogram" -info = "XMPP client for mobile" +desc = "XMPP client for mobile" +info = [""" +The Cheogram Android app allows you to join a worldwide communication network. \ +It especially focuses on features useful to users who want to contact those on \ +other networks as well, such as SMS-enabled phone numbers. +"""] url = "https://cheogram.com" +platform = "Android" # [[clients]] # name = "Conversations" @@ -635,8 +645,13 @@ url = "https://cheogram.com" [[clients]] name = "Converse" -info = "XMPP client for desktop and mobile" +desc = "XMPP client for desktop and mobile" +info = [""" +Converse is a free and open-source XMPP chat client that runs in a web browser \ +or on your desktop. +"""] url = "https://conversejs.org" +platform = "HTML (Web)" # [[clients]] # name = "Gajim" @@ -650,13 +665,31 @@ url = "https://conversejs.org" [[clients]] name = "monocles chat" -info = "XMPP client for mobile" +desc = "XMPP client for mobile" +info = """ +monocles chat is a modern and secure Android XMPP chat client. Based on \ +blabber.im and Conversations with a lot of changes and additional features \ +to improve usability and security. +""" url = "https://monocles.chat" +platform = "Android" [[clients]] name = "Movim" -info = "XMPP client for desktop and mobile" +desc = "XMPP client for desktop and mobile" +info = [""" +Movim is a social and chat platform that acts as a frontend for the XMPP network. + +Once deployed Movim offers a complete social and chat experience for the \ +decentralized XMPP network users. It can easily connect to several XMPP \ +servers at the same time. + +With a simple configuration it can also be restricted to one XMPP server \ +and will then act as a powerful frontend for it. Movim is fully compatible \ +with the most used XMPP servers such as ejabberd or Prosody. +"""] url = "https://mov.im" +platform = "HTML (Web)" # [[clients]] # name = "Moxxy" @@ -665,18 +698,50 @@ url = "https://mov.im" [[clients]] name = "Poezio" -info = "XMPP client for console" +desc = "XMPP client for console" +info = [""" +Poezio is a free console XMPP client (the protocol on which the Jabber IM \ +network is built). + +Its goal is to let you connect very easily (no account creation needed) to \ +the network and join various chatrooms, immediately. It tries to look like \ +the most famous IRC clients (weechat, irssi, etc). Many commands are identical \ +and you won't be lost if you already know these clients. Configuration can be \ +made in a configuration file or directly from the client. +"""] url = "https://poez.io" +platform = "FreeBSD and Linux" [[clients]] name = "Psi" -info = "XMPP client for desktop" +desc = "XMPP client for desktop" +info = [""" +Instant messaging as free and open as it should be. + +Psi is a free instant messaging application designed for the XMPP network. \ +Fast and lightweight, Psi is fully open-source and compatible with Windows, \ +Linux, and macOS. + +With Psi's full Unicode support and localizations, easy file transfers, \ +customizable iconsets, and many other great features, you'll learn why users \ +around the world are making the switch to free, open instant messaging. +"""] url = "https://psi-im.org" +platform = "Any" [[clients]] name = "Psi+" -info = "XMPP client for desktop" +desc = "XMPP client for desktop" +info = [""" +In 2009 a Psi fork named Psi+ was started. Project purpose are: implementation \ +of new features, writing of patches and plugins for transferring them to upstream. \ +As of 2017 the most of active Psi+ developers have become official Psi developers, \ +but Psi+ still has a number of unique features. From developers point of view Psi+ \ +is just a development branch of Psi IM client which is hosted at separate git \ +repositories and for which rolling release development model is used. +"""] url = "https://psi-plus.com" +platform = "Any" # [[clients]] # name = "Swift" @@ -689,7 +754,7 @@ url = "https://psi-plus.com" # url = "https://yaxim.org" [[services]] -title = "Recommended News Services" +title = "Online Services" subtitle = [""" Below are online services that extend the syndication experience by means \ of bookmarking and multimedia, and also enhance it by restoring access to \ @@ -706,7 +771,7 @@ link = "https://www.fivefilters.org/feed-creator/" [[services]] name = "Kill the Newsletter" -info = "Kill the Newsletter converts email newsletters into Web feeds." +info = ["Kill the Newsletter converts email newsletters into Web feeds."] link = "https://kill-the-newsletter.com" [[services]] @@ -737,7 +802,7 @@ It's capable of generating RSS feeds from pretty much everything. link = "https://docs.rsshub.app" [[software]] -title = "Recommended News Software" +title = "News Software" subtitle = [""" Take back control of your news. With free, quality, software for your \ desktop, home and mobile devices. @@ -749,7 +814,7 @@ info = [""" A self-hosted RSS reader, based on Dropwizard and React/TypeScript. """] link = "https://commafeed.com" -os = "Any (HTML)" +platform = "HTML (Web)" [[software]] name = "FreshRSS" @@ -758,7 +823,7 @@ FreshRSS is a self-hosted RSS and Atom feed aggregator. It is lightweight, easy to work with, powerful, and customizable. """] link = "https://freshrss.org" -os = "Any (HTML)" +platform = "HTML (Web)" [[software]] name = "Liferea" @@ -769,7 +834,7 @@ it easy to organize and browse feeds. Its GUI is similar to a desktop \ mail/news client, with an embedded web browser. """] link = "https://lzone.de/liferea/" -os = "FreeBSD and Linux" +platform = "FreeBSD and Linux" [[software]] name = "NetNewsWire" @@ -787,7 +852,7 @@ can switch to NetNewsWire to get news directly and more reliably from the \ sites you trust. """] link = "https://netnewswire.com" -os = "MacOS" +platform = "MacOS" [[software]] name = "Newsboat" @@ -796,7 +861,7 @@ Newsboat is an RSS/Atom feed reader for the text console. It’s an actively \ maintained fork of Newsbeuter """] link = "https://newsboat.org" -os = "Any" +platform = "HTML (Web)" [[software]] name = "Spot-On" @@ -806,7 +871,7 @@ search and other forms of communications into a single communications \ orchestra. """] link = "https://textbrowser.github.io/spot-on/" -os = "Any" +platform = "Any" [[software]] name = "Vienna RSS" @@ -816,7 +881,7 @@ help you make sense of the flood of information that is distributed via \ these formats today. """] link = "https://vienna-rss.com" -os = "MacOS" +platform = "MacOS" [[resources]] title = "Useful Resources" @@ -824,15 +889,34 @@ subtitle = "Technologies which Slixfeed is based upon" [[resources]] name = "feedparser" -info = "Syndication Library" +info = "Syndication library" +desc = "Parse Atom and RSS feeds in Python." url = "https://pythonhosted.org/feedparser" [[resources]] name = "Slixmpp" -info = "XMPP Library" +info = "XMPP library" +desc = """ +Slixmpp is an MIT licensed XMPP library for Python 3.7+. It is a fork of \ +SleekXMPP. + +Slixmpp's goals is to only rewrite the core of the SleekXMPP library \ +(the low level socket handling, the timers, the events dispatching) \ +in order to remove all threads. +""" url = "https://slixmpp.readthedocs.io" [[resources]] name = "XMPP" -info = "Messaging Protocol" +info = "Messaging protocol (also known as Jabber)" +desc = """ +XMPP is the Extensible Messaging and Presence Protocol, a set of open \ +technologies for instant messaging, presence, multi-party chat, voice and \ +video calls, collaboration, lightweight middleware, content syndication, \ +and generalized routing of XML data. + +XMPP was originally developed in the Jabber open-source community to \ +provide an open, decentralized alternative to the closed instant messaging \ +services at that time. +""" url = "https://xmpp.org/about" diff --git a/slixfeed/assets/feeds.toml b/slixfeed/assets/feeds.toml index 3522f0d..36bf4e2 100644 --- a/slixfeed/assets/feeds.toml +++ b/slixfeed/assets/feeds.toml @@ -160,6 +160,30 @@ name = "Κόμμα Πειρατών Ελλάδας – Pirate party of Greece" link = "https://www.pirateparty.gr/feed/" tags = ["greece", "party", "pirate"] +[[feeds]] +lang = "en" +name = "Cycling Together with Fiona and Marc" +link = "https://pixelfed.social/users/cyclingtogether.atom" +tags = ["sports", "cycling", "adventure", "life"] + +[[feeds]] +lang = "en" +name = "Lagrange Gemini Client" +link = "https://skyjake.fi/@lagrange.rss" +tags = ["gemini", "gopher", "browser", "telecommunication", "internet"] + +[[feeds]] +lang = "en" +name = "[ngn.tf] | blog" +link = "https://api.ngn.tf/blog/feed.atom" +tags = ["computer", "service", "technology", "telecommunication", "xmpp"] + +[[feeds]] +lang = "en" +name = "The SWORD Project" +link = "http://www.crosswire.org/sword/sword.rss.jsp" +tags = ["bible", "religion", "christianity", "history", "education", "life"] + [[feeds]] lang = "en-au" name = "Pirate Party Australia" @@ -268,11 +292,23 @@ name = "The Brexit Party" link = "https://www.thebrexitparty.org/feed/" tags = ["europe", "politics", "uk"] +[[feeds]] +lang = "en-us" +name = "4chan /diy/ - Do It Yourself" +link = "https://boards.4chan.org/diy/index.rss" +tags = ["design", "diy", "household"] + +[[feeds]] +lang = "en-us" +name = "12bytes.org" +link = "https://12bytes.org/feed.xml" +tags = ["conspiracy", "health", "government", "war", "world"] + [[feeds]] lang = "en-us" name = "153 News - Videos Being Watched" link = "https://153news.net/rss.php?mode=watching" -tags = ["news", "politics", "usa", "video"] +tags = ["europe", "news", "politics", "usa", "video", "world"] [[feeds]] lang = "en-us" @@ -290,7 +326,7 @@ tags = ["lifestyle", "men"] lang = "en-us" name = "BlackListed News" link = "https://www.blacklistednews.com/rss.php" -tags = ["news", "politics", "usa", "world"] +tags = ["conspiracy", "health", "government", "news", "politics", "usa", "world"] [[feeds]] lang = "en-us" @@ -518,7 +554,7 @@ tags = ["gemini", "internet"] lang = "en-us" name = "Public Intelligence Blog" link = "https://phibetaiota.net/feed/" -tags = ["cia", "conspiracy", "health", "government", "war"] +tags = ["cia", "conspiracy", "health", "government", "war", "world"] [[feeds]] lang = "en-us" @@ -694,6 +730,12 @@ name = "Disroot Blog" link = "https://disroot.org/es/blog.atom" tags = ["decentralization", "privacy"] +[[feeds]] +lang = "ch-fr" +name = "Demoniak Network" +link = "https://demoniak.ch/index.xml" +tags = ["computer", "technology"] + [[feeds]] lang = "fr-fr" name = "Agate Blue" diff --git a/slixfeed/sqlite.py b/slixfeed/sqlite.py index 9c0d0c5..ab7ca5c 100644 --- a/slixfeed/sqlite.py +++ b/slixfeed/sqlite.py @@ -920,7 +920,7 @@ def get_feed_properties(db_file, feed_id): """ SELECT * FROM feeds_properties - WHERE feed_id = ? + WHERE id = :feed_id """ ) par = (feed_id,) @@ -1406,20 +1406,20 @@ def get_entries_rejected(db_file, num): return result -def get_enclosure_by_entry_id(db_file, ix): +def get_enclosure_by_entry_id(db_file, entry_id): function_name = sys._getframe().f_code.co_name - logger.debug('{}: db_file: {} ix: {}' - .format(function_name, db_file, ix)) + logger.debug('{}: db_file: {} entry_id: {}' + .format(function_name, db_file, entry_id)) with create_connection(db_file) as conn: cur = conn.cursor() sql = ( """ SELECT url FROM entries_properties_links - WHERE entry_id = :ix AND rel = "enclosure" + WHERE entry_id = :entry_id AND rel = "enclosure" """ ) - par = (ix,) + par = (entry_id,) result = cur.execute(sql, par).fetchone() return result @@ -1831,6 +1831,24 @@ async def set_feed_title(db_file, feed_id, title): cur.execute(sql, par) +def get_entry_properties(db_file, ix): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: db_file: {} ix: {}' + .format(function_name, db_file, ix)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT * + FROM entries_properties + WHERE id = :ix + """ + ) + par = (ix,) + title = cur.execute(sql, par).fetchone() + return title + + def get_entry_title(db_file, ix): function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} ix: {}' @@ -2516,6 +2534,98 @@ async def maintain_archive(db_file, limit): cur.execute(sql, par) +def get_authors_by_entry_id(db_file, entry_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{} db_file: {} entry_id: {}' + .format(function_name, db_file, entry_id)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT * + FROM entries_properties_authors + WHERE entry_id = :entry_id + ORDER BY name DESC + """ + ) + par = (entry_id,) + result = cur.execute(sql, par).fetchall() + return result + + +def get_contributors_by_entry_id(db_file, entry_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{} db_file: {} entry_id: {}' + .format(function_name, db_file, entry_id)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT * + FROM entries_properties_contributors + WHERE entry_id = :entry_id + ORDER BY name DESC + """ + ) + par = (entry_id,) + result = cur.execute(sql, par).fetchall() + return result + + +def get_links_by_entry_id(db_file, entry_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: db_file: {} entry_id: {}' + .format(function_name, db_file, entry_id)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT * + FROM entries_properties_links + WHERE entry_id = :entry_id + """ + ) + par = (entry_id,) + result = cur.execute(sql, par).fetchall() + return result + + +def get_tags_by_entry_id(db_file, entry_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: db_file: {} entry_id: {}' + .format(function_name, db_file, entry_id)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT * + FROM entries_properties_tags + WHERE entry_id = :entry_id + """ + ) + par = (entry_id,) + result = cur.execute(sql, par).fetchall() + return result + + +def get_contents_by_entry_id(db_file, entry_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: db_file: {} entry_id: {}' + .format(function_name, db_file, entry_id)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT * + FROM entries_properties_contents + WHERE entry_id = :entry_id + """ + ) + par = (entry_id,) + result = cur.execute(sql, par).fetchall() + return result + + # TODO Move entries that don't exist into table archive. # NOTE Entries that are read from archive are deleted. # NOTE Unlike entries from table entries, entries from @@ -2538,7 +2648,7 @@ def get_entries_of_feed(db_file, feed_id): cur = conn.cursor() sql = ( """ - SELECT id, title, link, identifier, published, read + SELECT id, title, link, identifier, published FROM entries_properties WHERE feed_id = ? ORDER BY published DESC diff --git a/slixfeed/task.py b/slixfeed/task.py index b0753fb..65f2e30 100644 --- a/slixfeed/task.py +++ b/slixfeed/task.py @@ -170,7 +170,7 @@ async def task_publish(self, jid_bare): if jid_bare not in self.settings: Config.add_settings_jid(self.settings, jid_bare, db_file) while True: - await action.xmpp_send_pubsub(self, jid_bare) + await action.xmpp_pubsub_send_unread_items(self, jid_bare) await asyncio.sleep(60 * 180) @@ -260,7 +260,7 @@ async def task_message(self, jid_bare): await sqlite.update_last_update_time(db_file) else: await sqlite.set_last_update_time(db_file) - await action.xmpp_send_message(self, jid_bare) + await action.xmpp_chat_send_unread_items(self, jid_bare) refresh_task(self, jid_bare, task_message, 'interval') await start_tasks_xmpp_chat(self, jid_bare, ['status']) diff --git a/slixfeed/version.py b/slixfeed/version.py index 9ec3d57..e998a61 100644 --- a/slixfeed/version.py +++ b/slixfeed/version.py @@ -1,2 +1,2 @@ -__version__ = '0.1.57' -__version_info__ = (0, 1, 57) +__version__ = '0.1.58' +__version_info__ = (0, 1, 58) diff --git a/slixfeed/xmpp/bookmark.py b/slixfeed/xmpp/bookmark.py index c2befa7..5116045 100644 --- a/slixfeed/xmpp/bookmark.py +++ b/slixfeed/xmpp/bookmark.py @@ -21,7 +21,7 @@ class XmppBookmark: return conferences - async def properties(self, jid): + async def get_bookmark_properties(self, jid): result = await self.plugin['xep_0048'].get_bookmarks() groupchats = result['private']['bookmarks']['conferences'] for groupchat in groupchats: diff --git a/slixfeed/xmpp/chat.py b/slixfeed/xmpp/chat.py new file mode 100644 index 0000000..41a684b --- /dev/null +++ b/slixfeed/xmpp/chat.py @@ -0,0 +1,1569 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" + +TODO + +1) Deprecate "add" (see above) and make it interactive. + Slixfeed: Do you still want to add this URL to subscription list? + See: case _ if message_lowercase.startswith("add"): + +2) If subscription is inadequate (see XmppPresence.request), send a message that says so. + + elif not self.client_roster[jid]["to"]: + breakpoint() + message.reply("Share online status to activate bot.").send() + return + +3) Set timeout for moderator interaction. + If moderator interaction has been made, and moderator approves the bot, then + the bot will add the given groupchat to bookmarks; otherwise, the bot will + send a message that it was not approved and therefore leaves the groupchat. + +""" + +import asyncio +from feedparser import parse +import logging +import os +import slixfeed.action as action +import slixfeed.config as config +import slixfeed.crawl as crawl +from slixfeed.config import Config +import slixfeed.dt as dt +import slixfeed.fetch as fetch +import slixfeed.sqlite as sqlite +import slixfeed.task as task +import slixfeed.url as uri +from slixfeed.version import __version__ +from slixfeed.xmpp.bookmark import XmppBookmark +from slixfeed.xmpp.muc import XmppGroupchat +from slixfeed.xmpp.message import XmppMessage +from slixfeed.xmpp.presence import XmppPresence +from slixfeed.xmpp.upload import XmppUpload +from slixfeed.xmpp.utility import get_chat_type, is_moderator, is_operator +import time + +from random import randrange + +try: + import tomllib +except: + import tomli as tomllib + + + # for task in main_task: + # task.cancel() + + # Deprecated in favour of event "presence_available" + # if not main_task: + # await select_file() + +class Chat: + + async def process_message(self, message): + """ + Process incoming message stanzas. Be aware that this also + includes MUC messages and error messages. It is usually + a good practice to check the messages's type before + processing or sending replies. + + Parameters + ---------- + message : str + The received message stanza. See the documentation + for stanza objects and the Message stanza to see + how it may be used. + """ + if message['type'] in ('chat', 'groupchat', 'normal'): + jid_bare = message['from'].bare + jid_file = jid_bare + message_text = ' '.join(message['body'].split()) + command_time_start = time.time() + + # if (message['type'] == 'groupchat' and + # message['muc']['nick'] == self.alias): + # return + + # FIXME Code repetition. See below. + # TODO Check alias by nickname associated with conference + if message['type'] == 'groupchat': + if (message['muc']['nick'] == self.alias): + return + jid_full = str(message['from']) + if not is_moderator(self, jid_bare, jid_full): + return + + # NOTE This is an exceptional case in which we treat + # type groupchat the same as type chat in a way that + # doesn't require an exclamation mark for actionable + # command. + if (message_text.lower().startswith('http') and + message_text.lower().endswith('.opml')): + url = message_text + key_list = ['status'] + task.clean_tasks_xmpp_chat(self, jid_bare, key_list) + status_type = 'dnd' + status_message = '📥️ Procesing request to import feeds...' + # pending_tasks_num = len(self.pending_tasks[jid_bare]) + pending_tasks_num = randrange(10000, 99999) + self.pending_tasks[jid_bare][pending_tasks_num] = status_message + # self.pending_tasks_counter += 1 + # self.pending_tasks[jid_bare][self.pending_tasks_counter] = status_message + XmppPresence.send(self, jid_bare, status_message, + status_type=status_type) + db_file = config.get_pathname_to_database(jid_file) + result = await fetch.http(url) + count = await action.import_opml(db_file, result) + if count: + response = 'Successfully imported {} feeds.'.format(count) + else: + response = 'OPML file was not imported.' + del self.pending_tasks[jid_bare][pending_tasks_num] + # del self.pending_tasks[jid_bare][self.pending_tasks_counter] + key_list = ['status'] + await task.start_tasks_xmpp_chat(self, jid_bare, key_list) + XmppMessage.send_reply(self, message, response) + return + + + if message['type'] == 'groupchat': + # nick = message['from'][message['from'].index('/')+1:] + # nick = str(message['from']) + # nick = nick[nick.index('/')+1:] + if (message['muc']['nick'] == self.alias or + not message['body'].startswith('!')): + return + # token = await initdb( + # jid_bare, + # sqlite.get_setting_value, + # 'token' + # ) + # if token == 'accepted': + # operator = await initdb( + # jid_bare, + # sqlite.get_setting_value, + # 'masters' + # ) + # if operator: + # if nick not in operator: + # return + # approved = False + jid_full = str(message['from']) + if not is_moderator(self, jid_bare, jid_full): + return + # if role == 'moderator': + # approved = True + # TODO Implement a list of temporary operators + # Once an operator is appointed, the control would last + # untile the participant has been disconnected from MUC + # An operator is a function to appoint non moderators. + # Changing nickname is fine and consist of no problem. + # if not approved: + # operator = await initdb( + # jid_bare, + # sqlite.get_setting_value, + # 'masters' + # ) + # if operator: + # if nick in operator: + # approved = True + # if not approved: + # return + + # # Begin processing new JID + # # Deprecated in favour of event 'presence_available' + # db_dir = config.get_default_data_directory() + # os.chdir(db_dir) + # if jid + '.db' not in os.listdir(): + # await task_jid(jid) + + # await compose.message(self, jid_bare, message) + + if message['type'] == 'groupchat': + message_text = message_text[1:] + message_lowercase = message_text.lower() + + logging.debug([str(message['from']), ':', message_text]) + + # Support private message via groupchat + # See https://codeberg.org/poezio/slixmpp/issues/3506 + if message['type'] == 'chat' and message.get_plugin('muc', check=True): + # jid_bare = message['from'].bare + jid_full = str(message['from']) + if (jid_bare == jid_full[:jid_full.index('/')]): + jid = str(message['from']) + jid_file = jid_full.replace('/', '_') + + response = None + match message_lowercase: + # case 'breakpoint': + # if is_operator(self, jid_bare): + # breakpoint() + # print('task_manager[jid]') + # print(task_manager[jid]) + # await self.get_roster() + # print('roster 1') + # print(self.client_roster) + # print('roster 2') + # print(self.client_roster.keys()) + # print('jid') + # print(jid) + # else: + # response = ( + # 'This action is restricted. ' + # 'Type: breakpoint.' + # ) + # XmppMessage.send_reply(self, message, response) + case 'help': + command_list = ' '.join(action.manual('commands.toml')) + response = ('Available command keys:\n' + '```\n{}\n```\n' + 'Usage: `help `' + .format(command_list)) + print(response) + XmppMessage.send_reply(self, message, response) + case 'help all': + command_list = action.manual('commands.toml', section='all') + response = ('Complete list of commands:\n' + '```\n{}\n```' + .format(command_list)) + print(response) + XmppMessage.send_reply(self, message, response) + case _ if message_lowercase.startswith('help'): + command = message_text[5:].lower() + command = command.split(' ') + if len(command) == 2: + command_root = command[0] + command_name = command[1] + command_list = action.manual('commands.toml', + section=command_root, + command=command_name) + if command_list: + command_list = ''.join(command_list) + response = (command_list) + else: + response = ('KeyError for {} {}' + .format(command_root, command_name)) + elif len(command) == 1: + command = command[0] + command_list = action.manual('commands.toml', command) + if command_list: + command_list = ' '.join(command_list) + response = ('Available command `{}` keys:\n' + '```\n{}\n```\n' + 'Usage: `help {} `' + .format(command, command_list, command)) + else: + response = 'KeyError for {}'.format(command) + else: + response = ('Invalid. Enter command key ' + 'or command key & name') + XmppMessage.send_reply(self, message, response) + case 'info': + config_dir = config.get_default_config_directory() + with open(config_dir + '/' + 'information.toml', mode="rb") as information: + entries = tomllib.load(information) + response = ('Available command options:\n' + '```\n{}\n```\n' + 'Usage: `info