Add functionality for handling with PubSub nodes;

Add functionality to submit items from database to PubSub nodes;
Change the fashion by which items are sent;
Fix minor issues.

slixfeed/action.py

@@ -28,6 +28,7 @@ TODO
 from asyncio.exceptions import IncompleteReadError
 from bs4 import BeautifulSoup
 from feedparser import parse
+import hashlib
 from http.client import IncompleteRead
 import json
 from slixfeed.log import Logger
@@ -39,7 +40,6 @@ import slixfeed.crawl as crawl
 import slixfeed.dt as dt
 import slixfeed.fetch as fetch
 import slixfeed.sqlite as sqlite
-import slixfeed.url as uri
 from slixfeed.url import (
     complete_url,
     join_url,
@@ -56,10 +56,11 @@ from slixfeed.xmpp.presence import XmppPresence
 from slixfeed.xmpp.publish import XmppPubsub
 from slixfeed.xmpp.upload import XmppUpload
 from slixfeed.xmpp.utility import get_chat_type
+from slixmpp.xmlstream import ET
 import sys
 from urllib import error
 from urllib.parse import parse_qs, urlsplit
-import xml.etree.ElementTree as ET
+import xml.etree.ElementTree as ETR
 
 try:
     import tomllib
@@ -174,10 +175,7 @@ async def xmpp_send_status_message(self, jid):
     jid_file = jid.replace('/', '_')
     db_file = config.get_pathname_to_database(jid_file)
     enabled = Config.get_setting_value(self.settings, jid, 'enabled')
-    if not enabled:
+    if enabled:
-        status_mode = 'xa'
-        status_text = '📪️ Send "Start" to receive updates'
-    else:
         jid_task = self.pending_tasks[jid]
         if len(jid_task):
             status_mode = 'dnd'
@@ -202,7 +200,9 @@ async def xmpp_send_status_message(self, jid):
                 else:
                     status_mode = 'available'
                     status_text = '📭️ No news'
-
+    else:
+        status_mode = 'xa'
+        status_text = '📪️ Send "Start" to receive updates'
     # breakpoint()
     # print(await current_time(), status_text, "for", jid)
     XmppPresence.send(self, jid, status_text, status_type=status_mode)
@@ -215,69 +215,267 @@ async def xmpp_send_status_message(self, jid):
     # )
 
 
-async def xmpp_send_pubsub(self, jid_bare, num=None):
+async def xmpp_pubsub_send_selected_entry(self, jid_bare, jid_file, node_id, entry_id):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{}: jid_bare: {} jid_file: {}'.format(function_name, jid_bare, jid_file))
+    # jid_file = jid_bare.replace('/', '_')
+    db_file = config.get_pathname_to_database(jid_file)
+    report = {}
+    if jid_bare == self.boundjid.bare:
+        node_id = 'urn:xmpp:microblog:0'
+        node_subtitle = None
+        node_title = None
+    else:
+        feed_id = sqlite.get_feed_id_by_entry_index(db_file, entry_id)
+        feed_id = feed_id[0]
+        feed_properties = sqlite.get_feed_properties(db_file, feed_id)
+        node_id = feed_properties[2]
+        node_title = feed_properties[3]
+        node_subtitle = feed_properties[5]
+    xep = None
+    iq_create_node = XmppPubsub.create_node(
+        self, jid_bare, node_id, xep, node_title, node_subtitle)
+    await XmppIQ.send(self, iq_create_node)
+    entry = sqlite.get_entry_properties(db_file, entry_id)
+    print('xmpp_pubsub_send_selected_entry',jid_bare)
+    print(node_id)
+    entry_dict = pack_entry_into_dict(db_file, entry)
+    node_item = create_rfc4287_entry(entry_dict)
+    entry_url = entry_dict['link']
+    item_id = hash_url_to_md5(entry_url)
+    iq_create_entry = XmppPubsub.create_entry(
+        self, jid_bare, node_id, item_id, node_item)
+    await XmppIQ.send(self, iq_create_entry)
+    await sqlite.mark_as_read(db_file, entry_id)
+    report = entry_url
+    return report
+
+
+async def xmpp_pubsub_send_unread_items(self, jid_bare):
     function_name = sys._getframe().f_code.co_name
     logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare))
     jid_file = jid_bare.replace('/', '_')
     db_file = config.get_pathname_to_database(jid_file)
-    enabled = Config.get_setting_value(self.settings, jid_bare, 'enabled')
+    report = {}
-    if enabled:
+    subscriptions = sqlite.get_active_feeds_url(db_file)
-        if num: counter = 0
+    for url in subscriptions:
-        report = {}
+        url = url[0]
-        subscriptions = sqlite.get_active_feeds_url(db_file)
+        if jid_bare == self.boundjid.bare:
-        for url in subscriptions:
+            node_id = 'urn:xmpp:microblog:0'
-            url = url[0]
+            node_subtitle = None
-            if jid_bare == self.boundjid.bare:
+            node_title = None
-                node = 'urn:xmpp:microblog:0'
+        else:
-                feed_title = None
+            # feed_id = sqlite.get_feed_id(db_file, url)
-                feed_subtitle = None
+            # feed_id = feed_id[0]
-            else:
+            # feed_properties = sqlite.get_feed_properties(db_file, feed_id)
-                feed_id = sqlite.get_feed_id(db_file, url)
+            # node_id = feed_properties[2]
-                feed_id = feed_id[0]
+            # node_title = feed_properties[3]
-                feed_title = sqlite.get_feed_title(db_file, feed_id)
+            # node_subtitle = feed_properties[5]
-                feed_title = feed_title[0]
+            feed_id = sqlite.get_feed_id(db_file, url)
-                feed_subtitle = sqlite.get_feed_subtitle(db_file, feed_id)
+            feed_id = feed_id[0]
-                feed_subtitle = feed_subtitle[0]
+            node_id = sqlite.get_feed_identifier(db_file, feed_id)
-                node = sqlite.get_feed_identifier(db_file, feed_id)
+            node_id = node_id[0]
-                node = node[0]
+            node_title = sqlite.get_feed_title(db_file, feed_id)
-            xep = None
+            node_title = node_title[0]
-            iq_create_node = XmppPubsub.create_node(
+            node_subtitle = sqlite.get_feed_subtitle(db_file, feed_id)
-                self, jid_bare, node, xep, feed_title, feed_subtitle)
+            node_subtitle = node_subtitle[0]
-            await XmppIQ.send(self, iq_create_node)
+        xep = None
-            entries = sqlite.get_unread_entries_of_feed(db_file, feed_id)
+        iq_create_node = XmppPubsub.create_node(
-            feed_properties = sqlite.get_feed_properties(db_file, feed_id)
+            self, jid_bare, node_id, xep, node_title, node_subtitle)
-            feed_version = feed_properties[2]
+        await XmppIQ.send(self, iq_create_node)
-            print('xmpp_send_pubsub',jid_bare)
+        entries = sqlite.get_unread_entries_of_feed(db_file, feed_id)
-            print(node)
+        print('xmpp_pubsub_send_unread_items',jid_bare)
-            # if num and counter < num:
+        print(node_id)
-            report[url] = len(entries)
+        report[url] = len(entries)
-            for entry in entries:
+        for entry in entries:
-                feed_entry = {'authors'      : entry[3],
+            feed_entry = pack_entry_into_dict(db_file, entry)
-                              'content'      : entry[6],
+            node_entry = create_rfc4287_entry(feed_entry)
-                              'content_type' : entry[7],
+            entry_url = feed_entry['link']
-                              'contact'      : entry[4],
+            item_id = hash_url_to_md5(entry_url)
-                              'contributors' : entry[5],
+            iq_create_entry = XmppPubsub.create_entry(
-                              'summary'      : entry[8],
+                self, jid_bare, node_id, item_id, node_entry)
-                              'summary_type' : entry[9],
+            await XmppIQ.send(self, iq_create_entry)
-                              'enclosures'   : entry[13],
+            ix = entry[0]
-                              'language'     : entry[10],
+            await sqlite.mark_as_read(db_file, ix)
-                              'link'         : entry[2],
+    return report
-                              'links'        : entry[11],
-                              'published'    : entry[15],
-                              'tags'         : entry[12],
-                              'title'        : entry[1],
-                              'updated'      : entry[16]}
-                iq_create_entry = XmppPubsub.create_entry(
-                    self, jid_bare, node, feed_entry, feed_version)
-                await XmppIQ.send(self, iq_create_entry)
-                ix = entry[0]
-                await sqlite.mark_as_read(db_file, ix)
-                    # counter += 1
-                    # if num and counter > num: break
-        return report
 
 
-async def xmpp_send_message(self, jid, num=None):
+def pack_entry_into_dict(db_file, entry):
+    entry_id = entry[0]
+    authors = sqlite.get_authors_by_entry_id(db_file, entry_id)
+    entry_authors = []
+    for author in authors:
+        entry_author = {
+            'name': author[2],
+            'email': author[3],
+            'url': author[4]}
+        entry_authors.extend([entry_author])
+
+    contributors = sqlite.get_contributors_by_entry_id(db_file, entry_id)
+    entry_contributors = []
+    for contributor in contributors:
+        entry_contributor = {
+            'name': contributor[2],
+            'email': contributor[3],
+            'url': contributor[4]}
+        entry_contributors.extend([entry_contributor])
+
+    links = sqlite.get_links_by_entry_id(db_file, entry_id)
+    entry_links = []
+    for link in links:
+        entry_link = {
+            'url': link[2],
+            'type': link[3],
+            'rel': link[4],
+            'size': link[5]}
+        entry_links.extend([entry_link])
+        
+
+    tags = sqlite.get_tags_by_entry_id(db_file, entry_id)
+    entry_tags = []
+    for tag in tags:
+        entry_tag = {
+            'term': tag[2],
+            'scheme': tag[3],
+            'label': tag[4]}
+        entry_tags.extend([entry_tag])
+
+    contents = sqlite.get_contents_by_entry_id(db_file, entry_id)
+    entry_contents = []
+    for content in contents:
+        entry_content = {
+            'text': content[2],
+            'type': content[3],
+            'base': content[4],
+            'lang': content[5]}
+        entry_contents.extend([entry_content])
+        
+    feed_entry = {
+        'authors'      : entry_authors,
+        'category'     : entry[10],
+        'comments'     : entry[12],
+        'contents'     : entry_contents,
+        'contributors' : entry_contributors,
+        'summary_base' : entry[9],
+        'summary_lang' : entry[7],
+        'summary_text' : entry[6],
+        'summary_type' : entry[8],
+        'enclosures'   : entry[13],
+        'href'         : entry[11],
+        'link'         : entry[3],
+        'links'        : entry_links,
+        'published'    : entry[14],
+        'rating'       : entry[13],
+        'tags'         : entry_tags,
+        'title'        : entry[4],
+        'title_type'   : entry[3],
+        'updated'      : entry[15]}
+    return feed_entry
+
+
+# NOTE Warning: Entry might not have a link
+# TODO Handle situation error
+def hash_url_to_md5(url):
+    url_encoded = url.encode()
+    url_hashed = hashlib.md5(url_encoded)
+    url_digest = url_hashed.hexdigest()
+    return url_digest
+    
+
+def create_rfc4287_entry(feed_entry):
+    node_entry = ET.Element('entry')
+    node_entry.set('xmlns', 'http://www.w3.org/2005/Atom')
+
+    # Title
+    title = ET.SubElement(node_entry, 'title')
+    if feed_entry['title']:
+        if feed_entry['title_type']: title.set('type', feed_entry['title_type'])
+        title.text = feed_entry['title']
+    elif feed_entry['summary_text']:
+        if feed_entry['summary_type']: title.set('type', feed_entry['summary_type'])
+        title.text = feed_entry['summary_text']
+        # if feed_entry['summary_base']: title.set('base', feed_entry['summary_base'])
+        # if feed_entry['summary_lang']: title.set('lang', feed_entry['summary_lang'])
+    else:
+        title.text = feed_entry['published']
+
+    # Some feeds have identical content for contents and summary
+    # So if content is present, do not add summary
+    if feed_entry['contents']:
+        # Content
+        for feed_entry_content in feed_entry['contents']:
+            content = ET.SubElement(node_entry, 'content')
+            # if feed_entry_content['base']: content.set('base', feed_entry_content['base'])
+            if feed_entry_content['lang']: content.set('lang', feed_entry_content['lang'])
+            if feed_entry_content['type']: content.set('type', feed_entry_content['type'])
+            content.text = feed_entry_content['text']
+    else:
+        # Summary
+        summary = ET.SubElement(node_entry, 'summary') # TODO Try 'content'
+        # if feed_entry['summary_base']: summary.set('base', feed_entry['summary_base'])
+        # TODO Check realization of "lang"
+        if feed_entry['summary_type']: summary.set('type', feed_entry['summary_type'])
+        if feed_entry['summary_lang']: summary.set('lang', feed_entry['summary_lang'])
+        summary.text = feed_entry['summary_text']
+
+    # Authors
+    for feed_entry_author in feed_entry['authors']:
+        author = ET.SubElement(node_entry, 'author')
+        name = ET.SubElement(author, 'name')
+        name.text = feed_entry_author['name']
+        if feed_entry_author['url']:
+            uri = ET.SubElement(author, 'uri')
+            uri.text = feed_entry_author['url']
+        if feed_entry_author['email']:
+            email = ET.SubElement(author, 'email')
+            email.text = feed_entry_author['email']
+
+    # Contributors
+    for feed_entry_contributor in feed_entry['contributors']:
+        contributor = ET.SubElement(node_entry, 'author')
+        name = ET.SubElement(contributor, 'name')
+        name.text = feed_entry_contributor['name']
+        if feed_entry_contributor['url']:
+            uri = ET.SubElement(contributor, 'uri')
+            uri.text = feed_entry_contributor['url']
+        if feed_entry_contributor['email']:
+            email = ET.SubElement(contributor, 'email')
+            email.text = feed_entry_contributor['email']
+
+    # Category
+    category = ET.SubElement(node_entry, "category")
+    category.set('category', feed_entry['category'])
+
+    # Tags
+    for feed_entry_tag in feed_entry['tags']:
+        tag = ET.SubElement(node_entry, 'category')
+        tag.set('term', feed_entry_tag['term'])
+
+    # Link
+    link = ET.SubElement(node_entry, "link")
+    link.set('href', feed_entry['link'])
+
+    # Links
+    for feed_entry_link in feed_entry['links']:
+        link = ET.SubElement(node_entry, "link")
+        link.set('href', feed_entry_link['url'])
+        link.set('type', feed_entry_link['type'])
+        link.set('rel', feed_entry_link['rel'])
+
+    # Date updated
+    if feed_entry['updated']:
+        updated = ET.SubElement(node_entry, 'updated')
+        updated.text = feed_entry['updated']
+
+    # Date published
+    if feed_entry['published']:
+        published = ET.SubElement(node_entry, 'published')
+        published.text = feed_entry['published']
+
+    return node_entry
+
+
+async def xmpp_chat_send_unread_items(self, jid, num=None):
     """
     Send news items as messages.
 
@@ -292,56 +490,54 @@ async def xmpp_send_message(self, jid, num=None):
     logger.debug('{}: jid: {} num: {}'.format(function_name, jid, num))
     jid_file = jid.replace('/', '_')
     db_file = config.get_pathname_to_database(jid_file)
-    enabled = Config.get_setting_value(self.settings, jid, 'enabled')
+    show_media = Config.get_setting_value(self.settings, jid, 'media')
-    if enabled:
+    if not num:
-        show_media = Config.get_setting_value(self.settings, jid, 'media')
+        num = Config.get_setting_value(self.settings, jid, 'quantum')
-        if not num:
+    else:
-            num = Config.get_setting_value(self.settings, jid, 'quantum')
+        num = int(num)
-        else:
+    results = sqlite.get_unread_entries(db_file, num)
-            num = int(num)
+    news_digest = ''
-        results = sqlite.get_unread_entries(db_file, num)
+    media = None
-        news_digest = ''
+    chat_type = await get_chat_type(self, jid)
-        media = None
+    for result in results:
-        chat_type = await get_chat_type(self, jid)
+        ix = result[0]
-        for result in results:
+        title_e = result[1]
-            ix = result[0]
+        url = result[2]
-            title_e = result[1]
+        summary = result[3]
-            url = result[2]
+        feed_id = result[4]
-            summary = result[3]
+        date = result[5]
-            feed_id = result[4]
+        enclosure = sqlite.get_enclosure_by_entry_id(db_file, ix)
-            date = result[5]
+        if enclosure: enclosure = enclosure[0]
-            enclosure = sqlite.get_enclosure_by_entry_id(db_file, ix)
+        title_f = sqlite.get_feed_title(db_file, feed_id)
-            if enclosure: enclosure = enclosure[0]
+        title_f = title_f[0]
-            title_f = sqlite.get_feed_title(db_file, feed_id)
+        news_digest += await list_unread_entries(self, result, title_f, jid)
-            title_f = title_f[0]
+        # print(db_file)
-            news_digest += await list_unread_entries(self, result, title_f, jid)
+        # print(result[0])
-            # print(db_file)
+        # breakpoint()
-            # print(result[0])
+        await sqlite.mark_as_read(db_file, ix)
-            # breakpoint()
-            await sqlite.mark_as_read(db_file, ix)
 
-            # Find media
+        # Find media
-            # if url.startswith("magnet:"):
+        # if url.startswith("magnet:"):
-            #     media = action.get_magnet(url)
+        #     media = action.get_magnet(url)
-            # elif enclosure.startswith("magnet:"):
+        # elif enclosure.startswith("magnet:"):
-            #     media = action.get_magnet(enclosure)
+        #     media = action.get_magnet(enclosure)
-            # elif enclosure:
+        # elif enclosure:
-            if show_media:
+        if show_media:
-                if enclosure:
+            if enclosure:
-                    media = enclosure
+                media = enclosure
-                else:
+            else:
-                    media = await extract_image_from_html(url)
+                media = await extract_image_from_html(url)
-            
+        
-            if media and news_digest:
+        if media and news_digest:
-                # Send textual message
+            # Send textual message
-                XmppMessage.send(self, jid, news_digest, chat_type)
-                news_digest = ''
-                # Send media
-                XmppMessage.send_oob(self, jid, media, chat_type)
-                media = None
-                
-        if news_digest:
             XmppMessage.send(self, jid, news_digest, chat_type)
+            news_digest = ''
+            # Send media
+            XmppMessage.send_oob(self, jid, media, chat_type)
+            media = None
+            
+    if news_digest:
+        XmppMessage.send(self, jid, news_digest, chat_type)
             # TODO Add while loop to assure delivery.
             # print(await current_time(), ">>> ACT send_message",jid)
             # NOTE Do we need "if statement"? See NOTE at is_muc.
@@ -807,25 +1003,25 @@ def export_to_opml(jid, filename, results):
     function_name = sys._getframe().f_code.co_name
     logger.debug('{} jid: {} filename: {}'
                 .format(function_name, jid, filename))
-    root = ET.Element("opml")
+    root = ETR.Element("opml")
     root.set("version", "1.0")
-    head = ET.SubElement(root, "head")
+    head = ETR.SubElement(root, "head")
-    ET.SubElement(head, "title").text = "{}".format(jid)
+    ETR.SubElement(head, "title").text = "{}".format(jid)
-    ET.SubElement(head, "description").text = (
+    ETR.SubElement(head, "description").text = (
         "Set of subscriptions exported by Slixfeed")
-    ET.SubElement(head, "generator").text = "Slixfeed"
+    ETR.SubElement(head, "generator").text = "Slixfeed"
-    ET.SubElement(head, "urlPublic").text = (
+    ETR.SubElement(head, "urlPublic").text = (
         "https://gitgud.io/sjehuda/slixfeed")
     time_stamp = dt.current_time()
-    ET.SubElement(head, "dateCreated").text = time_stamp
+    ETR.SubElement(head, "dateCreated").text = time_stamp
-    ET.SubElement(head, "dateModified").text = time_stamp
+    ETR.SubElement(head, "dateModified").text = time_stamp
-    body = ET.SubElement(root, "body")
+    body = ETR.SubElement(root, "body")
     for result in results:
-        outline = ET.SubElement(body, "outline")
+        outline = ETR.SubElement(body, "outline")
         outline.set("text", result[1])
         outline.set("xmlUrl", result[2])
         # outline.set("type", result[2])
-    tree = ET.ElementTree(root)
+    tree = ETR.ElementTree(root)
     tree.write(filename)
 
 
@@ -835,7 +1031,7 @@ async def import_opml(db_file, result):
                 .format(function_name, db_file))
     if not result['error']:
         document = result['content']
-        root = ET.fromstring(document)
+        root = ETR.fromstring(document)
         before = sqlite.get_number_of_items(db_file, 'feeds_properties')
         feeds = []
         for child in root.findall(".//outline"):
@@ -1789,6 +1985,9 @@ def generate_txt(text, filename):
     with open(filename, 'w') as file:
         file.write(text)
 
+
+# This works too
+# ''.join(xml.etree.ElementTree.fromstring(text).itertext())
 def remove_html_tags(data):
     function_name = sys._getframe().f_code.co_name
     logger.debug('{}'.format(function_name))

slixfeed/assets/about.toml

@@ -4,7 +4,7 @@ subtitle = "Slixfeed, slixmpp and more"
 
 [[about]]
 name = "Slixfeed"
-about = "XMPP news bot"
+desc = "XMPP news bot"
 info = ["""
 Slixfeed is a news broker bot for syndicated news which aims to be \
 an easy to use and fully-featured news aggregating bot.
@@ -34,7 +34,7 @@ url = "https://gitgud.io/sjehuda/slixfeed"
 
 [[about]]
 name = "slixmpp"
-about = "Slixmpp XMPP Library"
+desc = "XMPP library"
 info = ["""
 Slixmpp is an MIT licensed XMPP library for Python 3.7+. It is a fork of \
 SleekXMPP.
@@ -47,7 +47,7 @@ url = "https://codeberg.org/poezio/slixmpp"
 
 [[about]]
 name = "SleekXMPP"
-about = "SleekXMPP XMPP Library"
+desc = "XMPP library"
 info = ["""
 SleekXMPP is an MIT licensed XMPP library for Python 2.6/3.1+, and is \
 featured in examples in the book XMPP: The Definitive Guide by Kevin Smith, \
@@ -57,18 +57,22 @@ url = "https://codeberg.org/fritzy/SleekXMPP"
 
 [[about]]
 name = "XMPP"
-about = "Previously known as Jabber"
+desc = "Messaging protocol (also known as Jabber)"
 info = ["""
 XMPP is the Extensible Messaging and Presence Protocol, a set of open \
 technologies for instant messaging, presence, multi-party chat, voice and \
-video calls, collaboration, lightweight middleware, content syndication, and \
+video calls, collaboration, lightweight middleware, content syndication, \
-generalized routing of XML data.
+and generalized routing of XML data.
+
+XMPP was originally developed in the Jabber open-source community to \
+provide an open, decentralized alternative to the closed instant messaging \
+services at that time.
 """]
 url = "https://xmpp.org/about"
 
 [[about]]
 name = "RSS Task Force"
-about = "Swiss Organization"
+desc = "Swiss organization"
 info = ["""
 The RSS Task Force (previously known as The Syndication Society) is an \
 international organization headquartered in Switzerland.
@@ -117,17 +121,17 @@ title = "Contributors"
 subtitle = "People who have contributed to Slixfeed"
 
 [[contributors]]
-name = "grym from #python"
+name = "grym from IRC channel #python"
 role = "Contributor"
 info = ["""
-Correcting code structure to be better prepared for packaging 18c93083.
+Correcting code structure to be better prepared for packaging (18c93083).
 """]
 
 [[contributors]]
 name = "Guus der Kinderen"
 role = "XMPP server administrator"
 info = ["""
-Providing OpenFire server for testing various of features.
+Providing an Openfire server for testing various of features.
 
 XEP-0060: Publish-Subscribe
 XEP-0114: Jabber Component Protocol
@@ -139,7 +143,7 @@ url = "http://goodbytes.im"
 name = "Simone (roughnecks) Canaletti"
 role = "XMPP server administrator"
 info = ["""
-Providing Prosody server and Movim instance for testing PubSub.
+Providing a Prosody server and a Movim instance for testing PubSub.
 
 XEP-0472: Pubsub Social Feed
 """]
@@ -616,7 +620,7 @@ All your data belongs to us.
 """]
 
 [[clients]]
-title = "Recommended Clients"
+title = "Clients"
 subtitle = """
 As a chat bot, Slixfeed works with any XMPP messenger, yet we have deemed it \
 appropriate to list the software that work best with Slixfeed, namely those \
@@ -625,8 +629,14 @@ that provide support for XEP-0050: Ad-Hoc Commands.
 
 [[clients]]
 name = "Cheogram"
-info = "XMPP client for mobile"
+desc = "XMPP client for mobile"
+info = ["""
+The Cheogram Android app allows you to join a worldwide communication network. \
+It especially focuses on features useful to users who want to contact those on \
+other networks as well, such as SMS-enabled phone numbers.
+"""]
 url = "https://cheogram.com"
+platform = "Android"
 
 # [[clients]]
 # name = "Conversations"
@@ -635,8 +645,13 @@ url = "https://cheogram.com"
 
 [[clients]]
 name = "Converse"
-info = "XMPP client for desktop and mobile"
+desc = "XMPP client for desktop and mobile"
+info = ["""
+Converse is a free and open-source XMPP chat client that runs in a web browser \
+or on your desktop.
+"""]
 url = "https://conversejs.org"
+platform = "HTML (Web)"
 
 # [[clients]]
 # name = "Gajim"
@@ -650,13 +665,31 @@ url = "https://conversejs.org"
 
 [[clients]]
 name = "monocles chat"
-info = "XMPP client for mobile"
+desc = "XMPP client for mobile"
+info = """
+monocles chat is a modern and secure Android XMPP chat client. Based on \
+blabber.im and Conversations with a lot of changes and additional features \
+to improve usability and security.
+"""
 url = "https://monocles.chat"
+platform = "Android"
 
 [[clients]]
 name = "Movim"
-info = "XMPP client for desktop and mobile"
+desc = "XMPP client for desktop and mobile"
+info = ["""
+Movim is a social and chat platform that acts as a frontend for the XMPP network.
+
+Once deployed Movim offers a complete social and chat experience for the \
+decentralized XMPP network users. It can easily connect to several XMPP \
+servers at the same time.
+
+With a simple configuration it can also be restricted to one XMPP server \
+and will then act as a powerful frontend for it. Movim is fully compatible \
+with the most used XMPP servers such as ejabberd or Prosody.
+"""]
 url = "https://mov.im"
+platform = "HTML (Web)"
 
 # [[clients]]
 # name = "Moxxy"
@@ -665,18 +698,50 @@ url = "https://mov.im"
 
 [[clients]]
 name = "Poezio"
-info = "XMPP client for console"
+desc = "XMPP client for console"
+info = ["""
+Poezio is a free console XMPP client (the protocol on which the Jabber IM \
+network is built).
+
+Its goal is to let you connect very easily (no account creation needed) to \
+the network and join various chatrooms, immediately. It tries to look like \
+the most famous IRC clients (weechat, irssi, etc). Many commands are identical \
+and you won't be lost if you already know these clients. Configuration can be \
+made in a configuration file or directly from the client.
+"""]
 url = "https://poez.io"
+platform = "FreeBSD and Linux"
 
 [[clients]]
 name = "Psi"
-info = "XMPP client for desktop"
+desc = "XMPP client for desktop"
+info = ["""
+Instant messaging as free and open as it should be.
+
+Psi is a free instant messaging application designed for the XMPP network. \
+Fast and lightweight, Psi is fully open-source and compatible with Windows, \
+Linux, and macOS.
+
+With Psi's full Unicode support and localizations, easy file transfers, \
+customizable iconsets, and many other great features, you'll learn why users \
+around the world are making the switch to free, open instant messaging.
+"""]
 url = "https://psi-im.org"
+platform = "Any"
 
 [[clients]]
 name = "Psi+"
-info = "XMPP client for desktop"
+desc = "XMPP client for desktop"
+info = ["""
+In 2009 a Psi fork named Psi+ was started. Project purpose are: implementation \
+of new features, writing of patches and plugins for transferring them to upstream. \
+As of 2017 the most of active Psi+ developers have become official Psi developers, \
+but Psi+ still has a number of unique features. From developers point of view Psi+ \
+is just a development branch of Psi IM client which is hosted at separate git \
+repositories and for which rolling release development model is used.
+"""]
 url = "https://psi-plus.com"
+platform = "Any"
 
 # [[clients]]
 # name = "Swift"
@@ -689,7 +754,7 @@ url = "https://psi-plus.com"
 # url = "https://yaxim.org"
 
 [[services]]
-title = "Recommended News Services"
+title = "Online Services"
 subtitle = ["""
 Below are online services that extend the syndication experience by means \
 of bookmarking and multimedia, and also enhance it by restoring access to \
@@ -706,7 +771,7 @@ link = "https://www.fivefilters.org/feed-creator/"
 
 [[services]]
 name = "Kill the Newsletter"
-info = "Kill the Newsletter converts email newsletters into Web feeds."
+info = ["Kill the Newsletter converts email newsletters into Web feeds."]
 link = "https://kill-the-newsletter.com"
 
 [[services]]
@@ -737,7 +802,7 @@ It's capable of generating RSS feeds from pretty much everything.
 link = "https://docs.rsshub.app"
 
 [[software]]
-title = "Recommended News Software"
+title = "News Software"
 subtitle = ["""
 Take back control of your news. With free, quality, software for your \
 desktop, home and mobile devices.
@@ -749,7 +814,7 @@ info = ["""
 A self-hosted RSS reader, based on Dropwizard and React/TypeScript.
 """]
 link = "https://commafeed.com"
-os = "Any (HTML)"
+platform = "HTML (Web)"
 
 [[software]]
 name = "FreshRSS"
@@ -758,7 +823,7 @@ FreshRSS is a self-hosted RSS and Atom feed aggregator.
 It is lightweight, easy to work with, powerful, and customizable.
 """]
 link = "https://freshrss.org"
-os = "Any (HTML)"
+platform = "HTML (Web)"
 
 [[software]]
 name = "Liferea"
@@ -769,7 +834,7 @@ it easy to organize and browse feeds. Its GUI is similar to a desktop \
 mail/news client, with an embedded web browser.
 """]
 link = "https://lzone.de/liferea/"
-os = "FreeBSD and Linux"
+platform = "FreeBSD and Linux"
 
 [[software]]
 name = "NetNewsWire"
@@ -787,7 +852,7 @@ can switch to NetNewsWire to get news directly and more reliably from the \
 sites you trust.
 """]
 link = "https://netnewswire.com"
-os = "MacOS"
+platform = "MacOS"
 
 [[software]]
 name = "Newsboat"
@@ -796,7 +861,7 @@ Newsboat is an RSS/Atom feed reader for the text console. It’s an actively \
 maintained fork of Newsbeuter
 """]
 link = "https://newsboat.org"
-os = "Any"
+platform = "HTML (Web)"
 
 [[software]]
 name = "Spot-On"
@@ -806,7 +871,7 @@ search and other forms of communications into a single communications \
 orchestra.
 """]
 link = "https://textbrowser.github.io/spot-on/"
-os = "Any"
+platform = "Any"
 
 [[software]]
 name = "Vienna RSS"
@@ -816,7 +881,7 @@ help you make sense of the flood of information that is distributed via \
 these formats today.
 """]
 link = "https://vienna-rss.com"
-os = "MacOS"
+platform = "MacOS"
 
 [[resources]]
 title = "Useful Resources"
@@ -824,15 +889,34 @@ subtitle = "Technologies which Slixfeed is based upon"
 
 [[resources]]
 name = "feedparser"
-info = "Syndication Library"
+info = "Syndication library"
+desc = "Parse Atom and RSS feeds in Python."
 url = "https://pythonhosted.org/feedparser"
 
 [[resources]]
 name = "Slixmpp"
-info = "XMPP Library"
+info = "XMPP library"
+desc = """
+Slixmpp is an MIT licensed XMPP library for Python 3.7+. It is a fork of \
+SleekXMPP.
+
+Slixmpp's goals is to only rewrite the core of the SleekXMPP library \
+(the low level socket handling, the timers, the events dispatching) \
+in order to remove all threads.
+"""
 url = "https://slixmpp.readthedocs.io"
 
 [[resources]]
 name = "XMPP"
-info = "Messaging Protocol"
+info = "Messaging protocol (also known as Jabber)"
+desc = """
+XMPP is the Extensible Messaging and Presence Protocol, a set of open \
+technologies for instant messaging, presence, multi-party chat, voice and \
+video calls, collaboration, lightweight middleware, content syndication, \
+and generalized routing of XML data.
+
+XMPP was originally developed in the Jabber open-source community to \
+provide an open, decentralized alternative to the closed instant messaging \
+services at that time.
+"""
 url = "https://xmpp.org/about"

slixfeed/assets/feeds.toml

@@ -160,6 +160,30 @@ name = "Κόμμα Πειρατών Ελλάδας – Pirate party of Greece"
 link = "https://www.pirateparty.gr/feed/"
 tags = ["greece", "party", "pirate"]
 
+[[feeds]]
+lang = "en"
+name = "Cycling Together with Fiona and Marc"
+link = "https://pixelfed.social/users/cyclingtogether.atom"
+tags = ["sports", "cycling", "adventure", "life"]
+
+[[feeds]]
+lang = "en"
+name = "Lagrange Gemini Client"
+link = "https://skyjake.fi/@lagrange.rss"
+tags = ["gemini", "gopher", "browser", "telecommunication", "internet"]
+
+[[feeds]]
+lang = "en"
+name = "[ngn.tf] | blog"
+link = "https://api.ngn.tf/blog/feed.atom"
+tags = ["computer", "service", "technology", "telecommunication", "xmpp"]
+
+[[feeds]]
+lang = "en"
+name = "The SWORD Project"
+link = "http://www.crosswire.org/sword/sword.rss.jsp"
+tags = ["bible", "religion", "christianity", "history", "education", "life"]
+
 [[feeds]]
 lang = "en-au"
 name = "Pirate Party Australia"
@@ -268,11 +292,23 @@ name = "The Brexit Party"
 link = "https://www.thebrexitparty.org/feed/"
 tags = ["europe", "politics", "uk"]
 
+[[feeds]]
+lang = "en-us"
+name = "4chan /diy/ - Do It Yourself"
+link = "https://boards.4chan.org/diy/index.rss"
+tags = ["design", "diy", "household"]
+
+[[feeds]]
+lang = "en-us"
+name = "12bytes.org"
+link = "https://12bytes.org/feed.xml"
+tags = ["conspiracy", "health", "government", "war", "world"]
+
 [[feeds]]
 lang = "en-us"
 name = "153 News - Videos Being Watched"
 link = "https://153news.net/rss.php?mode=watching"
-tags = ["news", "politics", "usa", "video"]
+tags = ["europe", "news", "politics", "usa", "video", "world"]
 
 [[feeds]]
 lang = "en-us"
@@ -290,7 +326,7 @@ tags = ["lifestyle", "men"]
 lang = "en-us"
 name = "BlackListed News"
 link = "https://www.blacklistednews.com/rss.php"
-tags = ["news", "politics", "usa", "world"]
+tags = ["conspiracy", "health", "government", "news", "politics", "usa", "world"]
 
 [[feeds]]
 lang = "en-us"
@@ -518,7 +554,7 @@ tags = ["gemini", "internet"]
 lang = "en-us"
 name = "Public Intelligence Blog"
 link = "https://phibetaiota.net/feed/"
-tags = ["cia", "conspiracy", "health", "government", "war"]
+tags = ["cia", "conspiracy", "health", "government", "war", "world"]
 
 [[feeds]]
 lang = "en-us"
@@ -694,6 +730,12 @@ name = "Disroot Blog"
 link = "https://disroot.org/es/blog.atom"
 tags = ["decentralization", "privacy"]
 
+[[feeds]]
+lang = "ch-fr"
+name = "Demoniak Network"
+link = "https://demoniak.ch/index.xml"
+tags = ["computer", "technology"]
+
 [[feeds]]
 lang = "fr-fr"
 name = "Agate Blue"

slixfeed/sqlite.py

@@ -920,7 +920,7 @@ def get_feed_properties(db_file, feed_id):
             """
             SELECT *
             FROM feeds_properties
-            WHERE feed_id = ?
+            WHERE id = :feed_id
             """
             )
         par = (feed_id,)
@@ -1406,20 +1406,20 @@ def get_entries_rejected(db_file, num):
         return result
 
 
-def get_enclosure_by_entry_id(db_file, ix):
+def get_enclosure_by_entry_id(db_file, entry_id):
     function_name = sys._getframe().f_code.co_name
-    logger.debug('{}: db_file: {} ix: {}'
+    logger.debug('{}: db_file: {} entry_id: {}'
-                .format(function_name, db_file, ix))
+                .format(function_name, db_file, entry_id))
     with create_connection(db_file) as conn:
         cur = conn.cursor()
         sql = (
             """
             SELECT url
             FROM entries_properties_links
-            WHERE entry_id = :ix AND rel = "enclosure"
+            WHERE entry_id = :entry_id AND rel = "enclosure"
             """
             )
-        par = (ix,)
+        par = (entry_id,)
         result = cur.execute(sql, par).fetchone()
         return result
 
@@ -1831,6 +1831,24 @@ async def set_feed_title(db_file, feed_id, title):
             cur.execute(sql, par)
 
 
+def get_entry_properties(db_file, ix):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{}: db_file: {} ix: {}'
+                .format(function_name, db_file, ix))
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            SELECT *
+            FROM entries_properties
+            WHERE id = :ix
+            """
+            )
+        par = (ix,)
+        title = cur.execute(sql, par).fetchone()
+        return title
+
+
 def get_entry_title(db_file, ix):
     function_name = sys._getframe().f_code.co_name
     logger.debug('{}: db_file: {} ix: {}'
@@ -2516,6 +2534,98 @@ async def maintain_archive(db_file, limit):
                 cur.execute(sql, par)
 
 
+def get_authors_by_entry_id(db_file, entry_id):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{} db_file: {} entry_id: {}'
+                .format(function_name, db_file, entry_id))
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            SELECT *
+            FROM entries_properties_authors
+            WHERE entry_id = :entry_id
+            ORDER BY name DESC
+            """
+            )
+        par = (entry_id,)
+        result = cur.execute(sql, par).fetchall()
+        return result
+
+
+def get_contributors_by_entry_id(db_file, entry_id):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{} db_file: {} entry_id: {}'
+                .format(function_name, db_file, entry_id))
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            SELECT *
+            FROM entries_properties_contributors
+            WHERE entry_id = :entry_id
+            ORDER BY name DESC
+            """
+            )
+        par = (entry_id,)
+        result = cur.execute(sql, par).fetchall()
+        return result
+
+
+def get_links_by_entry_id(db_file, entry_id):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{}: db_file: {} entry_id: {}'
+                .format(function_name, db_file, entry_id))
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            SELECT *
+            FROM entries_properties_links
+            WHERE entry_id = :entry_id
+            """
+            )
+        par = (entry_id,)
+        result = cur.execute(sql, par).fetchall()
+        return result
+
+
+def get_tags_by_entry_id(db_file, entry_id):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{}: db_file: {} entry_id: {}'
+                .format(function_name, db_file, entry_id))
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            SELECT *
+            FROM entries_properties_tags
+            WHERE entry_id = :entry_id
+            """
+            )
+        par = (entry_id,)
+        result = cur.execute(sql, par).fetchall()
+        return result
+
+
+def get_contents_by_entry_id(db_file, entry_id):
+    function_name = sys._getframe().f_code.co_name
+    logger.debug('{}: db_file: {} entry_id: {}'
+                .format(function_name, db_file, entry_id))
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            SELECT *
+            FROM entries_properties_contents
+            WHERE entry_id = :entry_id
+            """
+            )
+        par = (entry_id,)
+        result = cur.execute(sql, par).fetchall()
+        return result
+
+
 # TODO Move entries that don't exist into table archive.
 # NOTE Entries that are read from archive are deleted.
 # NOTE Unlike entries from table entries, entries from
@@ -2538,7 +2648,7 @@ def get_entries_of_feed(db_file, feed_id):
         cur = conn.cursor()
         sql = (
             """
-            SELECT id, title, link, identifier, published, read
+            SELECT id, title, link, identifier, published
             FROM entries_properties
             WHERE feed_id = ?
             ORDER BY published DESC

slixfeed/task.py

@@ -170,7 +170,7 @@ async def task_publish(self, jid_bare):
     if jid_bare not in self.settings:
         Config.add_settings_jid(self.settings, jid_bare, db_file)
     while True:
-        await action.xmpp_send_pubsub(self, jid_bare)
+        await action.xmpp_pubsub_send_unread_items(self, jid_bare)
         await asyncio.sleep(60 * 180)
 
 
@@ -260,7 +260,7 @@ async def task_message(self, jid_bare):
         await sqlite.update_last_update_time(db_file)
     else:
         await sqlite.set_last_update_time(db_file)
-    await action.xmpp_send_message(self, jid_bare)
+    await action.xmpp_chat_send_unread_items(self, jid_bare)
     refresh_task(self, jid_bare, task_message, 'interval')
     await start_tasks_xmpp_chat(self, jid_bare, ['status'])
 

slixfeed/version.py

@@ -1,2 +1,2 @@
-__version__ = '0.1.57'
+__version__ = '0.1.58'
-__version_info__ = (0, 1, 57)
+__version_info__ = (0, 1, 58)

slixfeed/xmpp/bookmark.py

@@ -21,7 +21,7 @@ class XmppBookmark:
         return conferences
 
 
-    async def properties(self, jid):
+    async def get_bookmark_properties(self, jid):
         result = await self.plugin['xep_0048'].get_bookmarks()
         groupchats = result['private']['bookmarks']['conferences']
         for groupchat in groupchats:

slixfeed/xmpp/component.py

@@ -42,7 +42,7 @@ from slixfeed.xmpp.connect import XmppConnect
 from slixfeed.xmpp.muc import XmppGroupchat
 from slixfeed.xmpp.iq import XmppIQ
 from slixfeed.xmpp.message import XmppMessage
-import slixfeed.xmpp.process as process
+from slixfeed.xmpp.chat import Chat
 import slixfeed.xmpp.profile as profile
 from slixfeed.xmpp.publish import XmppPubsub
 # from slixfeed.xmpp.roster import XmppRoster
@@ -330,7 +330,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
                 self.pending_tasks[jid_bare] = {}
             # if jid_full not in self.pending_tasks:
             #     self.pending_tasks[jid_full] = {}
-            await process.message(self, message)
+            await Chat.process_message(self, message)
         # chat_type = message["type"]
         # message_body = message["body"]
         # message_reply = message.reply

slixfeed/xmpp/publish.py

@@ -7,7 +7,6 @@ Functions create_node and create_entry are derived from project atomtopubsub.
 
 """
 
-import hashlib
 import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub
 from slixmpp.xmlstream import ET
 
@@ -32,6 +31,33 @@ class XmppPubsub:
         return results
 
 
+    async def get_node_properties(self, jid, node):
+        config = await self.plugin['xep_0060'].get_node_config(jid, node)
+        subscriptions = await self.plugin['xep_0060'].get_node_subscriptions(jid, node)
+        affiliations = await self.plugin['xep_0060'].get_node_affiliations(jid, node)
+        properties = {'config': config,
+                      'subscriptions': subscriptions,
+                      'affiliations': affiliations}
+        breakpoint()
+        return properties
+
+
+    async def get_nodes(self, jid):
+        nodes = await self.plugin['xep_0060'].get_nodes(jid)
+        # 'self' would lead to slixmpp.jid.InvalidJID: idna validation failed:
+        return nodes
+
+
+    async def get_item(self, jid, node, item_id):
+        item = await self.plugin['xep_0060'].get_item(jid, node, item_id)
+        return item
+
+
+    async def get_items(self, jid, node):
+        items = await self.plugin['xep_0060'].get_items(jid, node)
+        return items
+
+
     def delete_node(self, jid, node):
         jid_from = str(self.boundjid) if self.is_component else None
         self.plugin['xep_0060'].delete_node(jid, node, ifrom=jid_from)
@@ -87,9 +113,11 @@ class XmppPubsub:
         return iq
 
 
-    def create_entry(self, jid, node, entry, version):
+    # TODO Consider to create a separate function called "create_atom_entry"
+    # or "create_rfc4287_entry" for anything related to variable "node_entry".
+    def create_entry(self, jid, node_id, item_id, node_item):
         iq = self.Iq(stype="set", sto=jid)
-        iq['pubsub']['publish']['node'] = node
+        iq['pubsub']['publish']['node'] = node_id
 
         item = pubsub.Item()
 
@@ -102,33 +130,8 @@ class XmppPubsub:
         # cross reference, and namely - in another project to utilize PubSub as
         # links sharing system (see del.icio.us) - to share node entries.
 
-        # NOTE Warning: Entry might not have a link
+        item['id'] = item_id
-        # TODO Handle situation error
+        item['payload'] = node_item
-        url_encoded = entry['link'].encode()
-        url_hashed = hashlib.md5(url_encoded)
-        url_digest = url_hashed.hexdigest()
-        item['id'] = url_digest
-
-        node_entry = ET.Element("entry")
-        node_entry.set('xmlns', 'http://www.w3.org/2005/Atom')
-
-        title = ET.SubElement(node_entry, "title")
-        title.text = entry['title']
-
-        updated = ET.SubElement(node_entry, "updated")
-        updated.text = entry['updated']
-
-        # Content
-        content = ET.SubElement(node_entry, "content")
-        content.set('type', 'text/html')
-        content.text = entry['description']
-
-        # Links
-        link = ET.SubElement(node_entry, "link")
-        link.set('href', entry['link'])
-
-        item['payload'] = node_entry
-
         iq['pubsub']['publish'].append(item)
 
         return iq