WIP: Closer to fix double message. See task.py

slixfeed/__main__.py

@@ -8,9 +8,9 @@ FIXME
     Consequently, it might result in database lock error upon
     feed removal attempt
 
-TODO
+2) Communicate to messages of new contacts (not subscribed and not in roster)
 
-1) SQL prepared statements;
+TODO
 
 2) Machine Learning for scrapping Title, Link, Summary and Timstamp;
    Scrape element </article> (example: Liferea)
@@ -22,15 +22,9 @@ TODO
    Perhaps not, as it would require to check every feed for this setting.
    Maybe a separate bot;
 
-4) Support categories;
+5) OMEMO;
 
-5) XMPP commands;
+6) Logging;
-
-6) Bot as service;
-
-7) OMEMO;
-
-8) Logging;
    https://docs.python.org/3/howto/logging.html
 
 9) Readability
@@ -43,12 +37,6 @@ TODO
     Store 5 upcoming summaries.
     This would help making the database files smaller.
 
-11) Support protocol Gopher
-    See project /michael-lazar/pygopherd
-    See project /gopherball/gb
-
-12) Support ActivityPub @person@domain (see Tip Of The Day).
-
 13) Tip Of The Day.
     Did you know that you can follow you favorite Mastodon feeds by just
     sending the URL address?
@@ -58,20 +46,14 @@ TODO
 
 14) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger
 
-15) See project /offpunk/offblocklist.py
-
 16) Search messages of government regulated publishers, and promote other sources.
     Dear reader, we couldn't get news from XYZ as they don't provide RSS feeds.
     However, you might want to get news from (1) (2) and (3) instead!
 
-17) Make the program portable (directly use the directory assets) -- Thorsten
+17) The operator account will be given reports from the bot about its
-
-18) The operator account will be given reports from the bot about its
     activities every X minutes.
     When a suspicious activity is detected, it will be reported immediately.
 
-19) Communicate to messages of new contacts (not subscribed and not in roster)
-
 """
 
 # vars and their meanings:

slixfeed/action.py

@@ -86,6 +86,7 @@ except ImportError:
         "Package readability was not found.\n"
         "Arc90 Lab algorithm is disabled.")
 
+
 def manual(filename, section=None, command=None):
     config_dir = config.get_default_config_directory()
     with open(config_dir + '/' + filename, mode="rb") as commands:
@@ -111,7 +112,8 @@ def manual(filename, section=None, command=None):
     return cmd_list
 
 
-async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, session=None):
+async def xmpp_change_interval(self, key, val, jid, jid_file, message=None,
+                               session=None):
     if val:
         # response = (
         #     'Updates will be sent every {} minutes.'
@@ -123,7 +125,7 @@ async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, sess
             await sqlite.set_settings_value(db_file, [key, val])
         # NOTE Perhaps this should be replaced
         # by functions clean and start
-        await task.refresh_task(self, jid, task.send_update, key, val)
+        await task.refresh_task(self, jid, task.task_send, key, val)
         response = ('Updates will be sent every {} minutes.'
                     .format(val))
     else:
@@ -344,17 +346,17 @@ def list_search_results(query, results):
     return message
 
 
-def list_feeds_by_query(query, results):
+def list_feeds_by_query(db_file, query):
+    results = sqlite.search_feeds(db_file, query)
     message = (
-        "Feeds containing '{}':\n\n```"
+        'Feeds containing "{}":\n\n```'
-        ).format(query)
+        .format(query))
     for result in results:
         message += (
-            "\nName : {} [{}]"
+            '\nName : {} [{}]'
-            "\nURL  : {}"
+            '\nURL  : {}'
-            "\n"
+            '\n'
-            ).format(
+            .format(str(result[0]), str(result[1]), str(result[2])))
-                str(result[0]), str(result[1]), str(result[2]))
     if len(results):
         message += "\n```\nTotal of {} feeds".format(len(results))
     else:

slixfeed/assets/commands.toml

@@ -141,15 +141,19 @@ interval <minutes>
 Set interval update to every given <minutes>.
 """
 length = """
-length
+length <number>
 Set maximum length of news item description. (0 for no limit)
 """
+media = """
+media [off|on]
+Attach media (i.e. audio, image, video) to messages when available.
+"""
 quantum = """
 quantum <number>
 Set amount of updates per message by given <number>.
 """
 random = """
-random
+random [off|on]
 Send messages by random order instead of date.
 """
 
@@ -190,7 +194,7 @@ Disable bot and stop updates.
 [preview]
 read = """
 read <url>
-Display most recent 5 titles of given <url>.
+Display recent 5 titles of given <url>.
 """
 read_num = """
 read <url> <index>

slixfeed/assets/information.toml

@@ -3,10 +3,10 @@ Slixfeed
 
 A Syndication bot for the XMPP communication network.
 
-Slixfeed aims to be an easy to use and fully-featured news \
+Slixfeed is a news broker which aims to be an easy to use and fully-\
-aggregator bot for XMPP.  It provides a convenient access to Blogs, \
+featured news aggregator bot.  It provides a convenient access to \
-News websites and even Fediverse instances, along with filtering \
+Blogs, News websites and even Fediverse instances, along with \
-functionality.
+filtering functionality.
 
 Slixfeed is primarily designed for XMPP (aka Jabber). \
 Visit https://xmpp.org/software/ for more information.
@@ -61,7 +61,7 @@ No operator was specified for this instance.
 
 platforms = """
 Supported platforms: XMPP
-Platforms to be added in future: ActivityPub, Briar, Email, IRC, LXMF, Matrix, MQTT, Nostr, Tox.
+Platforms to be added in future: ActivityPub, Briar, Email, IRC, LXMF, Matrix, MQTT, Nostr, Session, Tox.
 For ideal experience, we recommend using XMPP.
 """
 
@@ -78,10 +78,15 @@ Protocols to be added in future: Dat, FTP, Gemini, Gopher, IPFS.
 resources = """
 Slixfeed
 https://gitgud.io/sjehuda/slixfeed
+
 Slixmpp
 https://slixmpp.readthedocs.io/
+
 feedparser
 https://pythonhosted.org/feedparser
+
+XMPP
+https://xmpp.org/about/
 """
 
 terms = """
@@ -113,13 +118,14 @@ imattau (atomtopubsub), \
 Jaussoin Timothée <mov.im> (Movim, France), \
 Justin Karneges <jblog.andbit.net> (Psi, California), \
 Kevin Smith <isode.com> (Swift IM, Wales), \
+Lars Windolf (Liferea, Germany), \
 Luis Henrique Mello (SalixOS, Brazil), \
 magicfelix, \
 Markus Muttilainen (SalixOS), \
 Martin <debacle@debian.org> (Debian, Germany), \
 Mathieu Pasquet (slixmpp, France), \
 Maxime Buquet (slixmpp, France), \
-Phillip Watkins (United Kingdom, SalixOS), \
+Phillip Watkins (SalixOS, United Kingdom), \
 Pierrick Le Brun (SalixOS, France), \
 Raphael Groner (Fedora, Germany), \
 Remko Tronçon <mko.re> (Psi , Belgium), \

slixfeed/config.py

@@ -19,6 +19,10 @@ TODO
 
 6) Use TOML https://ruudvanasseldonk.com/2023/01/11/the-yaml-document-from-hell
 
+7) Make the program portable (directly use the directory assets) -- Thorsten
+
+7.1) Read missing files from base directories or either set error message.
+
 """
 
 import configparser

slixfeed/crawl.py

@@ -3,6 +3,14 @@
 
 """
 
+FIXME
+
+1) https://wiki.pine64.org
+     File "/slixfeed/crawl.py", line 178, in feed_mode_guess
+       address = join_url(url, parted_url.path.split('/')[1] + path)
+                               ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^
+   IndexError: list index out of range
+
 TODO
 
 1.1) Attempt to scan more paths: /blog/, /news/ etc., including root / 

slixfeed/fetch.py

@@ -21,6 +21,14 @@ TODO
 4) Replace sqlite.remove_nonexistent_entries by sqlite.check_entry_exist
    Same check, just reverse.
 
+5) Support protocol Gopher
+   See project /michael-lazar/pygopherd
+   See project /gopherball/gb
+
+6) Support ActivityPub @person@domain (see Tip Of The Day).
+    
+7) See project /offpunk/offblocklist.py
+
 """
 
 from aiohttp import ClientError, ClientSession, ClientTimeout
@@ -41,18 +49,25 @@ except:
         "BitTorrent is disabled.")
 
 
+# class FetchDat:
 # async def dat():
 
+# class FetchFtp:
 # async def ftp():
 
+# class FetchGemini:
 # async def gemini():
 
+# class FetchGopher:
 # async def gopher():
 
+# class FetchHttp:
 # async def http():
 
+# class FetchIpfs:
 # async def ipfs():
 
+
 def http_response(url):
     """
     Download response headers.

slixfeed/sqlite.py

@@ -10,6 +10,11 @@ TODO
    All other functions to receive cursor.
 
 2) Merge function add_metadata into function import_feeds.
+
+3) SQL prepared statements.
+
+4) Support categories;
+
 """
 
 from asyncio import Lock
@@ -66,6 +71,51 @@ def create_tables(db_file):
         Path to database file.
     """
     with create_connection(db_file) as conn:
+        archive_table_sql = (
+            """
+            CREATE TABLE IF NOT EXISTS archive (
+                id INTEGER NOT NULL,
+                title TEXT NOT NULL,
+                link TEXT NOT NULL,
+                enclosure TEXT,
+                entry_id TEXT NOT NULL,
+                feed_id INTEGER NOT NULL,
+                timestamp TEXT,
+                read INTEGER NOT NULL DEFAULT 0,
+                FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
+                  ON UPDATE CASCADE
+                  ON DELETE CASCADE,
+                PRIMARY KEY ("id")
+              );
+            """
+            )
+        categories_table_sql = (
+            """
+            CREATE TABLE IF NOT EXISTS categories (
+                id INTEGER NOT NULL,
+                name TEXT,
+                PRIMARY KEY ("id")
+              );
+            """
+            )
+        entries_table_sql = (
+            """
+            CREATE TABLE IF NOT EXISTS entries (
+                id INTEGER NOT NULL,
+                title TEXT NOT NULL,
+                link TEXT NOT NULL,
+                enclosure TEXT,
+                entry_id TEXT NOT NULL,
+                feed_id INTEGER NOT NULL,
+                timestamp TEXT,
+                read INTEGER NOT NULL DEFAULT 0,
+                FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
+                  ON UPDATE CASCADE
+                  ON DELETE CASCADE,
+                PRIMARY KEY ("id")
+              );
+            """
+            )
         feeds_table_sql = (
             """
             CREATE TABLE IF NOT EXISTS feeds (
@@ -76,18 +126,19 @@ def create_tables(db_file):
               );
             """
             )
-        feeds_statistics_table_sql = (
+        # TODO Rethink!
+        # Albeit, probably, more expensive, we might want to have feed_id
+        # as foreign key, as it is with feeds_properties and feeds_state
+        feeds_categories_table_sql = (
             """
-            CREATE TABLE IF NOT EXISTS statistics (
+            CREATE TABLE IF NOT EXISTS feeds_categories (
                 id INTEGER NOT NULL,
-                feed_id INTEGER NOT NULL UNIQUE,
+                category_id INTEGER NOT NULL UNIQUE,
-                offline INTEGER,
+                feed_id INTEGER,
-                entries INTEGER,
+                FOREIGN KEY ("category_id") REFERENCES "categories" ("id")
-                entries INTEGER,
-                FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
                   ON UPDATE CASCADE
                   ON DELETE CASCADE,
-                PRIMARY KEY ("id")
+                PRIMARY KEY (id)
               );
             """
             )
@@ -127,53 +178,32 @@ def create_tables(db_file):
               );
             """
             )
+        feeds_statistics_table_sql = (
+            """
+            CREATE TABLE IF NOT EXISTS statistics (
+                id INTEGER NOT NULL,
+                feed_id INTEGER NOT NULL UNIQUE,
+                offline INTEGER,
+                entries INTEGER,
+                entries INTEGER,
+                FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
+                  ON UPDATE CASCADE
+                  ON DELETE CASCADE,
+                PRIMARY KEY ("id")
+              );
+            """
+            )
         # TODO
         # Consider parameter unique:
         # entry_id TEXT NOT NULL UNIQUE,
         # Will eliminate function:
         # check_entry_exist
-        entries_table_sql = (
+        filters_table_sql = (
             """
-            CREATE TABLE IF NOT EXISTS entries (
+            CREATE TABLE IF NOT EXISTS filters (
-                id INTEGER NOT NULL,
-                title TEXT NOT NULL,
-                link TEXT NOT NULL,
-                enclosure TEXT,
-                entry_id TEXT NOT NULL,
-                feed_id INTEGER NOT NULL,
-                timestamp TEXT,
-                read INTEGER NOT NULL DEFAULT 0,
-                FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
-                  ON UPDATE CASCADE
-                  ON DELETE CASCADE,
-                PRIMARY KEY ("id")
-              );
-            """
-            )
-        archive_table_sql = (
-            """
-            CREATE TABLE IF NOT EXISTS archive (
-                id INTEGER NOT NULL,
-                title TEXT NOT NULL,
-                link TEXT NOT NULL,
-                enclosure TEXT,
-                entry_id TEXT NOT NULL,
-                feed_id INTEGER NOT NULL,
-                timestamp TEXT,
-                read INTEGER NOT NULL DEFAULT 0,
-                FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
-                  ON UPDATE CASCADE
-                  ON DELETE CASCADE,
-                PRIMARY KEY ("id")
-              );
-            """
-            )
-        status_table_sql = (
-            """
-            CREATE TABLE IF NOT EXISTS status (
                 id INTEGER NOT NULL,
                 key TEXT NOT NULL,
-                value INTEGER,
+                value TEXT,
                 PRIMARY KEY ("id")
               );
             """
@@ -188,12 +218,12 @@ def create_tables(db_file):
               );
             """
             )
-        filters_table_sql = (
+        status_table_sql = (
             """
-            CREATE TABLE IF NOT EXISTS filters (
+            CREATE TABLE IF NOT EXISTS status (
                 id INTEGER NOT NULL,
                 key TEXT NOT NULL,
-                value TEXT,
+                value INTEGER,
                 PRIMARY KEY ("id")
               );
             """
@@ -863,17 +893,17 @@ async def archive_entry(db_file, ix):
                     )
 
 
-def get_feed_title(db_file, ix):
+def get_feed_title(db_file, feed_id):
     with create_connection(db_file) as conn:
         cur = conn.cursor()
         sql = (
             """
             SELECT name
             FROM feeds
-            WHERE id = :ix
+            WHERE id = :feed_id
             """
             )
-        par = (ix,)
+        par = (feed_id,)
         title = cur.execute(sql, par).fetchone()
         return title
 
@@ -1520,7 +1550,7 @@ async def last_entries(db_file, num):
         return results
 
 
-async def search_feeds(db_file, query):
+def search_feeds(db_file, query):
     """
     Query feeds.
 

slixfeed/task.py

@@ -5,12 +5,18 @@
 
 FIXME
 
+0) URGENT!!! Place "await asyncio.sleep(next_update_time)" ***inside*** the
+   task, and not outside as it is now!
+
+
 1) Function check_readiness or event "changed_status" is causing for
    triple status messages and also false ones that indicate of lack
    of feeds.
 
 TODO
 
+0) Move functions send_status and send_update to module action
+
 1) Deprecate "add" (see above) and make it interactive.
    Slixfeed: Do you still want to add this URL to subscription list?
    See: case _ if message_lowercase.startswith("add"):
@@ -70,13 +76,13 @@ loop = asyncio.get_event_loop()
 #     task_ping = asyncio.create_task(ping(self, jid=None))
 
 
-def ping_task(self):
+def task_ping(self):
-    # global ping_task_instance
+    # global task_ping_instance
     try:
-        self.ping_task_instance.cancel()
+        self.task_ping_instance.cancel()
     except:
         logging.info('No ping task to cancel.')
-    self.ping_task_instance = asyncio.create_task(XmppConnect.ping(self))
+    self.task_ping_instance = asyncio.create_task(XmppConnect.ping(self))
 
 
 """
@@ -133,34 +139,8 @@ async def start_tasks_xmpp(self, jid, tasks=None):
                 self.task_manager[jid]['status'] = asyncio.create_task(
                     send_status(self, jid))
             case 'interval':
-                jid_file = jid.replace('/', '_')
-                db_file = config.get_pathname_to_database(jid_file)
-                update_interval = await config.get_setting_value(db_file,
-                                                                 'interval')
-                update_interval = 60 * int(update_interval)
-                last_update_time = await sqlite.get_last_update_time(db_file)
-                if last_update_time:
-                    last_update_time = float(last_update_time)
-                    diff = time.time() - last_update_time
-                    if diff < update_interval:
-                        next_update_time = update_interval - diff
-                        await asyncio.sleep(next_update_time)
-
-                        # print("jid              :", jid, "\n"
-                        #       "time             :", time.time(), "\n"
-                        #       "last_update_time :", last_update_time, "\n"
-                        #       "difference       :", diff, "\n"
-                        #       "update interval  :", update_interval, "\n"
-                        #       "next_update_time :", next_update_time, "\n"
-                        #       )
-
-                    # elif diff > val:
-                    #     next_update_time = val
-                    await sqlite.update_last_update_time(db_file)
-                else:
-                    await sqlite.set_last_update_time(db_file)
                 self.task_manager[jid]['interval'] = asyncio.create_task(
-                    send_update(self, jid))
+                    task_send(self, jid))
     # for task in self.task_manager[jid].values():
     #     print("task_manager[jid].values()")
     #     print(self.task_manager[jid].values())
@@ -172,20 +152,57 @@ async def start_tasks_xmpp(self, jid, tasks=None):
     #     await task
 
 
-def clean_tasks_xmpp(self, jid, tasks=None):
+async def task_send(self, jid):
-    if not tasks:
+    print("task_send for", jid)
-        tasks = ['interval', 'status', 'check']
-    logging.info('Stopping tasks {} for JID {}'.format(tasks, jid))
-    for task in tasks:
-        # if self.task_manager[jid][task]:
     try:
-            self.task_manager[jid][task].cancel()
+        self.task_manager[jid]['interval'].cancel()
     except:
-            logging.debug('No task {} for JID {} (clean_tasks_xmpp)'
+        logging.info('No task interval for JID {} (start_tasks_xmpp)'
-                          .format(task, jid))
+                     .format(jid))
+    jid_file = jid.replace('/', '_')
+    print(jid_file)
+    db_file = config.get_pathname_to_database(jid_file)
+    print(db_file)
+    update_interval = await config.get_setting_value(db_file, 'interval')
+    print(update_interval)
+    update_interval = 60 * int(update_interval)
+    print(update_interval)
+    last_update_time = await sqlite.get_last_update_time(db_file)
+    print(last_update_time)
+    if last_update_time:
+        print('if')
+        last_update_time = float(last_update_time)
+        diff = time.time() - last_update_time
+        if diff < update_interval:
+            next_update_time = update_interval - diff
+            print('next_update_time')
+            print(next_update_time)
+            print(next_update_time/60/60)
+            await asyncio.sleep(next_update_time) # FIXME!
+            print('after await sleep')
+
+            # print("jid              :", jid, "\n"
+            #       "time             :", time.time(), "\n"
+            #       "last_update_time :", last_update_time, "\n"
+            #       "difference       :", diff, "\n"
+            #       "update interval  :", update_interval, "\n"
+            #       "next_update_time :", next_update_time, "\n"
+            #       )
+
+        # elif diff > val:
+        #     next_update_time = val
+        print('await (if)')
+        await sqlite.update_last_update_time(db_file)
+    else:
+        print('await (else)')
+        await sqlite.set_last_update_time(db_file)
+    print("await is done for", jid)
+    await xmpp_send_update(self, jid)
+    await start_tasks_xmpp(self, jid, ['status'])
+    await refresh_task(self, jid, task_send, 'interval')
 
 
-async def send_update(self, jid, num=None):
+async def xmpp_send_update(self, jid, num=None):
     """
     Send news items as messages.
 
@@ -196,21 +213,28 @@ async def send_update(self, jid, num=None):
     num : str, optional
         Number. The default is None.
     """
-    logging.info('Sending a news update to JID {}'.format(jid))
+    print('Sending a news update to JID {}'.format(jid))
     jid_file = jid.replace('/', '_')
+    print(jid_file)
     db_file = config.get_pathname_to_database(jid_file)
+    print(db_file)
     enabled = await config.get_setting_value(db_file, 'enabled')
+    print(enabled)
     if enabled:
+        print('enabled')
         if not num:
             num = await config.get_setting_value(db_file, 'quantum')
         else:
             num = int(num)
-        news_digest = []
+        print(num)
         results = await sqlite.get_unread_entries(db_file, num)
+        print(results)
         news_digest = ''
         media = None
         chat_type = await get_chat_type(self, jid)
+        print(jid, num, chat_type)
         for result in results:
+            print(result)
             ix = result[0]
             title_e = result[1]
             url = result[2]
@@ -239,6 +263,7 @@ async def send_update(self, jid, num=None):
             if media and news_digest:
                 print('SENDING MESSAGE (if media and news_digest)')
                 print(news_digest)
+                print(media)
                 # Send textual message
                 XmppMessage.send(self, jid, news_digest, chat_type)
                 news_digest = ''
@@ -249,12 +274,13 @@ async def send_update(self, jid, num=None):
         if news_digest:
             print('SENDING MESSAGE (if news_digest)')
             print(news_digest)
+            XmppMessage.send(self, jid, news_digest, chat_type)
             # TODO Add while loop to assure delivery.
             # print(await current_time(), ">>> ACT send_message",jid)
             # NOTE Do we need "if statement"? See NOTE at is_muc.
-            if chat_type in ('chat', 'groupchat'):
+            # if chat_type in ('chat', 'groupchat'):
-                # TODO Provide a choice (with or without images)
+            #     # TODO Provide a choice (with or without images)
-                XmppMessage.send(self, jid, news_digest, chat_type)
+            #     XmppMessage.send(self, jid, news_digest, chat_type)
         # See XEP-0367
         # if media:
         #     # message = xmpp.Slixfeed.make_message(
@@ -266,7 +292,10 @@ async def send_update(self, jid, num=None):
                 
         # TODO Do not refresh task before
         # verifying that it was completed.
-        await refresh_task(self, jid, send_update, 'interval')
+
+        # await start_tasks_xmpp(self, jid, ['status'])
+        # await refresh_task(self, jid, send_update, 'interval')
+
     # interval = await initdb(
     #     jid,
     #     sqlite.get_settings_value,
@@ -292,6 +321,19 @@ async def send_update(self, jid, num=None):
     # await handle_event()
 
 
+def clean_tasks_xmpp(self, jid, tasks=None):
+    if not tasks:
+        tasks = ['interval', 'status', 'check']
+    logging.info('Stopping tasks {} for JID {}'.format(tasks, jid))
+    for task in tasks:
+        # if self.task_manager[jid][task]:
+        try:
+            self.task_manager[jid][task].cancel()
+        except:
+            logging.debug('No task {} for JID {} (clean_tasks_xmpp)'
+                          .format(task, jid))
+
+
 async def send_status(self, jid):
     """
     Send status message.

slixfeed/url.py

@@ -3,6 +3,13 @@
 
 """
 
+FIXME
+
+1) Do not handle base64
+   https://www.lilithsaintcrow.com/2024/02/love-anonymous/
+   data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAeAAQAAAAAQ6M16AAAAAnRSTlMAAHaTzTgAAAFmSURBVBgZ7cEBAQAAAIKg/q92SMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgWE3LAAGyZmPPAAAAAElFTkSuQmCC
+   https://www.lilithsaintcrow.com/2024/02/love-anonymous//image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAeAAQAAAAAQ6M16AAAAAnRSTlMAAHaTzTgAAAFmSURBVBgZ7cEBAQAAAIKg/q92SMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgWE3LAAGyZmPPAAAAAElFTkSuQmCC
+
 TODO
 
 1) ActivityPub URL revealer activitypub_to_http.
@@ -40,6 +47,7 @@ def get_hostname(url):
     parted_url = urlsplit(url)
     return parted_url.netloc
 
+
 def replace_hostname(url, url_type):
     """
     Replace hostname.
@@ -120,6 +128,8 @@ def remove_tracking_parameters(url):
     url : str
         URL.
     """
+    if url.startswith('data:') and ';base64,' in url:
+        return url
     parted_url = urlsplit(url)
     protocol = parted_url.scheme
     hostname = parted_url.netloc
@@ -192,6 +202,8 @@ def complete_url(source, link):
     str
         URL.
     """
+    if link.startswith('data:') and ';base64,' in link:
+        return link
     if link.startswith('www.'):
         return 'http://' + link
     parted_link = urlsplit(link)
@@ -270,6 +282,8 @@ def join_url(source, link):
     str
         URL.
     """
+    if link.startswith('data:') and ';base64,' in link:
+        return link
     if link.startswith('www.'):
         new_link = 'http://' + link
     elif link.startswith('%20') and link.endswith('%20'):
@@ -296,6 +310,8 @@ def trim_url(url):
     url : str
         URL.
     """
+    if url.startswith('data:') and ';base64,' in url:
+        return url
     parted_url = urlsplit(url)
     protocol = parted_url.scheme
     hostname = parted_url.netloc

slixfeed/version.py

@@ -1,2 +1,2 @@
-__version__ = '0.1.2'
+__version__ = '0.1.3'
-__version_info__ = (0, 1, 2)
+__version_info__ = (0, 1, 3)

slixfeed/xmpp/client.py

@@ -108,7 +108,7 @@ class Slixfeed(slixmpp.ClientXMPP):
         self.task_manager = {}
 
         # Handlers for ping
-        self.ping_task_instance = {}
+        self.task_ping_instance = {}
 
         # Handlers for connection events
         self.connection_attempts = 0
@@ -124,8 +124,8 @@ class Slixfeed(slixmpp.ClientXMPP):
                                self.on_changed_status)
         self.add_event_handler("presence_available",
                                self.on_presence_available)
-        self.add_event_handler("presence_unavailable",
+        # self.add_event_handler("presence_unavailable",
-                               self.on_presence_unavailable)
+        #                        self.on_presence_unavailable)
         self.add_event_handler("chatstate_active",
                                self.on_chatstate_active)
         self.add_event_handler("chatstate_composing",
@@ -218,7 +218,7 @@ class Slixfeed(slixmpp.ClientXMPP):
         await XmppGroupchat.autojoin(self)
         profile.set_identity(self, 'client')
         await profile.update(self)
-        task.ping_task(self)
+        task.task_ping(self)
         
         # Service.commands(self)
         # Service.reactions(self)
@@ -262,6 +262,8 @@ class Slixfeed(slixmpp.ClientXMPP):
     async def on_changed_status(self, presence):
         # await task.check_readiness(self, presence)
         jid = presence['from'].bare
+        if jid in self.boundjid.bare:
+            return
         if presence['show'] in ('away', 'dnd', 'xa'):
             task.clean_tasks_xmpp(self, jid, ['interval'])
             await task.start_tasks_xmpp(self, jid, ['status', 'check'])
@@ -270,6 +272,7 @@ class Slixfeed(slixmpp.ClientXMPP):
     async def on_presence_subscribe(self, presence):
         jid = presence['from'].bare
         if not self.client_roster[jid]['to']:
+        # XmppPresence.subscription(self, jid, 'subscribe')
             XmppPresence.subscription(self, jid, 'subscribed')
             await XmppRoster.add(self, jid)
             status_message = '✒️ Share online status to receive updates'
@@ -280,8 +283,9 @@ class Slixfeed(slixmpp.ClientXMPP):
                                       'chat')
 
 
-    async def on_presence_subscribed(self, presence):
+    def on_presence_subscribed(self, presence):
         jid = presence['from'].bare
+        # XmppPresence.subscription(self, jid, 'subscribed')
         message_subject = 'RSS News Bot'
         message_body = ('Greetings! I am {}, the news anchor.\n'
                         'My job is to bring you the latest '
@@ -297,13 +301,18 @@ class Slixfeed(slixmpp.ClientXMPP):
         # await task.start_tasks(self, presence)
         # NOTE Already done inside the start-task function
         jid = presence['from'].bare
+        if jid in self.boundjid.bare:
+            return
+        print('JID available:', jid)
         # FIXME TODO Find out what is the source responsible for a couple presences with empty message
         # NOTE This is a temporary solution
         await asyncio.sleep(10)
         await task.start_tasks_xmpp(self, jid)
+        self.add_event_handler("presence_unavailable",
+                               self.on_presence_unavailable)
 
 
-    async def on_presence_unsubscribed(self, presence):
+    def on_presence_unsubscribed(self, presence):
         jid = presence['from'].bare
         message_body = 'You have been unsubscribed.'
         # status_message = '🖋️ Subscribe to receive updates'
@@ -312,64 +321,81 @@ class Slixfeed(slixmpp.ClientXMPP):
         XmppPresence.subscription(self, jid, 'unsubscribed')
         # XmppPresence.send(self, jid, status_message,
         #                   presence_type='unsubscribed')
-        await XmppRoster.remove(self, jid)
+        XmppRoster.remove(self, jid)
 
 
-    async def on_presence_unavailable(self, presence):
+    def on_presence_unavailable(self, presence):
         jid = presence['from'].bare
+        print('JID unavailable:', jid)
         # await task.stop_tasks(self, jid)
         task.clean_tasks_xmpp(self, jid)
 
+        # NOTE Albeit nice to ~have~ see, this would constantly
+        #      send presence messages to server to no end.
+        status_message = 'Farewell'
+        XmppPresence.send(self, jid, status_message,
+                          presence_type='unavailable')
+        self.del_event_handler("presence_unavailable",
+                               self.on_presence_unavailable)
+
 
     # TODO
     # Send message that database will be deleted within 30 days
     # Check whether JID is in bookmarks or roster
     # If roster, remove contact JID into file 
     # If bookmarks, remove groupchat JID into file 
-    async def on_presence_error(self, presence):
+    def on_presence_error(self, presence):
-        print("on_presence_error")
-        print(presence)
         jid = presence["from"].bare
+        print('JID error:', jid)
         task.clean_tasks_xmpp(self, jid)
 
 
-    async def on_reactions(self, message):
+    def on_reactions(self, message):
         print(message['from'])
         print(message['reactions']['values'])
 
 
     async def on_chatstate_active(self, message):
-        if message['type'] in ('chat', 'normal'):
         jid = message['from'].bare
+        if jid in self.boundjid.bare:
+            return
+        if message['type'] in ('chat', 'normal'):
             # task.clean_tasks_xmpp(self, jid, ['status'])
             await task.start_tasks_xmpp(self, jid, ['status'])
 
 
-    async def on_chatstate_composing(self, message):
+    def on_chatstate_composing(self, message):
         if message['type'] in ('chat', 'normal'):
             jid = message['from'].bare
             # task.clean_tasks_xmpp(self, jid, ['status'])
-            status_message='💡 Press "help" for manual, or "info" for information.'
+            status_message = ('💡 Send "help" for manual, or "info" for '
+                              'information.')
             XmppPresence.send(self, jid, status_message)
 
 
     async def on_chatstate_gone(self, message):
-        if message['type'] in ('chat', 'normal'):
         jid = message['from'].bare
+        if jid in self.boundjid.bare:
+            return
+        if message['type'] in ('chat', 'normal'):
             # task.clean_tasks_xmpp(self, jid, ['status'])
             await task.start_tasks_xmpp(self, jid, ['status'])
 
 
     async def on_chatstate_inactive(self, message):
-        if message['type'] in ('chat', 'normal'):
         jid = message['from'].bare
+        if jid in self.boundjid.bare:
+            return
+        if message['type'] in ('chat', 'normal'):
             # task.clean_tasks_xmpp(self, jid, ['status'])
             await task.start_tasks_xmpp(self, jid, ['status'])
 
 
     async def on_chatstate_paused(self, message):
-        if message['type'] in ('chat', 'normal'):
         jid = message['from'].bare
+        if jid in self.boundjid.bare:
+            return
+        if message['type'] in ('chat', 'normal'):
             # task.clean_tasks_xmpp(self, jid, ['status'])
             await task.start_tasks_xmpp(self, jid, ['status'])
 

slixfeed/xmpp/component.py

@@ -13,7 +13,7 @@ TODO
 import asyncio
 import logging
 # import os
-from random import randrange
+# from random import randrange
 import slixmpp
 import slixfeed.task as task
 from time import sleep
@@ -31,11 +31,11 @@ import slixfeed.sqlite as sqlite
 from slixfeed.xmpp.bookmark import XmppBookmark
 from slixfeed.xmpp.connect import XmppConnect
 # NOTE MUC is possible for component
-from slixfeed.xmpp.muc import XmppGroupchat
+# from slixfeed.xmpp.muc import XmppGroupchat
 from slixfeed.xmpp.message import XmppMessage
 import slixfeed.xmpp.process as process
 import slixfeed.xmpp.profile as profile
-from slixfeed.xmpp.roster import XmppRoster
+# from slixfeed.xmpp.roster import XmppRoster
 # import slixfeed.xmpp.service as service
 from slixfeed.xmpp.presence import XmppPresence
 from slixfeed.xmpp.utility import get_chat_type
@@ -74,7 +74,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
         self.task_manager = {}
 
         # Handlers for ping
-        self.ping_task_instance = {}
+        self.task_ping_instance = {}
 
         # Handlers for connection events
         self.connection_attempts = 0
@@ -170,7 +170,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
         # await XmppGroupchat.autojoin(self)
         profile.set_identity(self, 'service')
         await profile.update(self)
-        task.ping_task(self)
+        task.task_ping(self)
         
         # Service.commands(self)
         # Service.reactions(self)
@@ -179,7 +179,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
         self.service_reactions()
 
 
-    async def on_session_resumed(self, event):
+    def on_session_resumed(self, event):
         self.send_presence()
         self['xep_0115'].update_caps()
         # await XmppGroupchat.autojoin(self)
@@ -220,13 +220,13 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
             await task.start_tasks_xmpp(self, jid, ['status', 'check'])
 
 
-    async def on_presence_subscribe(self, presence):
+    def on_presence_subscribe(self, presence):
         jid = presence['from'].bare
         # XmppPresence.request(self, jid)
         XmppPresence.subscription(self, jid, 'subscribe')
 
 
-    async def on_presence_subscribed(self, presence):
+    def on_presence_subscribed(self, presence):
         jid = presence['from'].bare
         message_subject = 'RSS News Bot'
         message_body = ('Greetings! I am {}, the news anchor.\n'
@@ -249,7 +249,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
         await task.start_tasks_xmpp(self, jid)
 
 
-    async def on_presence_unsubscribed(self, presence):
+    def on_presence_unsubscribed(self, presence):
         jid = presence['from'].bare
         message_body = 'You have been unsubscribed.'
         # status_message = '🖋️ Subscribe to receive updates'
@@ -260,7 +260,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
         #                   presence_type='unsubscribed')
 
 
-    async def on_presence_unavailable(self, presence):
+    def on_presence_unavailable(self, presence):
         jid = presence['from'].bare
         # await task.stop_tasks(self, jid)
         task.clean_tasks_xmpp(self, jid)
@@ -271,14 +271,14 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
     # Check whether JID is in bookmarks or roster
     # If roster, remove contact JID into file 
     # If bookmarks, remove groupchat JID into file 
-    async def on_presence_error(self, presence):
+    def on_presence_error(self, presence):
         print("on_presence_error")
         print(presence)
         jid = presence["from"].bare
         task.clean_tasks_xmpp(self, jid)
 
 
-    async def on_reactions(self, message):
+    def on_reactions(self, message):
         print(message['from'])
         print(message['reactions']['values'])
 
@@ -290,11 +290,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
             await task.start_tasks_xmpp(self, jid, ['status'])
 
 
-    async def on_chatstate_composing(self, message):
+    def on_chatstate_composing(self, message):
         if message['type'] in ('chat', 'normal'):
             jid = message['from'].bare
             # task.clean_tasks_xmpp(self, jid, ['status'])
-            status_message='💡 Press "help" for manual, or "info" for information.'
+            status_message = ('💡 Send "help" for manual, or "info" for '
+                              'information.')
             XmppPresence.send(self, jid, status_message)
 
 

slixfeed/xmpp/message.py

@@ -17,6 +17,7 @@ from slixfeed.xmpp.presence import XmppPresence
 from slixfeed.xmpp.upload import XmppUpload
 from slixfeed.xmpp.utility import get_chat_type
 import time
+import xml.sax.saxutils as saxutils
 
 """
 
@@ -24,28 +25,6 @@ NOTE
 
 See XEP-0367: Message Attaching
 
-FIXME
-
-ERROR:asyncio:Task exception was never retrieved
-future: <Task finished name='Task-3410' coro=<send_update() done, defined at /home/admin/.venv/lib/python3.11/site-packages/slixfeed/task.py:181> exception=ParseError('not well-formed (invalid token): line 1, column 198')>
-Traceback (most recent call last):
-  File "/home/jojo/.venv/lib/python3.11/site-packages/slixfeed/task.py", line 237, in send_update
-    XmppMessage.send_oob(self, jid, media, chat_type)
-  File "/home/jojo/.venv/lib/python3.11/site-packages/slixfeed/xmpp/message.py", line 56, in send_oob
-    message = self.make_message(mto=jid,
-              ^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/basexmpp.py", line 517, in make_message
-    message['html']['body'] = mhtml
-    ~~~~~~~~~~~~~~~^^^^^^^^
-  File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/xmlstream/stanzabase.py", line 792, in __setitem__
-    getattr(self, set_method)(value, **kwargs)
-  File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/plugins/xep_0071/stanza.py", line 38, in set_body
-    xhtml = ET.fromstring(content)
-            ^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.11/xml/etree/ElementTree.py", line 1338, in XML
-    parser.feed(text)
-xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 1, column 198
-
 """
 
 class XmppMessage:
@@ -71,8 +50,16 @@ class XmppMessage:
                           mnick=self.alias)
 
 
+    # NOTE We might want to add more characters
+    # def escape_to_xml(raw_string):
+    # escape_map = {
+    #     '"' : '&quot;',
+    #     "'" : '&apos;'
+    # }
+    # return saxutils.escape(raw_string, escape_map)
     def send_oob(self, jid, url, chat_type):
-        try:
+        url = saxutils.escape(url)
+        # try:
         html = (
             f'<body xmlns="http://www.w3.org/1999/xhtml">'
             f'<a href="{url}">{url}</a></body>')
@@ -83,9 +70,9 @@ class XmppMessage:
                                     mtype=chat_type)
         message['oob']['url'] = url
         message.send()
-        except:
+        # except:
-            logging.error('ERROR!')
+        #     logging.error('ERROR!')
-            logging.error(jid, url, chat_type, html)
+        #     logging.error(jid, url, chat_type, html)
 
 
     # FIXME Solve this function

slixfeed/xmpp/process.py

@@ -534,8 +534,7 @@ async def message(self, message):
                 if query:
                     if len(query) > 3:
                         db_file = config.get_pathname_to_database(jid_file)
-                        result = await sqlite.search_feeds(db_file, query)
+                        response = action.list_feeds_by_query(db_file, query)
-                        response = action.list_feeds_by_query(query, result)
                     else:
                         response = 'Enter at least 4 characters to search'
                 else:
@@ -546,7 +545,7 @@ async def message(self, message):
             case 'goodbye':
                 if message['type'] == 'groupchat':
                     await XmppGroupchat.leave(self, jid)
-                    await XmppBookmark.remove(self, muc_jid)
+                    await XmppBookmark.remove(self, jid)
                 else:
                     response = 'This command is valid in groupchat only.'
                 XmppMessage.send_reply(self, message, response)
@@ -631,7 +630,7 @@ async def message(self, message):
                 # num = message_text[5:]
                 # await task.send_update(self, jid, num)
 
-                await task.send_update(self, jid)
+                await task.xmpp_send_update(self, jid)
 
                 # task.clean_tasks_xmpp(self, jid, ['interval', 'status'])
                 # await task.start_tasks_xmpp(self, jid, ['status', 'interval'])
@@ -842,8 +841,11 @@ async def message(self, message):
                 try:
                     await sqlite.set_enabled_status(db_file, feed_id, 0)
                     await sqlite.mark_feed_as_read(db_file, feed_id)
-                    response = ('Updates are now disabled for news source {}.'
+                    name = sqlite.get_feed_title(db_file, feed_id)[0]
-                                .format(feed_id))
+                    addr = sqlite.get_feed_url(db_file, feed_id)[0]
+                    response = ('> {}\n'
+                                'Updates are now disabled for news source "{}"'
+                                .format(addr, name))
                 except:
                     response = 'No news source with index {}.'.format(feed_id)
                 XmppMessage.send_reply(self, message, response)
@@ -853,8 +855,11 @@ async def message(self, message):
                 db_file = config.get_pathname_to_database(jid_file)
                 try:
                     await sqlite.set_enabled_status(db_file, feed_id, 1)
-                    response = ('Updates are now enabled for news source {}.'
+                    name = sqlite.get_feed_title(db_file, feed_id)[0]
-                                .format(feed_id))
+                    addr = sqlite.get_feed_url(db_file, feed_id)[0]
+                    response = ('> {}\n'
+                                'Updates are now enabled for news source "{}"'
+                                .format(addr, name))
                 except:
                     response = 'No news source with index {}.'.format(ix)
                 XmppMessage.send_reply(self, message, response)

slixfeed/xmpp/roster.py

@@ -12,22 +12,6 @@ TODO
 
 class XmppRoster:
 
-    async def remove(self, jid):
-        """
-        Remove JID to roster.
-
-        Parameters
-        ----------
-        jid : str
-            Jabber ID.
-
-        Returns
-        -------
-        None.
-        """
-        self.update_roster(jid, subscription="remove")
-
-
     async def add(self, jid):
         """
         Add JID to roster.
@@ -45,5 +29,20 @@ class XmppRoster:
         """
         await self.get_roster()
         if jid not in self.client_roster.keys():
-            self.update_roster(jid, subscription="both")
+            self.update_roster(jid, subscription='both')
 
+
+    def remove(self, jid):
+        """
+        Remove JID from roster.
+
+        Parameters
+        ----------
+        jid : str
+            Jabber ID.
+
+        Returns
+        -------
+        None.
+        """
+        self.update_roster(jid, subscription='remove')