Improve update interval mechanism.

Add service discovery identity.
Add exception errors.

pyproject.toml

@@ -51,6 +51,7 @@ dependencies = [
     "pdfkit",
     "pysocks",
     "readability-lxml",
+    "xml2epub",
 ]
 
 [project.urls]
@@ -60,7 +61,7 @@ Issues = "https://gitgud.io/sjehuda/slixfeed/issues"
 
 
 [project.optional-dependencies]
-file-export = ["html2text", "pdfkit"]
+file-export = ["html2text", "pdfkit", "xml2epub"]
 proxy = ["pysocks"]
 readability = ["readability-lxml"]
 

slixfeed/__main__.py

@@ -100,7 +100,7 @@ import os
 #import slixfeed.irc
 #import slixfeed.matrix
 
-from slixfeed.config import get_value
+from slixfeed.config import get_default_config_directory, get_value
 
 import socks
 import socket
@@ -190,6 +190,10 @@ class JabberClient:
 
 def main():
 
+    config_dir = get_default_config_directory()
+    logging.info("Reading configuration from {}".format(config_dir))
+    print("Reading configuration from {}".format(config_dir))
+
     values = get_value(
         "accounts", "XMPP Proxy", ["socks5_host", "socks5_port"])
     if values[0] and values[1]:

slixfeed/action.py

@@ -1029,7 +1029,12 @@ def generate_document(data, url, ext, filename):
             "Check that package readability is installed.")
     match ext:
         case "epub":
-            generate_epub(content, filename)
+            error = generate_epub(content, filename)
+            if error:
+                logging.error(error)
+                # logging.error(
+                #     "Check that packages xml2epub is installed, "
+                #     "or try again.")
         case "html":
             generate_html(content, filename)
         case "md":
@@ -1042,14 +1047,14 @@ def generate_document(data, url, ext, filename):
                 error = (
                     "Package html2text was not found.")
         case "pdf":
-            try:
+            error = generate_pdf(content, filename)
-                generate_pdf(content, filename)
+            if error:
-            except:
+                logging.error(error)
-                logging.warning(
+                # logging.warning(
-                    "Check that packages pdfkit and wkhtmltopdf "
+                #     "Check that packages pdfkit and wkhtmltopdf "
-                    "are installed, or try again.")
+                #     "are installed, or try again.")
-                error = (
+                # error = (
-                    "Package pdfkit or wkhtmltopdf was not found.")
+                #     "Package pdfkit or wkhtmltopdf was not found.")
         case "txt":
             generate_txt(content, filename)
     if error:
@@ -1126,6 +1131,7 @@ def generate_epub(text, pathname):
     # chapter1 = xml2epub.create_chapter_from_url("https://dev.to/devteam/top-7-featured-dev-posts-from-the-past-week-h6h")
     # chapter2 = xml2epub.create_chapter_from_url("https://dev.to/ks1912/getting-started-with-docker-34g6")
     ## add chapters to your eBook
+    try:
         book.add_chapter(chapter0)
         # book.add_chapter(chapter1)
         # book.add_chapter(chapter2)
@@ -1134,6 +1140,9 @@ def generate_epub(text, pathname):
         book.create_epub(directory, epub_name=filename_tmp)
         pathname_tmp = os.path.join(directory, filename_tmp) + ".epub"
         os.rename(pathname_tmp, pathname)
+    except ValueError as error:
+        return error
+        
 
 
 def generate_html(text, filename):
@@ -1150,7 +1159,12 @@ def generate_markdown(text, filename):
 
 
 def generate_pdf(text, filename):
+    try:
         pdfkit.from_string(text, filename)
+    except IOError as error:
+        return error
+    except OSError as error:
+        return error
 
 
 def generate_txt(text, filename):

slixfeed/dt.py

@@ -9,7 +9,6 @@ from datetime import datetime
 from dateutil.parser import parse
 from email.utils import parsedate, parsedate_to_datetime
 
-
 def now():
     """
     ISO 8601 Timestamp.

slixfeed/sqlite.py

@@ -13,16 +13,10 @@ TODO
 """
 
 from asyncio import Lock
-from datetime import date
 import logging
-import slixfeed.config as config
 # from slixfeed.data import join_url
-from slixfeed.dt import (
-    current_time,
-    rfc2822_to_iso8601
-    )
 from sqlite3 import connect, Error, IntegrityError
-from slixfeed.url import join_url
+import time
 
 # from eliot import start_action, to_file
 # # with start_action(action_type="list_feeds()", db=db_file):
@@ -82,9 +76,9 @@ def create_tables(db_file):
               );
             """
             )
-        properties_table_sql = (
+        feeds_properties_table_sql = (
             """
-            CREATE TABLE IF NOT EXISTS properties (
+            CREATE TABLE IF NOT EXISTS feeds_properties (
                 id INTEGER NOT NULL,
                 feed_id INTEGER NOT NULL UNIQUE,
                 type TEXT,
@@ -98,9 +92,9 @@ def create_tables(db_file):
               );
             """
             )
-        status_table_sql = (
+        feeds_state_table_sql = (
             """
-            CREATE TABLE IF NOT EXISTS status (
+            CREATE TABLE IF NOT EXISTS feeds_state (
                 id INTEGER NOT NULL,
                 feed_id INTEGER NOT NULL UNIQUE,
                 enabled INTEGER NOT NULL DEFAULT 1,
@@ -169,6 +163,16 @@ def create_tables(db_file):
         #       );
         #     """
         #     )
+        status_table_sql = (
+            """
+            CREATE TABLE IF NOT EXISTS status (
+                id INTEGER NOT NULL,
+                key TEXT NOT NULL,
+                value INTEGER,
+                PRIMARY KEY ("id")
+              );
+            """
+            )
         settings_table_sql = (
             """
             CREATE TABLE IF NOT EXISTS settings (
@@ -191,14 +195,15 @@ def create_tables(db_file):
             )
         cur = conn.cursor()
         # cur = get_cursor(db_file)
-        cur.execute(feeds_table_sql)
-        cur.execute(status_table_sql)
-        cur.execute(properties_table_sql)
-        cur.execute(entries_table_sql)
         cur.execute(archive_table_sql)
+        cur.execute(entries_table_sql)
+        cur.execute(feeds_table_sql)
+        cur.execute(feeds_state_table_sql)
+        cur.execute(feeds_properties_table_sql)
+        cur.execute(filters_table_sql)
         # cur.execute(statistics_table_sql)
         cur.execute(settings_table_sql)
-        cur.execute(filters_table_sql)
+        cur.execute(status_table_sql)
 
 
 def get_cursor(db_file):
@@ -298,7 +303,7 @@ def insert_feed_status(cur, feed_id):
     sql = (
         """
         INSERT
-        INTO status(
+        INTO feeds_state(
             feed_id)
         VALUES(
             ?)
@@ -309,7 +314,7 @@ def insert_feed_status(cur, feed_id):
         cur.execute(sql, par)
     except IntegrityError as e:
         logging.warning(
-            "Skipping feed_id {} for table status".format(feed_id))
+            "Skipping feed_id {} for table feeds_state".format(feed_id))
         logging.error(e)
 
 
@@ -325,7 +330,7 @@ def insert_feed_properties(cur, feed_id):
     sql = (
         """
         INSERT
-        INTO properties(
+        INTO feeds_properties(
             feed_id)
         VALUES(
             ?)
@@ -336,7 +341,7 @@ def insert_feed_properties(cur, feed_id):
         cur.execute(sql, par)
     except IntegrityError as e:
         logging.warning(
-            "Skipping feed_id {} for table properties".format(feed_id))
+            "Skipping feed_id {} for table feeds_properties".format(feed_id))
         logging.error(e)
 
 
@@ -395,7 +400,7 @@ async def insert_feed(
             sql = (
                 """
                 INSERT
-                INTO status(
+                INTO feeds_state(
                     feed_id, enabled, updated, status_code, valid)
                 VALUES(
                     ?, ?, ?, ?, ?)
@@ -408,7 +413,7 @@ async def insert_feed(
             sql = (
                 """
                 INSERT
-                INTO properties(
+                INTO feeds_properties(
                     feed_id, entries, type, encoding, language)
                 VALUES(
                     ?, ?, ?, ?, ?)
@@ -636,7 +641,7 @@ async def get_number_of_feeds_active(db_file):
         sql = (
             """
             SELECT count(id)
-            FROM status
+            FROM feeds_state
             WHERE enabled = 1
             """
             )
@@ -1036,7 +1041,7 @@ async def set_enabled_status(db_file, ix, status):
             cur = conn.cursor()
             sql = (
                 """
-                UPDATE status
+                UPDATE feeds_state
                 SET enabled = :status
                 WHERE feed_id = :id
                 """
@@ -1155,13 +1160,13 @@ async def add_entries_and_update_timestamp(db_file, feed_id, new_entries):
                 cur.execute(sql, par)
             sql = (
                 """
-                UPDATE status
+                UPDATE feeds_state
-                SET renewed = :today
+                SET renewed = :renewed
                 WHERE feed_id = :feed_id
                 """
                 )
             par = {
-                "today": date.today(),
+                "renewed": time.time(),
                 "feed_id": feed_id
                 }
             cur.execute(sql, par)
@@ -1183,13 +1188,13 @@ async def set_date(db_file, feed_id):
             cur = conn.cursor()
             sql = (
                 """
-                UPDATE status
+                UPDATE feeds_state
-                SET renewed = :today
+                SET renewed = :renewed
                 WHERE feed_id = :feed_id
                 """
                 )
             par = {
-                "today": date.today(),
+                "renewed": time.time(),
                 "feed_id": feed_id
                 }
             # cur = conn.cursor()
@@ -1214,14 +1219,14 @@ async def update_feed_status(db_file, feed_id, status_code):
             cur = conn.cursor()
             sql = (
                 """
-                UPDATE status
+                UPDATE feeds_state
                 SET status_code = :status_code, scanned = :scanned
                 WHERE feed_id = :feed_id
                 """
                 )
             par = {
                 "status_code": status_code,
-                "scanned": date.today(),
+                "scanned": time.time(),
                 "feed_id": feed_id
                 }
             cur.execute(sql, par)
@@ -1245,7 +1250,7 @@ async def update_feed_validity(db_file, feed_id, valid):
             cur = conn.cursor()
             sql = (
                 """
-                UPDATE status
+                UPDATE feeds_state
                 SET valid = :valid
                 WHERE feed_id = :feed_id
                 """
@@ -1277,7 +1282,7 @@ async def update_feed_properties(db_file, feed_id, entries, updated):
             cur = conn.cursor()
             sql = (
                 """
-                UPDATE properties
+                UPDATE feeds_properties
                 SET entries = :entries
                 WHERE feed_id = :feed_id
                 """
@@ -1643,8 +1648,8 @@ async def check_entry_exist(
                 result = cur.execute(sql, par).fetchone()
                 if result: exist = True
             except:
-                print(current_time(), "ERROR DATE: source =", feed_id)
+                logging.error("source =", feed_id)
-                print(current_time(), "ERROR DATE: date =", date)
+                logging.error("date =", date)
         else:
             sql = (
                 """
@@ -1900,3 +1905,95 @@ async def get_filters_value(db_file, key):
                 "No specific value set for key {}.".format(key)
                 )
     return value
+
+
+async def set_last_update_time(db_file):
+    """
+    Set value of last_update.
+
+    Parameters
+    ----------
+    db_file : str
+        Path to database file.
+
+    Returns
+    -------
+    None.
+    """
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            INSERT
+            INTO status(
+                key, value)
+            VALUES(
+                :key, :value)
+            """
+            )
+        par = {
+            "key": "last_update",
+            "value": time.time()
+            }
+        cur.execute(sql, par)
+
+
+async def get_last_update_time(db_file):
+    """
+    Get value of last_update.
+
+    Parameters
+    ----------
+    db_file : str
+        Path to database file.
+
+    Returns
+    -------
+    val : str
+        Time.
+    """
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        try:
+            sql = (
+                """
+                SELECT value
+                FROM status
+                WHERE key = "last_update"
+                """
+                )
+            value = cur.execute(sql).fetchone()[0]
+            value = str(value)
+        except:
+            value = None
+            logging.debug(
+                "No specific value set for key last_update.")
+    return value
+
+
+async def update_last_update_time(db_file):
+    """
+    Update value of last_update.
+
+    Parameters
+    ----------
+    db_file : str
+        Path to database file.
+
+    Returns
+    -------
+    None.
+    """
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            """
+            UPDATE status
+            SET value = :value
+            WHERE key = "last_update"
+            """
+            )
+        par = {
+            "value": time.time()
+            }
+        cur.execute(sql, par)

slixfeed/task.py

@@ -42,8 +42,6 @@ NOTE
 import asyncio
 import logging
 import os
-import slixmpp
-
 import slixfeed.action as action
 from slixfeed.config import (
     get_pathname_to_database,
@@ -51,19 +49,23 @@ from slixfeed.config import (
     get_value)
 # from slixfeed.dt import current_time
 from slixfeed.sqlite import (
+    delete_archived_entry,
     get_feed_title,
     get_feeds_url,
-    get_number_of_items,
+    get_last_update_time,
     get_number_of_entries_unread,
+    get_number_of_items,
     get_settings_value,
     get_unread_entries,
     mark_as_read,
     mark_entry_as_read,
-    delete_archived_entry
+    set_last_update_time,
+    update_last_update_time
     )
 # from xmpp import Slixfeed
 import slixfeed.xmpp.client as xmpp
 import slixfeed.xmpp.utility as utility
+import time
 
 main_task = []
 jid_tasker = {}
@@ -101,6 +103,30 @@ async def start_tasks_xmpp(self, jid, tasks):
                 task_manager[jid]["status"] = asyncio.create_task(
                     send_status(self, jid))
             case "interval":
+                db_file = get_pathname_to_database(jid)
+                update_interval = (
+                    await get_settings_value(db_file, "interval") or
+                    get_value("settings", "Settings", "interval")
+                    )
+                update_interval = 60 * int(update_interval)
+                last_update_time = await get_last_update_time(db_file)
+                if last_update_time:
+                    last_update_time = float(last_update_time)
+                    diff = time.time() - last_update_time
+                    if diff < update_interval:
+                        next_update_time = update_interval - diff
+                        print("jid              :", jid, "\n"
+                              "time             :", time.time(), "\n"
+                              "last_update_time :", last_update_time, "\n"
+                              "difference       :", diff, "\n"
+                              "update interval  :", update_interval, "\n"
+                              "next_update_time :", next_update_time, "\n")
+                        await asyncio.sleep(next_update_time)
+                    # elif diff > val:
+                    #     next_update_time = val
+                    await update_last_update_time(db_file)
+                else:
+                    await set_last_update_time(db_file)
                 task_manager[jid]["interval"] = asyncio.create_task(
                     send_update(self, jid))
     # for task in task_manager[jid].values():
@@ -152,11 +178,8 @@ async def task_jid(self, jid):
     """
     db_file = get_pathname_to_database(jid)
     enabled = (
-        await get_settings_value(
+        await get_settings_value(db_file, "enabled") or
-            db_file, "enabled")
+        get_value("settings", "Settings", "enabled")
-        ) or (
-        get_value(
-            "settings", "Settings", "enabled")
         )
     if enabled:
         # NOTE Perhaps we want to utilize super with keyword
@@ -208,20 +231,14 @@ async def send_update(self, jid, num=None):
     logging.debug("Sending a news update to JID {}".format(jid))
     db_file = get_pathname_to_database(jid)
     enabled = (
-        await get_settings_value(
+        await get_settings_value(db_file, "enabled") or
-            db_file, "enabled")
+        get_value("settings", "Settings", "enabled")
-        ) or (
-        get_value(
-            "settings", "Settings", "enabled")
         )
     if enabled:
         if not num:
             num = (
-                await get_settings_value(
+                await get_settings_value(db_file, "quantum") or
-                    db_file, "quantum")
+                get_value("settings", "Settings", "quantum")
-                ) or (
-                get_value(
-                    "settings", "Settings", "quantum")
                 )
         else:
             num = int(num)
@@ -341,11 +358,8 @@ async def send_status(self, jid):
     status_text = "📜️ Slixfeed RSS News Bot"
     db_file = get_pathname_to_database(jid)
     enabled = (
-        await get_settings_value(
+        await get_settings_value(db_file, "enabled") or
-            db_file, "enabled")
+        get_value("settings", "Settings", "enabled")
-        ) or (
-        get_value(
-            "settings", "Settings", "enabled")
         )
     if not enabled:
         status_mode = "xa"
@@ -414,11 +428,8 @@ async def refresh_task(self, jid, callback, key, val=None):
     if not val:
         db_file = get_pathname_to_database(jid)
         val = (
-            await get_settings_value(
+            await get_settings_value(db_file, key) or
-                db_file, key)
+            get_value("settings", "Settings", key)
-            ) or (
-            get_value(
-                "settings", "Settings", key)
             )
     # if task_manager[jid][key]:
     if jid in task_manager:

slixfeed/xmpp/client.py

@@ -72,6 +72,7 @@ import slixfeed.xmpp.muc as muc
 import slixfeed.xmpp.process as process
 import slixfeed.xmpp.profile as profile
 import slixfeed.xmpp.roster as roster
+import slixfeed.xmpp.service as service
 import slixfeed.xmpp.state as state
 import slixfeed.xmpp.status as status
 import slixfeed.xmpp.utility as utility
@@ -94,8 +95,7 @@ loop = asyncio.get_event_loop()
 
 class Slixfeed(slixmpp.ClientXMPP):
     """
-    Slixmpp
+    Slixfeed:
-    -------
     News bot that sends updates from RSS feeds.
     """
     def __init__(self, jid, password, hostname=None, port=None, alias=None):
@@ -178,6 +178,7 @@ class Slixfeed(slixmpp.ClientXMPP):
         await process.event(self, event)
         await muc.autojoin(self)
         await profile.update(self)
+        service.identity(self, "client")
 
 
     async def on_session_resumed(self, event):

slixfeed/xmpp/component.py

@@ -65,6 +65,7 @@ import slixfeed.xmpp.muc as muc
 import slixfeed.xmpp.process as process
 import slixfeed.xmpp.profile as profile
 import slixfeed.xmpp.roster as roster
+import slixfeed.xmpp.service as service
 import slixfeed.xmpp.state as state
 import slixfeed.xmpp.status as status
 import slixfeed.xmpp.utility as utility
@@ -162,6 +163,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
         await process.event_component(self, event)
         # await muc.autojoin(self)
         await profile.update(self)
+        service.identity(self, "service")
 
 
     async def on_session_resumed(self, event):

slixfeed/xmpp/muc.py

@@ -11,6 +11,8 @@ TODO
 
 3) If groupchat error is received, send that error message to inviter.
 
+4) Save name of groupchat instead of jid as name
+
 """
 import logging
 import slixfeed.xmpp.bookmark as bookmark

slixfeed/xmpp/process.py

@@ -92,6 +92,7 @@ async def message(self, message):
         #         return
 
         # FIXME Code repetition. See below.
+        # TODO Check alias by nickname associated with conference
         if message["type"] == "groupchat":
             if (message['muc']['nick'] == self.alias):
                 return
@@ -475,7 +476,9 @@ async def message(self, message):
                     await task.start_tasks_xmpp(
                         self, jid, ["status"])
                 else:
-                    response = "Unsupported filetype."
+                    response = (
+                        "Unsupported filetype. "
+                        "Try: html, md, opml, or xbel")
                     send_reply_message(self, message, response)
             case _ if (message_lowercase.startswith("gemini:") or
                         message_lowercase.startswith("gopher:")):
@@ -537,23 +540,27 @@ async def message(self, message):
                                     data, url, ext, filename)
                                 if error:
                                     response = (
+                                        "> {}\n"
                                         "Failed to export {}.  Reason: {}"
-                                        ).format(ext.upper(), error)
+                                        ).format(url, ext.upper(), error)
                                 else:
                                     url = await upload.start(self, jid, filename)
                                     await send_oob_message(self, jid, url)
                             else:
                                 response = (
-                                    "Failed to fetch {}  Reason: {}"
+                                    "> {}\n"
+                                    "Failed to fetch URL.  Reason: {}"
                                     ).format(url, code)
                         await task.start_tasks_xmpp(
                             self, jid, ["status"])
                     else:
                         response = "Missing entry index number."
                 else:
-                    response = "Unsupported filetype."
+                    response = (
+                        "Unsupported filetype. "
+                        "Try: epub, html, md (markdown), pdf, or text (txt)")
                 if response:
-                    print(response)
+                    logging.warning("Error for URL {}: {}".format(url, error))
                     send_reply_message(self, message, response)
             # case _ if (message_lowercase.startswith("http")) and(
             #     message_lowercase.endswith(".opml")):
@@ -741,10 +748,15 @@ async def message(self, message):
             # TODO Will you add support for number of messages?
             case "next":
                 # num = message_text[5:]
-                await task.clean_tasks_xmpp(
+                # await task.send_update(self, jid, num)
-                    jid, ["interval", "status"])
+
-                await task.start_tasks_xmpp(
+                await task.send_update(self, jid)
-                    self, jid, ["interval", "status"])
+
+                # await task.clean_tasks_xmpp(
+                #     jid, ["interval", "status"])
+                # await task.start_tasks_xmpp(
+                #     self, jid, ["interval", "status"])
+
                 # await refresh_task(
                 #     self,
                 #     jid,

slixfeed/xmpp/service.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+def identity(self, category):
+    """
+    Identify for Service Duscovery
+
+    Parameters
+    ----------
+    category : str
+        "client" or "service".
+
+    Returns
+    -------
+    None.
+
+    """
+    self["xep_0030"].add_identity(
+        category=category,
+        itype="news",
+        name="slixfeed",
+        node=None,
+        jid=self.boundjid.full,
+    )

slixfeed/xmpp/state.py

@@ -27,8 +27,7 @@ async def request(self, jid):
         breakpoint()
         self.send_raw(str(presence_probe))
         presence_probe.send()
-    else:
+    elif not self.client_roster[jid]["to"]:
-        if not self.client_roster[jid]["to"]:
         self.send_presence_subscription(
             pto=jid,
             pfrom=self.boundjid.bare,