Split more functions into smaller functions

slixfeed/fetch.py

@@ -18,6 +18,9 @@ TODO
 
 4) Refactor view_entry and view_feed - Why "if" twice?
 
+5) Replace sqlite.remove_nonexistent_entries by sqlite.check_entry_exist
+   Same check, just reverse.
+
 """
 
 from aiohttp import ClientError, ClientSession, ClientTimeout
@@ -64,10 +67,7 @@ async def download_updates(db_file, url=None):
             # next(urls)
             continue
         await sqlite.update_source_status(
-            db_file,
+            db_file, res[1], source)
-            res[1],
-            source
-            )
         if res[0]:
             try:
                 feed = parse(res[0])
@@ -82,9 +82,7 @@ async def download_updates(db_file, url=None):
                 else:
                     valid = 1
                 await sqlite.update_source_validity(
-                    db_file,
+                    db_file, source, valid)
-                    source,
-                    valid)
             except (
                     IncompleteReadError,
                     IncompleteRead,
@@ -107,10 +105,7 @@ async def download_updates(db_file, url=None):
             # length = len(entries)
             # await remove_entry(db_file, source, length)
             await sqlite.remove_nonexistent_entries(
-                db_file,
+                db_file, feed, source)
-                feed,
-                source
-                )
             # new_entry = 0
             for entry in entries:
                 # TODO Pass date too for comparion check
@@ -144,13 +139,8 @@ async def download_updates(db_file, url=None):
                 else:
                     eid = link
                 exist = await sqlite.check_entry_exist(
-                    db_file,
+                    db_file, source, eid=eid,
-                    source,
+                    title=title, link=link, date=date)
-                    eid=eid,
-                    title=title,
-                    link=link,
-                    date=date
-                    )
                 if not exist:
                     # new_entry = new_entry + 1
                     # TODO Enhance summary
@@ -176,16 +166,10 @@ async def download_updates(db_file, url=None):
                             pathname
                             )
                     allow_list = await config.is_listed(
-                        db_file,
+                        db_file, "filter-allow", string)
-                        "filter-allow",
-                        string
-                        )
                     if not allow_list:
                         reject_list = await config.is_listed(
-                            db_file,
+                            db_file, "filter-deny", string)
-                            "filter-deny",
-                            string
-                            )
                         if reject_list:
                             # print(">>> REJECTED", title)
                             summary = (
@@ -196,13 +180,7 @@ async def download_updates(db_file, url=None):
                             # summary = ""
                             read_status = 1
                     entry = (
-                        title,
+                        title, link, eid, source, date, read_status)
-                        link,
-                        eid,
-                        source,
-                        date,
-                        read_status
-                        )
                     if isinstance(date, int):
                         print("PROBLEM: date is int")
                         print(date)
@@ -210,10 +188,7 @@ async def download_updates(db_file, url=None):
                     # print(source)
                     # print(date)
                     await sqlite.add_entry_and_set_date(
-                        db_file,
+                        db_file, source, entry)
-                        source,
-                        entry
-                        )
                 #     print(current_time(), entry, title)
                 # else:
                 #     print(current_time(), exist, title)
@@ -293,12 +268,7 @@ async def view_feed(url):
                 "Link  : {}\n"
                 "Count : {}\n"
                 "\n"
-                ).format(
+                ).format(title, date, link, counter)
-                    title,
-                    date,
-                    link,
-                    counter
-                    )
             if counter > 4:
                 break
         msg += (
@@ -377,11 +347,7 @@ async def view_entry(url, num):
             "\n"
             "{}\n"
             "\n"
-            ).format(
+            ).format(title, summary, link)
-                title,
-                summary,
-                link
-                )
     else:
         msg = (
             ">{}\n"
@@ -452,19 +418,11 @@ async def add_feed(db_file, url):
             if utility.is_feed(url, feed):
                 status = res[1]
                 msg = await sqlite.insert_feed(
-                    db_file,
+                    db_file, url, title, status)
-                    url,
-                    title,
-                    status
-                    )
                 await download_updates(db_file, [url])
             else:
                 msg = await probe_page(
-                    add_feed,
+                    add_feed, url, res[0], db_file=db_file)
-                    url,
-                    res[0],
-                    db_file=db_file
-                    )
         else:
             status = res[1]
             msg = (
@@ -684,11 +642,7 @@ async def feed_mode_request(url, tree):
                     "Link : {}\n"
                     "Items: {}\n"
                     "\n"
-                    ).format(
+                    ).format(feed_name, feed_addr, feed_amnt)
-                        feed_name,
-                        feed_addr,
-                        feed_amnt
-                        )
         if counter > 1:
             msg += (
                 "```\nThe above feeds were extracted from\n{}"
@@ -805,11 +759,7 @@ async def feed_mode_scan(url, tree):
                     "Link  : {}\n"
                     "Count : {}\n"
                     "\n"
-                    ).format(
+                    ).format(feed_name, feed_addr, feed_amnt)
-                        feed_name,
-                        feed_addr,
-                        feed_amnt
-                        )
         if counter > 1:
             msg += (
                 "```\nThe above feeds were extracted from\n{}"

slixfeed/format.py

@@ -7,4 +7,13 @@ TODO
 
 Move code  from sqlite.get_entry_unread
 
+if num > 1:
+    news_list += (
+        "\n{}\n{}\n{}\n"
+        ).format(str(title), str(link), str(feed_title))
+else:
+    news_list = (
+        "{}\n{}\n{}"
+        ).format(str(title), str(link), str(feed_title))
+
 """

slixfeed/sqlite.py

@@ -26,11 +26,7 @@ from slixfeed.datetime import (
     rfc2822_to_iso8601
     )
 from sqlite3 import connect, Error
-from slixfeed.url import (
+from slixfeed.url import join_url
-    join_url,
-    remove_tracking_parameters,
-    replace_hostname
-    )
 
 # from eliot import start_action, to_file
 # # with start_action(action_type="list_feeds()", db=db_file):
@@ -397,7 +393,7 @@ async def get_number_of_entries_unread(db_file):
 
 
 # TODO Read from entries and archives
-async def get_entry_unread(db_file, num=None):
+async def get_unread_entries(db_file, num):
     """
     Extract information from unread entries.
 
@@ -413,10 +409,6 @@ async def get_entry_unread(db_file, num=None):
     entry : str
         News item message.
     """
-    if not num:
-        num = await get_settings_value(db_file, "quantum")
-    else:
-        num = int(num)
     with create_connection(db_file) as conn:
         cur = conn.cursor()
         # sql = (
@@ -450,78 +442,13 @@ async def get_entry_unread(db_file, num=None):
             )
         results = cur.execute(sql, (num,))
         results = results.fetchall()
-        
+        # print("### sqlite.get_unread_entries ###")
-        # TODO Add filtering
+        # print(results)
-        # TODO Do this when entry is added to list and mark it as read
-        # DONE!
-        # results = []
-        # if get_settings_value(db_file, "filter-deny"):
-        #     while len(results) < num:
-        #         result = cur.execute(sql).fetchone()
-        #         blacklist = await get_settings_value(db_file, "filter-deny").split(",")
-        #         for i in blacklist:
-        #             if i in result[1]:
-        #                 continue
-        #                 print("rejected:", result[1])
-        #         print("accepted:", result[1])
-        #         results.extend([result])
-
-        # news_list = "You've got {} news items:\n".format(num)
-        news_list = ""
-        # NOTE Why doesn't this work without list?
-        #      i.e. for result in results
-        # for result in results.fetchall():
-        for result in results:
-            ix = result[0]
-            title = result[1]
-            # # TODO Retrieve summary from feed
-            # # See fetch.view_entry
-            # summary = result[2]
-            # # Remove HTML tags
-            # try:
-            #     summary = BeautifulSoup(summary, "lxml").text
-            # except:
-            #     print(result[2])
         # breakpoint()
-            # # TODO Limit text length
+        return results
-            # summary = summary.replace("\n\n\n", "\n\n")
-            # length = await get_settings_value(db_file, "length")
-            # summary = summary[:length] + " […]"
-            # summary = summary.strip().split('\n')
-            # summary = ["> " + line for line in summary]
-            # summary = "\n".join(summary)
-            link = result[2]
-            link = remove_tracking_parameters(link)
-            link = (replace_hostname(link, "link")) or link
-            sql = (
-                "SELECT name "
-                "FROM feeds "
-                "WHERE address = :source "
-                )
-            source = result[3]
-            feed = cur.execute(sql, (source,))
-            feed = feed.fetchone()[0]
-            if num > 1:
-                news_list += (
-                    "\n{}\n{}\n{}\n"
-                    ).format(str(title), str(link), str(feed))
-            else:
-                news_list = (
-                    "{}\n{}\n{}"
-                    ).format(str(title), str(link), str(feed))
-            # TODO While `async with DBLOCK` does work well from
-            # outside of functions, it would be better practice
-            # to place it within the functions.
-            async with DBLOCK:
-                # NOTE: We can use DBLOCK once for both
-                # functions, because, due to exclusive
-                # ID, only one can ever occur.
-                await mark_entry_as_read(cur, ix)
-                await delete_entry(cur, ix)
-        return news_list
 
 
-async def mark_entry_as_read(cur, ix):
+def mark_entry_as_read(cur, ix):
     """
     Set read status of entry as read.
 
@@ -562,6 +489,32 @@ async def mark_source_as_read(db_file, source):
             cur.execute(sql, (source,))
 
 
+def get_feed_title(db_file, source):
+    with create_connection(db_file) as conn:
+        cur = conn.cursor()
+        sql = (
+            "SELECT name "
+            "FROM feeds "
+            "WHERE address = :source "
+            )
+        feed_title = cur.execute(sql, (source,))
+        feed_title = feed_title.fetchone()[0]
+        return feed_title
+
+
+async def mark_as_read(db_file, ix):
+    async with DBLOCK:
+        with create_connection(db_file) as conn:
+            cur = conn.cursor()
+            # TODO While `async with DBLOCK` does work well from
+            # outside of functions, it would be better practice
+            # to place it within the functions.
+            # NOTE: We can use DBLOCK once for both
+            # functions, because, due to exclusive
+            # ID, only one can ever occur.
+            mark_entry_as_read(cur, ix)
+            delete_archived_entry(cur, ix)
+
 async def mark_all_as_read(db_file):
     """
     Set read status of all entries as read.
@@ -585,7 +538,7 @@ async def mark_all_as_read(db_file):
             cur.execute(sql)
 
 
-async def delete_entry(cur, ix):
+def delete_archived_entry(cur, ix):
     """
     Delete entry from table archive.
 
@@ -614,8 +567,8 @@ async def statistics(db_file):
 
     Returns
     -------
-    msg : str
+    values : list
-        Statistics as message.
+        List of values.
     """
     values = []
     values.extend([await get_number_of_entries_unread(db_file)])
@@ -883,13 +836,15 @@ async def add_entry(cur, entry):
     try:
         cur.execute(sql, entry)
     except:
-        print("Unknown error for sqlite.add_entry")
+        print("")
+        # print("Unknown error for sqlite.add_entry")
+        # print(entry)
         # print(current_time(), "COROUTINE OBJECT NOW")
         # for i in entry:
         #     print(type(i))
         #     print(i)
         # print(type(entry))
-        print(entry)
+        # print(entry)
         # print(current_time(), "COROUTINE OBJECT NOW")
         # breakpoint()
 

slixfeed/task.py

@@ -51,13 +51,18 @@ from slixfeed.config import (
 from slixfeed.datetime import current_time
 from slixfeed.fetch import download_updates
 from slixfeed.sqlite import (
-    get_entry_unread,
+    get_unread_entries,
+    get_feed_title,
     get_settings_value,
     get_number_of_items,
-    get_number_of_entries_unread
+    get_number_of_entries_unread,
+    mark_as_read,
+    mark_entry_as_read,
+    delete_archived_entry
     )
 # from xmpp import Slixfeed
 import slixfeed.xmpp.client as xmpp
+from slixfeed.xmpp.compose import list_unread_entries
 import slixfeed.xmpp.utility as utility
 
 main_task = []
@@ -199,8 +204,25 @@ async def send_update(self, jid, num=None):
     db_file = get_pathname_to_database(jid)
     enabled = await get_settings_value(db_file, "enabled")
     if enabled:
-        new = await get_entry_unread(db_file, num)
+        if not num:
+            num = await get_settings_value(db_file, "quantum")
+        else:
+            num = int(num)
+        news_digest = []
+        results = await get_unread_entries(db_file, num)
+        for result in results:
+            title = get_feed_title(db_file, result[3])
+            news_item = list_unread_entries(result, title)
+            news_digest.extend([news_item])
+            # print(db_file)
+            # print(result[0])
+            # breakpoint()
+            await mark_as_read(db_file, result[0])
+        new = " ".join(news_digest)
+        # breakpoint()
         if new:
+            # print("if new")
+            # breakpoint()
             # TODO Add while loop to assure delivery.
             # print(await current_time(), ">>> ACT send_message",jid)
             chat_type = await utility.jid_type(self, jid)

slixfeed/xmpp/compose.py

@@ -9,6 +9,53 @@ import slixfeed.xmpp.bookmark as bookmark
 from slixfeed.url import remove_tracking_parameters, replace_hostname
 
 
+def list_unread_entries(result, feed_title):
+    # TODO Add filtering
+    # TODO Do this when entry is added to list and mark it as read
+    # DONE!
+    # results = []
+    # if get_settings_value(db_file, "filter-deny"):
+    #     while len(results) < num:
+    #         result = cur.execute(sql).fetchone()
+    #         blacklist = await get_settings_value(db_file, "filter-deny").split(",")
+    #         for i in blacklist:
+    #             if i in result[1]:
+    #                 continue
+    #                 print("rejected:", result[1])
+    #         print("accepted:", result[1])
+    #         results.extend([result])
+
+    # news_list = "You've got {} news items:\n".format(num)
+    # NOTE Why doesn't this work without list?
+    #      i.e. for result in results
+    # for result in results.fetchall():
+    ix = result[0]
+    title = result[1]
+    # # TODO Retrieve summary from feed
+    # # See fetch.view_entry
+    # summary = result[2]
+    # # Remove HTML tags
+    # try:
+    #     summary = BeautifulSoup(summary, "lxml").text
+    # except:
+    #     print(result[2])
+    #     breakpoint()
+    # # TODO Limit text length
+    # summary = summary.replace("\n\n\n", "\n\n")
+    # length = await get_settings_value(db_file, "length")
+    # summary = summary[:length] + " […]"
+    # summary = summary.strip().split('\n')
+    # summary = ["> " + line for line in summary]
+    # summary = "\n".join(summary)
+    link = result[2]
+    link = remove_tracking_parameters(link)
+    link = (replace_hostname(link, "link")) or link
+    news_item = (
+        "\n{}\n{}\n{}\n"
+        ).format(str(title), str(link), str(feed_title))
+    return news_item
+
+
 def list_search_results(query, results):
     results_list = (
         "Search results for '{}':\n\n```"
@@ -76,7 +123,7 @@ def list_statistics(values):
     return msg
 
 
-async def list_last_entries(results, num):
+def list_last_entries(results, num):
     titles_list = "Recent {} titles:\n\n```".format(num)
     counter = 0
     for result in results:
@@ -91,7 +138,7 @@ async def list_last_entries(results, num):
         return "There are no news at the moment."
 
 
-async def list_feeds(results):
+def list_feeds(results):
     feeds_list = "\nList of subscriptions:\n\n```\n"
     counter = 0
     for result in results:

slixfeed/xmpp/process.py

@@ -144,7 +144,7 @@ async def message(self, message):
     
         print(current_time(), "ACCOUNT: " + str(message["from"]))
         print(current_time(), "COMMAND:", message_text)
-    
+        response = 0
         match message_lowercase:
             # case "breakpoint":
             #     if jid == get_value("accounts", "XMPP", "operator"):
@@ -712,7 +712,7 @@ async def message(self, message):
         # commands are sent one close to the next
         # if response: message.reply(response).send()
 
-        response = "EMPTY MESSAGE - ACTION ONLY"
+        if not response: response = "EMPTY MESSAGE - ACTION ONLY"
         log_dir = get_default_dbdir()
         if not os.path.isdir(log_dir):
             os.mkdir(log_dir)