From 4406e61fbec348cfc01108ee6a2dd69ab3ba6b69 Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Fri, 26 Jan 2024 11:34:07 +0000 Subject: [PATCH] Improve update interval mechanism. Add service discovery identity. Add exception errors. --- pyproject.toml | 3 +- slixfeed/__main__.py | 6 +- slixfeed/action.py | 50 +++++++---- slixfeed/dt.py | 1 - slixfeed/sqlite.py | 169 +++++++++++++++++++++++++++++-------- slixfeed/task.py | 69 ++++++++------- slixfeed/xmpp/client.py | 5 +- slixfeed/xmpp/component.py | 2 + slixfeed/xmpp/muc.py | 2 + slixfeed/xmpp/process.py | 30 +++++-- slixfeed/xmpp/service.py | 24 ++++++ slixfeed/xmpp/state.py | 55 ++++++------ 12 files changed, 291 insertions(+), 125 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 64bb25b..d9040ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ dependencies = [ "pdfkit", "pysocks", "readability-lxml", + "xml2epub", ] [project.urls] @@ -60,7 +61,7 @@ Issues = "https://gitgud.io/sjehuda/slixfeed/issues" [project.optional-dependencies] -file-export = ["html2text", "pdfkit"] +file-export = ["html2text", "pdfkit", "xml2epub"] proxy = ["pysocks"] readability = ["readability-lxml"] diff --git a/slixfeed/__main__.py b/slixfeed/__main__.py index 1de4aa4..d5ac187 100644 --- a/slixfeed/__main__.py +++ b/slixfeed/__main__.py @@ -100,7 +100,7 @@ import os #import slixfeed.irc #import slixfeed.matrix -from slixfeed.config import get_value +from slixfeed.config import get_default_config_directory, get_value import socks import socket @@ -190,6 +190,10 @@ class JabberClient: def main(): + config_dir = get_default_config_directory() + logging.info("Reading configuration from {}".format(config_dir)) + print("Reading configuration from {}".format(config_dir)) + values = get_value( "accounts", "XMPP Proxy", ["socks5_host", "socks5_port"]) if values[0] and values[1]: diff --git a/slixfeed/action.py b/slixfeed/action.py index 2060ff1..6ef4a7a 100644 --- a/slixfeed/action.py +++ b/slixfeed/action.py @@ -1029,7 +1029,12 @@ def generate_document(data, url, ext, filename): "Check that package readability is installed.") match ext: case "epub": - generate_epub(content, filename) + error = generate_epub(content, filename) + if error: + logging.error(error) + # logging.error( + # "Check that packages xml2epub is installed, " + # "or try again.") case "html": generate_html(content, filename) case "md": @@ -1042,14 +1047,14 @@ def generate_document(data, url, ext, filename): error = ( "Package html2text was not found.") case "pdf": - try: - generate_pdf(content, filename) - except: - logging.warning( - "Check that packages pdfkit and wkhtmltopdf " - "are installed, or try again.") - error = ( - "Package pdfkit or wkhtmltopdf was not found.") + error = generate_pdf(content, filename) + if error: + logging.error(error) + # logging.warning( + # "Check that packages pdfkit and wkhtmltopdf " + # "are installed, or try again.") + # error = ( + # "Package pdfkit or wkhtmltopdf was not found.") case "txt": generate_txt(content, filename) if error: @@ -1126,14 +1131,18 @@ def generate_epub(text, pathname): # chapter1 = xml2epub.create_chapter_from_url("https://dev.to/devteam/top-7-featured-dev-posts-from-the-past-week-h6h") # chapter2 = xml2epub.create_chapter_from_url("https://dev.to/ks1912/getting-started-with-docker-34g6") ## add chapters to your eBook - book.add_chapter(chapter0) - # book.add_chapter(chapter1) - # book.add_chapter(chapter2) - ## generate epub file - filename_tmp = "slixfeedepub" - book.create_epub(directory, epub_name=filename_tmp) - pathname_tmp = os.path.join(directory, filename_tmp) + ".epub" - os.rename(pathname_tmp, pathname) + try: + book.add_chapter(chapter0) + # book.add_chapter(chapter1) + # book.add_chapter(chapter2) + ## generate epub file + filename_tmp = "slixfeedepub" + book.create_epub(directory, epub_name=filename_tmp) + pathname_tmp = os.path.join(directory, filename_tmp) + ".epub" + os.rename(pathname_tmp, pathname) + except ValueError as error: + return error + def generate_html(text, filename): @@ -1150,7 +1159,12 @@ def generate_markdown(text, filename): def generate_pdf(text, filename): - pdfkit.from_string(text, filename) + try: + pdfkit.from_string(text, filename) + except IOError as error: + return error + except OSError as error: + return error def generate_txt(text, filename): diff --git a/slixfeed/dt.py b/slixfeed/dt.py index f0332c6..d9f388d 100644 --- a/slixfeed/dt.py +++ b/slixfeed/dt.py @@ -9,7 +9,6 @@ from datetime import datetime from dateutil.parser import parse from email.utils import parsedate, parsedate_to_datetime - def now(): """ ISO 8601 Timestamp. diff --git a/slixfeed/sqlite.py b/slixfeed/sqlite.py index 877620a..44ab1c0 100644 --- a/slixfeed/sqlite.py +++ b/slixfeed/sqlite.py @@ -13,16 +13,10 @@ TODO """ from asyncio import Lock -from datetime import date import logging -import slixfeed.config as config # from slixfeed.data import join_url -from slixfeed.dt import ( - current_time, - rfc2822_to_iso8601 - ) from sqlite3 import connect, Error, IntegrityError -from slixfeed.url import join_url +import time # from eliot import start_action, to_file # # with start_action(action_type="list_feeds()", db=db_file): @@ -82,9 +76,9 @@ def create_tables(db_file): ); """ ) - properties_table_sql = ( + feeds_properties_table_sql = ( """ - CREATE TABLE IF NOT EXISTS properties ( + CREATE TABLE IF NOT EXISTS feeds_properties ( id INTEGER NOT NULL, feed_id INTEGER NOT NULL UNIQUE, type TEXT, @@ -98,9 +92,9 @@ def create_tables(db_file): ); """ ) - status_table_sql = ( + feeds_state_table_sql = ( """ - CREATE TABLE IF NOT EXISTS status ( + CREATE TABLE IF NOT EXISTS feeds_state ( id INTEGER NOT NULL, feed_id INTEGER NOT NULL UNIQUE, enabled INTEGER NOT NULL DEFAULT 1, @@ -169,6 +163,16 @@ def create_tables(db_file): # ); # """ # ) + status_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS status ( + id INTEGER NOT NULL, + key TEXT NOT NULL, + value INTEGER, + PRIMARY KEY ("id") + ); + """ + ) settings_table_sql = ( """ CREATE TABLE IF NOT EXISTS settings ( @@ -191,14 +195,15 @@ def create_tables(db_file): ) cur = conn.cursor() # cur = get_cursor(db_file) - cur.execute(feeds_table_sql) - cur.execute(status_table_sql) - cur.execute(properties_table_sql) - cur.execute(entries_table_sql) cur.execute(archive_table_sql) + cur.execute(entries_table_sql) + cur.execute(feeds_table_sql) + cur.execute(feeds_state_table_sql) + cur.execute(feeds_properties_table_sql) + cur.execute(filters_table_sql) # cur.execute(statistics_table_sql) cur.execute(settings_table_sql) - cur.execute(filters_table_sql) + cur.execute(status_table_sql) def get_cursor(db_file): @@ -298,7 +303,7 @@ def insert_feed_status(cur, feed_id): sql = ( """ INSERT - INTO status( + INTO feeds_state( feed_id) VALUES( ?) @@ -309,7 +314,7 @@ def insert_feed_status(cur, feed_id): cur.execute(sql, par) except IntegrityError as e: logging.warning( - "Skipping feed_id {} for table status".format(feed_id)) + "Skipping feed_id {} for table feeds_state".format(feed_id)) logging.error(e) @@ -325,7 +330,7 @@ def insert_feed_properties(cur, feed_id): sql = ( """ INSERT - INTO properties( + INTO feeds_properties( feed_id) VALUES( ?) @@ -336,7 +341,7 @@ def insert_feed_properties(cur, feed_id): cur.execute(sql, par) except IntegrityError as e: logging.warning( - "Skipping feed_id {} for table properties".format(feed_id)) + "Skipping feed_id {} for table feeds_properties".format(feed_id)) logging.error(e) @@ -395,7 +400,7 @@ async def insert_feed( sql = ( """ INSERT - INTO status( + INTO feeds_state( feed_id, enabled, updated, status_code, valid) VALUES( ?, ?, ?, ?, ?) @@ -408,7 +413,7 @@ async def insert_feed( sql = ( """ INSERT - INTO properties( + INTO feeds_properties( feed_id, entries, type, encoding, language) VALUES( ?, ?, ?, ?, ?) @@ -636,7 +641,7 @@ async def get_number_of_feeds_active(db_file): sql = ( """ SELECT count(id) - FROM status + FROM feeds_state WHERE enabled = 1 """ ) @@ -1036,7 +1041,7 @@ async def set_enabled_status(db_file, ix, status): cur = conn.cursor() sql = ( """ - UPDATE status + UPDATE feeds_state SET enabled = :status WHERE feed_id = :id """ @@ -1155,13 +1160,13 @@ async def add_entries_and_update_timestamp(db_file, feed_id, new_entries): cur.execute(sql, par) sql = ( """ - UPDATE status - SET renewed = :today + UPDATE feeds_state + SET renewed = :renewed WHERE feed_id = :feed_id """ ) par = { - "today": date.today(), + "renewed": time.time(), "feed_id": feed_id } cur.execute(sql, par) @@ -1183,13 +1188,13 @@ async def set_date(db_file, feed_id): cur = conn.cursor() sql = ( """ - UPDATE status - SET renewed = :today + UPDATE feeds_state + SET renewed = :renewed WHERE feed_id = :feed_id """ ) par = { - "today": date.today(), + "renewed": time.time(), "feed_id": feed_id } # cur = conn.cursor() @@ -1214,14 +1219,14 @@ async def update_feed_status(db_file, feed_id, status_code): cur = conn.cursor() sql = ( """ - UPDATE status + UPDATE feeds_state SET status_code = :status_code, scanned = :scanned WHERE feed_id = :feed_id """ ) par = { "status_code": status_code, - "scanned": date.today(), + "scanned": time.time(), "feed_id": feed_id } cur.execute(sql, par) @@ -1245,7 +1250,7 @@ async def update_feed_validity(db_file, feed_id, valid): cur = conn.cursor() sql = ( """ - UPDATE status + UPDATE feeds_state SET valid = :valid WHERE feed_id = :feed_id """ @@ -1277,7 +1282,7 @@ async def update_feed_properties(db_file, feed_id, entries, updated): cur = conn.cursor() sql = ( """ - UPDATE properties + UPDATE feeds_properties SET entries = :entries WHERE feed_id = :feed_id """ @@ -1643,8 +1648,8 @@ async def check_entry_exist( result = cur.execute(sql, par).fetchone() if result: exist = True except: - print(current_time(), "ERROR DATE: source =", feed_id) - print(current_time(), "ERROR DATE: date =", date) + logging.error("source =", feed_id) + logging.error("date =", date) else: sql = ( """ @@ -1900,3 +1905,95 @@ async def get_filters_value(db_file, key): "No specific value set for key {}.".format(key) ) return value + + +async def set_last_update_time(db_file): + """ + Set value of last_update. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + None. + """ + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + INSERT + INTO status( + key, value) + VALUES( + :key, :value) + """ + ) + par = { + "key": "last_update", + "value": time.time() + } + cur.execute(sql, par) + + +async def get_last_update_time(db_file): + """ + Get value of last_update. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + val : str + Time. + """ + with create_connection(db_file) as conn: + cur = conn.cursor() + try: + sql = ( + """ + SELECT value + FROM status + WHERE key = "last_update" + """ + ) + value = cur.execute(sql).fetchone()[0] + value = str(value) + except: + value = None + logging.debug( + "No specific value set for key last_update.") + return value + + +async def update_last_update_time(db_file): + """ + Update value of last_update. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + None. + """ + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + UPDATE status + SET value = :value + WHERE key = "last_update" + """ + ) + par = { + "value": time.time() + } + cur.execute(sql, par) diff --git a/slixfeed/task.py b/slixfeed/task.py index db91a59..35dfaa9 100644 --- a/slixfeed/task.py +++ b/slixfeed/task.py @@ -42,8 +42,6 @@ NOTE import asyncio import logging import os -import slixmpp - import slixfeed.action as action from slixfeed.config import ( get_pathname_to_database, @@ -51,19 +49,23 @@ from slixfeed.config import ( get_value) # from slixfeed.dt import current_time from slixfeed.sqlite import ( + delete_archived_entry, get_feed_title, get_feeds_url, - get_number_of_items, + get_last_update_time, get_number_of_entries_unread, + get_number_of_items, get_settings_value, get_unread_entries, mark_as_read, mark_entry_as_read, - delete_archived_entry + set_last_update_time, + update_last_update_time ) # from xmpp import Slixfeed import slixfeed.xmpp.client as xmpp import slixfeed.xmpp.utility as utility +import time main_task = [] jid_tasker = {} @@ -101,6 +103,30 @@ async def start_tasks_xmpp(self, jid, tasks): task_manager[jid]["status"] = asyncio.create_task( send_status(self, jid)) case "interval": + db_file = get_pathname_to_database(jid) + update_interval = ( + await get_settings_value(db_file, "interval") or + get_value("settings", "Settings", "interval") + ) + update_interval = 60 * int(update_interval) + last_update_time = await get_last_update_time(db_file) + if last_update_time: + last_update_time = float(last_update_time) + diff = time.time() - last_update_time + if diff < update_interval: + next_update_time = update_interval - diff + print("jid :", jid, "\n" + "time :", time.time(), "\n" + "last_update_time :", last_update_time, "\n" + "difference :", diff, "\n" + "update interval :", update_interval, "\n" + "next_update_time :", next_update_time, "\n") + await asyncio.sleep(next_update_time) + # elif diff > val: + # next_update_time = val + await update_last_update_time(db_file) + else: + await set_last_update_time(db_file) task_manager[jid]["interval"] = asyncio.create_task( send_update(self, jid)) # for task in task_manager[jid].values(): @@ -152,11 +178,8 @@ async def task_jid(self, jid): """ db_file = get_pathname_to_database(jid) enabled = ( - await get_settings_value( - db_file, "enabled") - ) or ( - get_value( - "settings", "Settings", "enabled") + await get_settings_value(db_file, "enabled") or + get_value("settings", "Settings", "enabled") ) if enabled: # NOTE Perhaps we want to utilize super with keyword @@ -208,20 +231,14 @@ async def send_update(self, jid, num=None): logging.debug("Sending a news update to JID {}".format(jid)) db_file = get_pathname_to_database(jid) enabled = ( - await get_settings_value( - db_file, "enabled") - ) or ( - get_value( - "settings", "Settings", "enabled") + await get_settings_value(db_file, "enabled") or + get_value("settings", "Settings", "enabled") ) if enabled: if not num: num = ( - await get_settings_value( - db_file, "quantum") - ) or ( - get_value( - "settings", "Settings", "quantum") + await get_settings_value(db_file, "quantum") or + get_value("settings", "Settings", "quantum") ) else: num = int(num) @@ -341,11 +358,8 @@ async def send_status(self, jid): status_text = "📜️ Slixfeed RSS News Bot" db_file = get_pathname_to_database(jid) enabled = ( - await get_settings_value( - db_file, "enabled") - ) or ( - get_value( - "settings", "Settings", "enabled") + await get_settings_value(db_file, "enabled") or + get_value("settings", "Settings", "enabled") ) if not enabled: status_mode = "xa" @@ -414,11 +428,8 @@ async def refresh_task(self, jid, callback, key, val=None): if not val: db_file = get_pathname_to_database(jid) val = ( - await get_settings_value( - db_file, key) - ) or ( - get_value( - "settings", "Settings", key) + await get_settings_value(db_file, key) or + get_value("settings", "Settings", key) ) # if task_manager[jid][key]: if jid in task_manager: diff --git a/slixfeed/xmpp/client.py b/slixfeed/xmpp/client.py index bb85aca..b7d9532 100644 --- a/slixfeed/xmpp/client.py +++ b/slixfeed/xmpp/client.py @@ -72,6 +72,7 @@ import slixfeed.xmpp.muc as muc import slixfeed.xmpp.process as process import slixfeed.xmpp.profile as profile import slixfeed.xmpp.roster as roster +import slixfeed.xmpp.service as service import slixfeed.xmpp.state as state import slixfeed.xmpp.status as status import slixfeed.xmpp.utility as utility @@ -94,8 +95,7 @@ loop = asyncio.get_event_loop() class Slixfeed(slixmpp.ClientXMPP): """ - Slixmpp - ------- + Slixfeed: News bot that sends updates from RSS feeds. """ def __init__(self, jid, password, hostname=None, port=None, alias=None): @@ -178,6 +178,7 @@ class Slixfeed(slixmpp.ClientXMPP): await process.event(self, event) await muc.autojoin(self) await profile.update(self) + service.identity(self, "client") async def on_session_resumed(self, event): diff --git a/slixfeed/xmpp/component.py b/slixfeed/xmpp/component.py index 94cffe3..b277760 100644 --- a/slixfeed/xmpp/component.py +++ b/slixfeed/xmpp/component.py @@ -65,6 +65,7 @@ import slixfeed.xmpp.muc as muc import slixfeed.xmpp.process as process import slixfeed.xmpp.profile as profile import slixfeed.xmpp.roster as roster +import slixfeed.xmpp.service as service import slixfeed.xmpp.state as state import slixfeed.xmpp.status as status import slixfeed.xmpp.utility as utility @@ -162,6 +163,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): await process.event_component(self, event) # await muc.autojoin(self) await profile.update(self) + service.identity(self, "service") async def on_session_resumed(self, event): diff --git a/slixfeed/xmpp/muc.py b/slixfeed/xmpp/muc.py index 5c911a5..c29b391 100644 --- a/slixfeed/xmpp/muc.py +++ b/slixfeed/xmpp/muc.py @@ -11,6 +11,8 @@ TODO 3) If groupchat error is received, send that error message to inviter. +4) Save name of groupchat instead of jid as name + """ import logging import slixfeed.xmpp.bookmark as bookmark diff --git a/slixfeed/xmpp/process.py b/slixfeed/xmpp/process.py index cdf39e2..c58e7b8 100644 --- a/slixfeed/xmpp/process.py +++ b/slixfeed/xmpp/process.py @@ -92,6 +92,7 @@ async def message(self, message): # return # FIXME Code repetition. See below. + # TODO Check alias by nickname associated with conference if message["type"] == "groupchat": if (message['muc']['nick'] == self.alias): return @@ -475,7 +476,9 @@ async def message(self, message): await task.start_tasks_xmpp( self, jid, ["status"]) else: - response = "Unsupported filetype." + response = ( + "Unsupported filetype. " + "Try: html, md, opml, or xbel") send_reply_message(self, message, response) case _ if (message_lowercase.startswith("gemini:") or message_lowercase.startswith("gopher:")): @@ -537,23 +540,27 @@ async def message(self, message): data, url, ext, filename) if error: response = ( + "> {}\n" "Failed to export {}. Reason: {}" - ).format(ext.upper(), error) + ).format(url, ext.upper(), error) else: url = await upload.start(self, jid, filename) await send_oob_message(self, jid, url) else: response = ( - "Failed to fetch {} Reason: {}" + "> {}\n" + "Failed to fetch URL. Reason: {}" ).format(url, code) await task.start_tasks_xmpp( self, jid, ["status"]) else: response = "Missing entry index number." else: - response = "Unsupported filetype." + response = ( + "Unsupported filetype. " + "Try: epub, html, md (markdown), pdf, or text (txt)") if response: - print(response) + logging.warning("Error for URL {}: {}".format(url, error)) send_reply_message(self, message, response) # case _ if (message_lowercase.startswith("http")) and( # message_lowercase.endswith(".opml")): @@ -741,10 +748,15 @@ async def message(self, message): # TODO Will you add support for number of messages? case "next": # num = message_text[5:] - await task.clean_tasks_xmpp( - jid, ["interval", "status"]) - await task.start_tasks_xmpp( - self, jid, ["interval", "status"]) + # await task.send_update(self, jid, num) + + await task.send_update(self, jid) + + # await task.clean_tasks_xmpp( + # jid, ["interval", "status"]) + # await task.start_tasks_xmpp( + # self, jid, ["interval", "status"]) + # await refresh_task( # self, # jid, diff --git a/slixfeed/xmpp/service.py b/slixfeed/xmpp/service.py index e69de29..15a72ff 100644 --- a/slixfeed/xmpp/service.py +++ b/slixfeed/xmpp/service.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +def identity(self, category): + """ + Identify for Service Duscovery + + Parameters + ---------- + category : str + "client" or "service". + + Returns + ------- + None. + + """ + self["xep_0030"].add_identity( + category=category, + itype="news", + name="slixfeed", + node=None, + jid=self.boundjid.full, + ) \ No newline at end of file diff --git a/slixfeed/xmpp/state.py b/slixfeed/xmpp/state.py index 723e978..795146e 100644 --- a/slixfeed/xmpp/state.py +++ b/slixfeed/xmpp/state.py @@ -27,34 +27,33 @@ async def request(self, jid): breakpoint() self.send_raw(str(presence_probe)) presence_probe.send() - else: - if not self.client_roster[jid]["to"]: - self.send_presence_subscription( - pto=jid, - pfrom=self.boundjid.bare, - ptype="subscribe", - pnick=self.alias - ) - self.send_message( - mto=jid, - mfrom=self.boundjid.bare, - # mtype="headline", - msubject="RSS News Bot", - mbody=( - "Share online status to receive updates." - ), - mnick=self.alias - ) - self.send_presence( - pto=jid, - pfrom=self.boundjid.bare, - # Accept symbol 🉑️ 👍️ ✍ - pstatus=( - "✒️ Share online status to receive updates." - ), - # ptype="subscribe", - pnick=self.alias - ) + elif not self.client_roster[jid]["to"]: + self.send_presence_subscription( + pto=jid, + pfrom=self.boundjid.bare, + ptype="subscribe", + pnick=self.alias + ) + self.send_message( + mto=jid, + mfrom=self.boundjid.bare, + # mtype="headline", + msubject="RSS News Bot", + mbody=( + "Share online status to receive updates." + ), + mnick=self.alias + ) + self.send_presence( + pto=jid, + pfrom=self.boundjid.bare, + # Accept symbol 🉑️ 👍️ ✍ + pstatus=( + "✒️ Share online status to receive updates." + ), + # ptype="subscribe", + pnick=self.alias + ) async def unsubscribed(self, presence):