Improve update interval mechanism.

Add service discovery identity.
Add exception errors.
This commit is contained in:
Schimon Jehudah 2024-01-26 11:34:07 +00:00
parent c1ef5acc7e
commit 4406e61fbe
12 changed files with 291 additions and 125 deletions

View file

@ -51,6 +51,7 @@ dependencies = [
"pdfkit",
"pysocks",
"readability-lxml",
"xml2epub",
]
[project.urls]
@ -60,7 +61,7 @@ Issues = "https://gitgud.io/sjehuda/slixfeed/issues"
[project.optional-dependencies]
file-export = ["html2text", "pdfkit"]
file-export = ["html2text", "pdfkit", "xml2epub"]
proxy = ["pysocks"]
readability = ["readability-lxml"]

View file

@ -100,7 +100,7 @@ import os
#import slixfeed.irc
#import slixfeed.matrix
from slixfeed.config import get_value
from slixfeed.config import get_default_config_directory, get_value
import socks
import socket
@ -190,6 +190,10 @@ class JabberClient:
def main():
config_dir = get_default_config_directory()
logging.info("Reading configuration from {}".format(config_dir))
print("Reading configuration from {}".format(config_dir))
values = get_value(
"accounts", "XMPP Proxy", ["socks5_host", "socks5_port"])
if values[0] and values[1]:

View file

@ -1029,7 +1029,12 @@ def generate_document(data, url, ext, filename):
"Check that package readability is installed.")
match ext:
case "epub":
generate_epub(content, filename)
error = generate_epub(content, filename)
if error:
logging.error(error)
# logging.error(
# "Check that packages xml2epub is installed, "
# "or try again.")
case "html":
generate_html(content, filename)
case "md":
@ -1042,14 +1047,14 @@ def generate_document(data, url, ext, filename):
error = (
"Package html2text was not found.")
case "pdf":
try:
generate_pdf(content, filename)
except:
logging.warning(
"Check that packages pdfkit and wkhtmltopdf "
"are installed, or try again.")
error = (
"Package pdfkit or wkhtmltopdf was not found.")
error = generate_pdf(content, filename)
if error:
logging.error(error)
# logging.warning(
# "Check that packages pdfkit and wkhtmltopdf "
# "are installed, or try again.")
# error = (
# "Package pdfkit or wkhtmltopdf was not found.")
case "txt":
generate_txt(content, filename)
if error:
@ -1126,14 +1131,18 @@ def generate_epub(text, pathname):
# chapter1 = xml2epub.create_chapter_from_url("https://dev.to/devteam/top-7-featured-dev-posts-from-the-past-week-h6h")
# chapter2 = xml2epub.create_chapter_from_url("https://dev.to/ks1912/getting-started-with-docker-34g6")
## add chapters to your eBook
book.add_chapter(chapter0)
# book.add_chapter(chapter1)
# book.add_chapter(chapter2)
## generate epub file
filename_tmp = "slixfeedepub"
book.create_epub(directory, epub_name=filename_tmp)
pathname_tmp = os.path.join(directory, filename_tmp) + ".epub"
os.rename(pathname_tmp, pathname)
try:
book.add_chapter(chapter0)
# book.add_chapter(chapter1)
# book.add_chapter(chapter2)
## generate epub file
filename_tmp = "slixfeedepub"
book.create_epub(directory, epub_name=filename_tmp)
pathname_tmp = os.path.join(directory, filename_tmp) + ".epub"
os.rename(pathname_tmp, pathname)
except ValueError as error:
return error
def generate_html(text, filename):
@ -1150,7 +1159,12 @@ def generate_markdown(text, filename):
def generate_pdf(text, filename):
pdfkit.from_string(text, filename)
try:
pdfkit.from_string(text, filename)
except IOError as error:
return error
except OSError as error:
return error
def generate_txt(text, filename):

View file

@ -9,7 +9,6 @@ from datetime import datetime
from dateutil.parser import parse
from email.utils import parsedate, parsedate_to_datetime
def now():
"""
ISO 8601 Timestamp.

View file

@ -13,16 +13,10 @@ TODO
"""
from asyncio import Lock
from datetime import date
import logging
import slixfeed.config as config
# from slixfeed.data import join_url
from slixfeed.dt import (
current_time,
rfc2822_to_iso8601
)
from sqlite3 import connect, Error, IntegrityError
from slixfeed.url import join_url
import time
# from eliot import start_action, to_file
# # with start_action(action_type="list_feeds()", db=db_file):
@ -82,9 +76,9 @@ def create_tables(db_file):
);
"""
)
properties_table_sql = (
feeds_properties_table_sql = (
"""
CREATE TABLE IF NOT EXISTS properties (
CREATE TABLE IF NOT EXISTS feeds_properties (
id INTEGER NOT NULL,
feed_id INTEGER NOT NULL UNIQUE,
type TEXT,
@ -98,9 +92,9 @@ def create_tables(db_file):
);
"""
)
status_table_sql = (
feeds_state_table_sql = (
"""
CREATE TABLE IF NOT EXISTS status (
CREATE TABLE IF NOT EXISTS feeds_state (
id INTEGER NOT NULL,
feed_id INTEGER NOT NULL UNIQUE,
enabled INTEGER NOT NULL DEFAULT 1,
@ -169,6 +163,16 @@ def create_tables(db_file):
# );
# """
# )
status_table_sql = (
"""
CREATE TABLE IF NOT EXISTS status (
id INTEGER NOT NULL,
key TEXT NOT NULL,
value INTEGER,
PRIMARY KEY ("id")
);
"""
)
settings_table_sql = (
"""
CREATE TABLE IF NOT EXISTS settings (
@ -191,14 +195,15 @@ def create_tables(db_file):
)
cur = conn.cursor()
# cur = get_cursor(db_file)
cur.execute(feeds_table_sql)
cur.execute(status_table_sql)
cur.execute(properties_table_sql)
cur.execute(entries_table_sql)
cur.execute(archive_table_sql)
cur.execute(entries_table_sql)
cur.execute(feeds_table_sql)
cur.execute(feeds_state_table_sql)
cur.execute(feeds_properties_table_sql)
cur.execute(filters_table_sql)
# cur.execute(statistics_table_sql)
cur.execute(settings_table_sql)
cur.execute(filters_table_sql)
cur.execute(status_table_sql)
def get_cursor(db_file):
@ -298,7 +303,7 @@ def insert_feed_status(cur, feed_id):
sql = (
"""
INSERT
INTO status(
INTO feeds_state(
feed_id)
VALUES(
?)
@ -309,7 +314,7 @@ def insert_feed_status(cur, feed_id):
cur.execute(sql, par)
except IntegrityError as e:
logging.warning(
"Skipping feed_id {} for table status".format(feed_id))
"Skipping feed_id {} for table feeds_state".format(feed_id))
logging.error(e)
@ -325,7 +330,7 @@ def insert_feed_properties(cur, feed_id):
sql = (
"""
INSERT
INTO properties(
INTO feeds_properties(
feed_id)
VALUES(
?)
@ -336,7 +341,7 @@ def insert_feed_properties(cur, feed_id):
cur.execute(sql, par)
except IntegrityError as e:
logging.warning(
"Skipping feed_id {} for table properties".format(feed_id))
"Skipping feed_id {} for table feeds_properties".format(feed_id))
logging.error(e)
@ -395,7 +400,7 @@ async def insert_feed(
sql = (
"""
INSERT
INTO status(
INTO feeds_state(
feed_id, enabled, updated, status_code, valid)
VALUES(
?, ?, ?, ?, ?)
@ -408,7 +413,7 @@ async def insert_feed(
sql = (
"""
INSERT
INTO properties(
INTO feeds_properties(
feed_id, entries, type, encoding, language)
VALUES(
?, ?, ?, ?, ?)
@ -636,7 +641,7 @@ async def get_number_of_feeds_active(db_file):
sql = (
"""
SELECT count(id)
FROM status
FROM feeds_state
WHERE enabled = 1
"""
)
@ -1036,7 +1041,7 @@ async def set_enabled_status(db_file, ix, status):
cur = conn.cursor()
sql = (
"""
UPDATE status
UPDATE feeds_state
SET enabled = :status
WHERE feed_id = :id
"""
@ -1155,13 +1160,13 @@ async def add_entries_and_update_timestamp(db_file, feed_id, new_entries):
cur.execute(sql, par)
sql = (
"""
UPDATE status
SET renewed = :today
UPDATE feeds_state
SET renewed = :renewed
WHERE feed_id = :feed_id
"""
)
par = {
"today": date.today(),
"renewed": time.time(),
"feed_id": feed_id
}
cur.execute(sql, par)
@ -1183,13 +1188,13 @@ async def set_date(db_file, feed_id):
cur = conn.cursor()
sql = (
"""
UPDATE status
SET renewed = :today
UPDATE feeds_state
SET renewed = :renewed
WHERE feed_id = :feed_id
"""
)
par = {
"today": date.today(),
"renewed": time.time(),
"feed_id": feed_id
}
# cur = conn.cursor()
@ -1214,14 +1219,14 @@ async def update_feed_status(db_file, feed_id, status_code):
cur = conn.cursor()
sql = (
"""
UPDATE status
UPDATE feeds_state
SET status_code = :status_code, scanned = :scanned
WHERE feed_id = :feed_id
"""
)
par = {
"status_code": status_code,
"scanned": date.today(),
"scanned": time.time(),
"feed_id": feed_id
}
cur.execute(sql, par)
@ -1245,7 +1250,7 @@ async def update_feed_validity(db_file, feed_id, valid):
cur = conn.cursor()
sql = (
"""
UPDATE status
UPDATE feeds_state
SET valid = :valid
WHERE feed_id = :feed_id
"""
@ -1277,7 +1282,7 @@ async def update_feed_properties(db_file, feed_id, entries, updated):
cur = conn.cursor()
sql = (
"""
UPDATE properties
UPDATE feeds_properties
SET entries = :entries
WHERE feed_id = :feed_id
"""
@ -1643,8 +1648,8 @@ async def check_entry_exist(
result = cur.execute(sql, par).fetchone()
if result: exist = True
except:
print(current_time(), "ERROR DATE: source =", feed_id)
print(current_time(), "ERROR DATE: date =", date)
logging.error("source =", feed_id)
logging.error("date =", date)
else:
sql = (
"""
@ -1900,3 +1905,95 @@ async def get_filters_value(db_file, key):
"No specific value set for key {}.".format(key)
)
return value
async def set_last_update_time(db_file):
"""
Set value of last_update.
Parameters
----------
db_file : str
Path to database file.
Returns
-------
None.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = (
"""
INSERT
INTO status(
key, value)
VALUES(
:key, :value)
"""
)
par = {
"key": "last_update",
"value": time.time()
}
cur.execute(sql, par)
async def get_last_update_time(db_file):
"""
Get value of last_update.
Parameters
----------
db_file : str
Path to database file.
Returns
-------
val : str
Time.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
try:
sql = (
"""
SELECT value
FROM status
WHERE key = "last_update"
"""
)
value = cur.execute(sql).fetchone()[0]
value = str(value)
except:
value = None
logging.debug(
"No specific value set for key last_update.")
return value
async def update_last_update_time(db_file):
"""
Update value of last_update.
Parameters
----------
db_file : str
Path to database file.
Returns
-------
None.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = (
"""
UPDATE status
SET value = :value
WHERE key = "last_update"
"""
)
par = {
"value": time.time()
}
cur.execute(sql, par)

View file

@ -42,8 +42,6 @@ NOTE
import asyncio
import logging
import os
import slixmpp
import slixfeed.action as action
from slixfeed.config import (
get_pathname_to_database,
@ -51,19 +49,23 @@ from slixfeed.config import (
get_value)
# from slixfeed.dt import current_time
from slixfeed.sqlite import (
delete_archived_entry,
get_feed_title,
get_feeds_url,
get_number_of_items,
get_last_update_time,
get_number_of_entries_unread,
get_number_of_items,
get_settings_value,
get_unread_entries,
mark_as_read,
mark_entry_as_read,
delete_archived_entry
set_last_update_time,
update_last_update_time
)
# from xmpp import Slixfeed
import slixfeed.xmpp.client as xmpp
import slixfeed.xmpp.utility as utility
import time
main_task = []
jid_tasker = {}
@ -101,6 +103,30 @@ async def start_tasks_xmpp(self, jid, tasks):
task_manager[jid]["status"] = asyncio.create_task(
send_status(self, jid))
case "interval":
db_file = get_pathname_to_database(jid)
update_interval = (
await get_settings_value(db_file, "interval") or
get_value("settings", "Settings", "interval")
)
update_interval = 60 * int(update_interval)
last_update_time = await get_last_update_time(db_file)
if last_update_time:
last_update_time = float(last_update_time)
diff = time.time() - last_update_time
if diff < update_interval:
next_update_time = update_interval - diff
print("jid :", jid, "\n"
"time :", time.time(), "\n"
"last_update_time :", last_update_time, "\n"
"difference :", diff, "\n"
"update interval :", update_interval, "\n"
"next_update_time :", next_update_time, "\n")
await asyncio.sleep(next_update_time)
# elif diff > val:
# next_update_time = val
await update_last_update_time(db_file)
else:
await set_last_update_time(db_file)
task_manager[jid]["interval"] = asyncio.create_task(
send_update(self, jid))
# for task in task_manager[jid].values():
@ -152,11 +178,8 @@ async def task_jid(self, jid):
"""
db_file = get_pathname_to_database(jid)
enabled = (
await get_settings_value(
db_file, "enabled")
) or (
get_value(
"settings", "Settings", "enabled")
await get_settings_value(db_file, "enabled") or
get_value("settings", "Settings", "enabled")
)
if enabled:
# NOTE Perhaps we want to utilize super with keyword
@ -208,20 +231,14 @@ async def send_update(self, jid, num=None):
logging.debug("Sending a news update to JID {}".format(jid))
db_file = get_pathname_to_database(jid)
enabled = (
await get_settings_value(
db_file, "enabled")
) or (
get_value(
"settings", "Settings", "enabled")
await get_settings_value(db_file, "enabled") or
get_value("settings", "Settings", "enabled")
)
if enabled:
if not num:
num = (
await get_settings_value(
db_file, "quantum")
) or (
get_value(
"settings", "Settings", "quantum")
await get_settings_value(db_file, "quantum") or
get_value("settings", "Settings", "quantum")
)
else:
num = int(num)
@ -341,11 +358,8 @@ async def send_status(self, jid):
status_text = "📜️ Slixfeed RSS News Bot"
db_file = get_pathname_to_database(jid)
enabled = (
await get_settings_value(
db_file, "enabled")
) or (
get_value(
"settings", "Settings", "enabled")
await get_settings_value(db_file, "enabled") or
get_value("settings", "Settings", "enabled")
)
if not enabled:
status_mode = "xa"
@ -414,11 +428,8 @@ async def refresh_task(self, jid, callback, key, val=None):
if not val:
db_file = get_pathname_to_database(jid)
val = (
await get_settings_value(
db_file, key)
) or (
get_value(
"settings", "Settings", key)
await get_settings_value(db_file, key) or
get_value("settings", "Settings", key)
)
# if task_manager[jid][key]:
if jid in task_manager:

View file

@ -72,6 +72,7 @@ import slixfeed.xmpp.muc as muc
import slixfeed.xmpp.process as process
import slixfeed.xmpp.profile as profile
import slixfeed.xmpp.roster as roster
import slixfeed.xmpp.service as service
import slixfeed.xmpp.state as state
import slixfeed.xmpp.status as status
import slixfeed.xmpp.utility as utility
@ -94,8 +95,7 @@ loop = asyncio.get_event_loop()
class Slixfeed(slixmpp.ClientXMPP):
"""
Slixmpp
-------
Slixfeed:
News bot that sends updates from RSS feeds.
"""
def __init__(self, jid, password, hostname=None, port=None, alias=None):
@ -178,6 +178,7 @@ class Slixfeed(slixmpp.ClientXMPP):
await process.event(self, event)
await muc.autojoin(self)
await profile.update(self)
service.identity(self, "client")
async def on_session_resumed(self, event):

View file

@ -65,6 +65,7 @@ import slixfeed.xmpp.muc as muc
import slixfeed.xmpp.process as process
import slixfeed.xmpp.profile as profile
import slixfeed.xmpp.roster as roster
import slixfeed.xmpp.service as service
import slixfeed.xmpp.state as state
import slixfeed.xmpp.status as status
import slixfeed.xmpp.utility as utility
@ -162,6 +163,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
await process.event_component(self, event)
# await muc.autojoin(self)
await profile.update(self)
service.identity(self, "service")
async def on_session_resumed(self, event):

View file

@ -11,6 +11,8 @@ TODO
3) If groupchat error is received, send that error message to inviter.
4) Save name of groupchat instead of jid as name
"""
import logging
import slixfeed.xmpp.bookmark as bookmark

View file

@ -92,6 +92,7 @@ async def message(self, message):
# return
# FIXME Code repetition. See below.
# TODO Check alias by nickname associated with conference
if message["type"] == "groupchat":
if (message['muc']['nick'] == self.alias):
return
@ -475,7 +476,9 @@ async def message(self, message):
await task.start_tasks_xmpp(
self, jid, ["status"])
else:
response = "Unsupported filetype."
response = (
"Unsupported filetype. "
"Try: html, md, opml, or xbel")
send_reply_message(self, message, response)
case _ if (message_lowercase.startswith("gemini:") or
message_lowercase.startswith("gopher:")):
@ -537,23 +540,27 @@ async def message(self, message):
data, url, ext, filename)
if error:
response = (
"> {}\n"
"Failed to export {}. Reason: {}"
).format(ext.upper(), error)
).format(url, ext.upper(), error)
else:
url = await upload.start(self, jid, filename)
await send_oob_message(self, jid, url)
else:
response = (
"Failed to fetch {} Reason: {}"
"> {}\n"
"Failed to fetch URL. Reason: {}"
).format(url, code)
await task.start_tasks_xmpp(
self, jid, ["status"])
else:
response = "Missing entry index number."
else:
response = "Unsupported filetype."
response = (
"Unsupported filetype. "
"Try: epub, html, md (markdown), pdf, or text (txt)")
if response:
print(response)
logging.warning("Error for URL {}: {}".format(url, error))
send_reply_message(self, message, response)
# case _ if (message_lowercase.startswith("http")) and(
# message_lowercase.endswith(".opml")):
@ -741,10 +748,15 @@ async def message(self, message):
# TODO Will you add support for number of messages?
case "next":
# num = message_text[5:]
await task.clean_tasks_xmpp(
jid, ["interval", "status"])
await task.start_tasks_xmpp(
self, jid, ["interval", "status"])
# await task.send_update(self, jid, num)
await task.send_update(self, jid)
# await task.clean_tasks_xmpp(
# jid, ["interval", "status"])
# await task.start_tasks_xmpp(
# self, jid, ["interval", "status"])
# await refresh_task(
# self,
# jid,

View file

@ -0,0 +1,24 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
def identity(self, category):
"""
Identify for Service Duscovery
Parameters
----------
category : str
"client" or "service".
Returns
-------
None.
"""
self["xep_0030"].add_identity(
category=category,
itype="news",
name="slixfeed",
node=None,
jid=self.boundjid.full,
)

View file

@ -27,34 +27,33 @@ async def request(self, jid):
breakpoint()
self.send_raw(str(presence_probe))
presence_probe.send()
else:
if not self.client_roster[jid]["to"]:
self.send_presence_subscription(
pto=jid,
pfrom=self.boundjid.bare,
ptype="subscribe",
pnick=self.alias
)
self.send_message(
mto=jid,
mfrom=self.boundjid.bare,
# mtype="headline",
msubject="RSS News Bot",
mbody=(
"Share online status to receive updates."
),
mnick=self.alias
)
self.send_presence(
pto=jid,
pfrom=self.boundjid.bare,
# Accept symbol 🉑️ 👍️ ✍
pstatus=(
"✒️ Share online status to receive updates."
),
# ptype="subscribe",
pnick=self.alias
)
elif not self.client_roster[jid]["to"]:
self.send_presence_subscription(
pto=jid,
pfrom=self.boundjid.bare,
ptype="subscribe",
pnick=self.alias
)
self.send_message(
mto=jid,
mfrom=self.boundjid.bare,
# mtype="headline",
msubject="RSS News Bot",
mbody=(
"Share online status to receive updates."
),
mnick=self.alias
)
self.send_presence(
pto=jid,
pfrom=self.boundjid.bare,
# Accept symbol 🉑️ 👍️ ✍
pstatus=(
"✒️ Share online status to receive updates."
),
# ptype="subscribe",
pnick=self.alias
)
async def unsubscribed(self, presence):