sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

WIP: Closer to fix double message. See task.py

Browse source
This commit is contained in:
Schimon Jehudah 2024-02-10 17:53:53 +00:00
parent c568145ecf
commit 9434833449
16 changed files with 369 additions and 242 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
slixfeed/__main__.py
View file

@ -8,9 +8,9 @@ FIXME
Consequently, it might result in database lock error upon
feed removal attempt
TODO
2) Communicate to messages of new contacts (not subscribed and not in roster)
1) SQL prepared statements;
TODO
2) Machine Learning for scrapping Title, Link, Summary and Timstamp;
Scrape element </article> (example: Liferea)
@ -22,15 +22,9 @@ TODO
Perhaps not, as it would require to check every feed for this setting.
Maybe a separate bot;
4) Support categories;
5) OMEMO;
5) XMPP commands;
6) Bot as service;
7) OMEMO;
8) Logging;
6) Logging;
https://docs.python.org/3/howto/logging.html
9) Readability
@ -43,12 +37,6 @@ TODO
Store 5 upcoming summaries.
This would help making the database files smaller.
11) Support protocol Gopher
See project /michael-lazar/pygopherd
See project /gopherball/gb
12) Support ActivityPub @person@domain (see Tip Of The Day).
13) Tip Of The Day.
Did you know that you can follow you favorite Mastodon feeds by just
sending the URL address?
@ -57,21 +45,15 @@ TODO
Mastodon, Misskey, Pixelfed, Pleroma, Socialhome, Soapbox.
14) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger
15) See project /offpunk/offblocklist.py
16) Search messages of government regulated publishers, and promote other sources.
Dear reader, we couldn't get news from XYZ as they don't provide RSS feeds.
However, you might want to get news from (1) (2) and (3) instead!
17) Make the program portable (directly use the directory assets) -- Thorsten
18) The operator account will be given reports from the bot about its
17) The operator account will be given reports from the bot about its
activities every X minutes.
When a suspicious activity is detected, it will be reported immediately.
19) Communicate to messages of new contacts (not subscribed and not in roster)
"""
# vars and their meanings:

22
slixfeed/action.py
View file

@ -86,6 +86,7 @@ except ImportError:
"Package readability was not found.\n"
"Arc90 Lab algorithm is disabled.")
def manual(filename, section=None, command=None):
config_dir = config.get_default_config_directory()
with open(config_dir + '/' + filename, mode="rb") as commands:
@ -111,7 +112,8 @@ def manual(filename, section=None, command=None):
return cmd_list
async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, session=None):
async def xmpp_change_interval(self, key, val, jid, jid_file, message=None,
session=None):
if val:
# response = (
# 'Updates will be sent every {} minutes.'
@ -123,7 +125,7 @@ async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, sess
await sqlite.set_settings_value(db_file, [key, val])
# NOTE Perhaps this should be replaced
# by functions clean and start
await task.refresh_task(self, jid, task.send_update, key, val)
await task.refresh_task(self, jid, task.task_send, key, val)
response = ('Updates will be sent every {} minutes.'
.format(val))
else:
@ -344,17 +346,17 @@ def list_search_results(query, results):
return message
def list_feeds_by_query(query, results):
def list_feeds_by_query(db_file, query):
results = sqlite.search_feeds(db_file, query)
message = (
"Feeds containing '{}':\n\n```"
).format(query)
'Feeds containing "{}":\n\n```'
.format(query))
for result in results:
message += (
"\nName : {} [{}]"
"\nURL : {}"
"\n"
).format(
str(result[0]), str(result[1]), str(result[2]))
'\nName : {} [{}]'
'\nURL : {}'
'\n'
.format(str(result[0]), str(result[1]), str(result[2])))
if len(results):
message += "\n```\nTotal of {} feeds".format(len(results))
else:

10
slixfeed/assets/commands.toml
View file

@ -141,15 +141,19 @@ interval <minutes>
Set interval update to every given <minutes>.
"""
length = """
length
length <number>
Set maximum length of news item description. (0 for no limit)
"""
media = """
media [off|on]
Attach media (i.e. audio, image, video) to messages when available.
"""
quantum = """
quantum <number>
Set amount of updates per message by given <number>.
"""
random = """
random
random [off|on]
Send messages by random order instead of date.
"""
@ -190,7 +194,7 @@ Disable bot and stop updates.
[preview]
read = """
read <url>
Display most recent 5 titles of given <url>.
Display recent 5 titles of given <url>.
"""
read_num = """
read <url> <index>

18
slixfeed/assets/information.toml
View file

@ -3,10 +3,10 @@ Slixfeed
A Syndication bot for the XMPP communication network.
Slixfeed aims to be an easy to use and fully-featured news \
aggregator bot for XMPP. It provides a convenient access to Blogs, \
News websites and even Fediverse instances, along with filtering \
functionality.
Slixfeed is a news broker which aims to be an easy to use and fully-\
featured news aggregator bot. It provides a convenient access to \
Blogs, News websites and even Fediverse instances, along with \
filtering functionality.
Slixfeed is primarily designed for XMPP (aka Jabber). \
Visit https://xmpp.org/software/ for more information.
@ -61,7 +61,7 @@ No operator was specified for this instance.
platforms = """
Supported platforms: XMPP
Platforms to be added in future: ActivityPub, Briar, Email, IRC, LXMF, Matrix, MQTT, Nostr, Tox.
Platforms to be added in future: ActivityPub, Briar, Email, IRC, LXMF, Matrix, MQTT, Nostr, Session, Tox.
For ideal experience, we recommend using XMPP.
"""
@ -78,10 +78,15 @@ Protocols to be added in future: Dat, FTP, Gemini, Gopher, IPFS.
resources = """
Slixfeed
https://gitgud.io/sjehuda/slixfeed
Slixmpp
https://slixmpp.readthedocs.io/
feedparser
https://pythonhosted.org/feedparser
XMPP
https://xmpp.org/about/
"""
terms = """
@ -113,13 +118,14 @@ imattau (atomtopubsub), \
Jaussoin Timothée <mov.im> (Movim, France), \
Justin Karneges <jblog.andbit.net> (Psi, California), \
Kevin Smith <isode.com> (Swift IM, Wales), \
Lars Windolf (Liferea, Germany), \
Luis Henrique Mello (SalixOS, Brazil), \
magicfelix, \
Markus Muttilainen (SalixOS), \
Martin <debacle@debian.org> (Debian, Germany), \
Mathieu Pasquet (slixmpp, France), \
Maxime Buquet (slixmpp, France), \
Phillip Watkins (United Kingdom, SalixOS), \
Phillip Watkins (SalixOS, United Kingdom), \
Pierrick Le Brun (SalixOS, France), \
Raphael Groner (Fedora, Germany), \
Remko Tronçon <mko.re> (Psi , Belgium), \

4
slixfeed/config.py
View file

@ -19,6 +19,10 @@ TODO
6) Use TOML https://ruudvanasseldonk.com/2023/01/11/the-yaml-document-from-hell
7) Make the program portable (directly use the directory assets) -- Thorsten
7.1) Read missing files from base directories or either set error message.
"""
import configparser

8
slixfeed/crawl.py
View file

@ -3,6 +3,14 @@
"""
FIXME
1) https://wiki.pine64.org
File "/slixfeed/crawl.py", line 178, in feed_mode_guess
address = join_url(url, parted_url.path.split('/')[1] + path)
~~~~~~~~~~~~~~~~~~~~~~~~~~^^^
IndexError: list index out of range
TODO
1.1) Attempt to scan more paths: /blog/, /news/ etc., including root /

17
slixfeed/fetch.py
View file

@ -21,6 +21,14 @@ TODO
4) Replace sqlite.remove_nonexistent_entries by sqlite.check_entry_exist
Same check, just reverse.
5) Support protocol Gopher
See project /michael-lazar/pygopherd
See project /gopherball/gb
6) Support ActivityPub @person@domain (see Tip Of The Day).
7) See project /offpunk/offblocklist.py
"""
from aiohttp import ClientError, ClientSession, ClientTimeout
@ -41,18 +49,25 @@ except:
"BitTorrent is disabled.")
# class FetchDat:
# async def dat():
# class FetchFtp:
# async def ftp():
# class FetchGemini:
# async def gemini():
# class FetchGopher:
# async def gopher():
# class FetchHttp:
# async def http():
# class FetchIpfs:
# async def ipfs():
def http_response(url):
"""
Download response headers.

138
slixfeed/sqlite.py
View file

@ -10,6 +10,11 @@ TODO
All other functions to receive cursor.
2) Merge function add_metadata into function import_feeds.
3) SQL prepared statements.
4) Support categories;
"""
from asyncio import Lock
@ -66,6 +71,51 @@ def create_tables(db_file):
Path to database file.
"""
with create_connection(db_file) as conn:
archive_table_sql = (
"""
CREATE TABLE IF NOT EXISTS archive (
id INTEGER NOT NULL,
title TEXT NOT NULL,
link TEXT NOT NULL,
enclosure TEXT,
entry_id TEXT NOT NULL,
feed_id INTEGER NOT NULL,
timestamp TEXT,
read INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
ON UPDATE CASCADE
ON DELETE CASCADE,
PRIMARY KEY ("id")
);
"""
)
categories_table_sql = (
"""
CREATE TABLE IF NOT EXISTS categories (
id INTEGER NOT NULL,
name TEXT,
PRIMARY KEY ("id")
);
"""
)
entries_table_sql = (
"""
CREATE TABLE IF NOT EXISTS entries (
id INTEGER NOT NULL,
title TEXT NOT NULL,
link TEXT NOT NULL,
enclosure TEXT,
entry_id TEXT NOT NULL,
feed_id INTEGER NOT NULL,
timestamp TEXT,
read INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
ON UPDATE CASCADE
ON DELETE CASCADE,
PRIMARY KEY ("id")
);
"""
)
feeds_table_sql = (
"""
CREATE TABLE IF NOT EXISTS feeds (
@ -76,18 +126,19 @@ def create_tables(db_file):
);
"""
)
feeds_statistics_table_sql = (
# TODO Rethink!
# Albeit, probably, more expensive, we might want to have feed_id
# as foreign key, as it is with feeds_properties and feeds_state
feeds_categories_table_sql = (
"""
CREATE TABLE IF NOT EXISTS statistics (
CREATE TABLE IF NOT EXISTS feeds_categories (
id INTEGER NOT NULL,
feed_id INTEGER NOT NULL UNIQUE,
offline INTEGER,
entries INTEGER,
entries INTEGER,
FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
category_id INTEGER NOT NULL UNIQUE,
feed_id INTEGER,
FOREIGN KEY ("category_id") REFERENCES "categories" ("id")
ON UPDATE CASCADE
ON DELETE CASCADE,
PRIMARY KEY ("id")
PRIMARY KEY (id)
);
"""
)
@ -127,53 +178,32 @@ def create_tables(db_file):
);
"""
)
feeds_statistics_table_sql = (
"""
CREATE TABLE IF NOT EXISTS statistics (
id INTEGER NOT NULL,
feed_id INTEGER NOT NULL UNIQUE,
offline INTEGER,
entries INTEGER,
entries INTEGER,
FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
ON UPDATE CASCADE
ON DELETE CASCADE,
PRIMARY KEY ("id")
);
"""
)
# TODO
# Consider parameter unique:
# entry_id TEXT NOT NULL UNIQUE,
# Will eliminate function:
# check_entry_exist
entries_table_sql = (
filters_table_sql = (
"""
CREATE TABLE IF NOT EXISTS entries (
id INTEGER NOT NULL,
title TEXT NOT NULL,
link TEXT NOT NULL,
enclosure TEXT,
entry_id TEXT NOT NULL,
feed_id INTEGER NOT NULL,
timestamp TEXT,
read INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
ON UPDATE CASCADE
ON DELETE CASCADE,
PRIMARY KEY ("id")
);
"""
)
archive_table_sql = (
"""
CREATE TABLE IF NOT EXISTS archive (
id INTEGER NOT NULL,
title TEXT NOT NULL,
link TEXT NOT NULL,
enclosure TEXT,
entry_id TEXT NOT NULL,
feed_id INTEGER NOT NULL,
timestamp TEXT,
read INTEGER NOT NULL DEFAULT 0,
FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
ON UPDATE CASCADE
ON DELETE CASCADE,
PRIMARY KEY ("id")
);
"""
)
status_table_sql = (
"""
CREATE TABLE IF NOT EXISTS status (
CREATE TABLE IF NOT EXISTS filters (
id INTEGER NOT NULL,
key TEXT NOT NULL,
value INTEGER,
value TEXT,
PRIMARY KEY ("id")
);
"""
@ -188,12 +218,12 @@ def create_tables(db_file):
);
"""
)
filters_table_sql = (
status_table_sql = (
"""
CREATE TABLE IF NOT EXISTS filters (
CREATE TABLE IF NOT EXISTS status (
id INTEGER NOT NULL,
key TEXT NOT NULL,
value TEXT,
value INTEGER,
PRIMARY KEY ("id")
);
"""
@ -863,17 +893,17 @@ async def archive_entry(db_file, ix):
)
def get_feed_title(db_file, ix):
def get_feed_title(db_file, feed_id):
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = (
"""
SELECT name
FROM feeds
WHERE id = :ix
WHERE id = :feed_id
"""
)
par = (ix,)
par = (feed_id,)
title = cur.execute(sql, par).fetchone()
return title
@ -1520,7 +1550,7 @@ async def last_entries(db_file, num):
return results
async def search_feeds(db_file, query):
def search_feeds(db_file, query):
"""
Query feeds.

142
slixfeed/task.py
View file

@ -5,12 +5,18 @@
FIXME
0) URGENT!!! Place "await asyncio.sleep(next_update_time)" ***inside*** the
task, and not outside as it is now!
1) Function check_readiness or event "changed_status" is causing for
triple status messages and also false ones that indicate of lack
of feeds.
TODO
0) Move functions send_status and send_update to module action
1) Deprecate "add" (see above) and make it interactive.
Slixfeed: Do you still want to add this URL to subscription list?
See: case _ if message_lowercase.startswith("add"):
@ -70,13 +76,13 @@ loop = asyncio.get_event_loop()
# task_ping = asyncio.create_task(ping(self, jid=None))
def ping_task(self):
# global ping_task_instance
def task_ping(self):
# global task_ping_instance
try:
self.ping_task_instance.cancel()
self.task_ping_instance.cancel()
except:
logging.info('No ping task to cancel.')
self.ping_task_instance = asyncio.create_task(XmppConnect.ping(self))
self.task_ping_instance = asyncio.create_task(XmppConnect.ping(self))
"""
@ -118,7 +124,7 @@ async def start_tasks_xmpp(self, jid, tasks=None):
self.task_manager[jid][task].cancel()
except:
logging.info('No task {} for JID {} (start_tasks_xmpp)'
.format(task, jid))
.format(task, jid))
logging.info('Starting tasks {} for JID {}'.format(tasks, jid))
for task in tasks:
# print("task:", task)
@ -133,34 +139,8 @@ async def start_tasks_xmpp(self, jid, tasks=None):
self.task_manager[jid]['status'] = asyncio.create_task(
send_status(self, jid))
case 'interval':
jid_file = jid.replace('/', '_')
db_file = config.get_pathname_to_database(jid_file)
update_interval = await config.get_setting_value(db_file,
'interval')
update_interval = 60 * int(update_interval)
last_update_time = await sqlite.get_last_update_time(db_file)
if last_update_time:
last_update_time = float(last_update_time)
diff = time.time() - last_update_time
if diff < update_interval:
next_update_time = update_interval - diff
await asyncio.sleep(next_update_time)
# print("jid :", jid, "\n"
# "time :", time.time(), "\n"
# "last_update_time :", last_update_time, "\n"
# "difference :", diff, "\n"
# "update interval :", update_interval, "\n"
# "next_update_time :", next_update_time, "\n"
# )
# elif diff > val:
# next_update_time = val
await sqlite.update_last_update_time(db_file)
else:
await sqlite.set_last_update_time(db_file)
self.task_manager[jid]['interval'] = asyncio.create_task(
send_update(self, jid))
task_send(self, jid))
# for task in self.task_manager[jid].values():
# print("task_manager[jid].values()")
# print(self.task_manager[jid].values())
@ -172,20 +152,57 @@ async def start_tasks_xmpp(self, jid, tasks=None):
# await task
def clean_tasks_xmpp(self, jid, tasks=None):
if not tasks:
tasks = ['interval', 'status', 'check']
logging.info('Stopping tasks {} for JID {}'.format(tasks, jid))
for task in tasks:
# if self.task_manager[jid][task]:
try:
self.task_manager[jid][task].cancel()
except:
logging.debug('No task {} for JID {} (clean_tasks_xmpp)'
.format(task, jid))
async def task_send(self, jid):
print("task_send for", jid)
try:
self.task_manager[jid]['interval'].cancel()
except:
logging.info('No task interval for JID {} (start_tasks_xmpp)'
.format(jid))
jid_file = jid.replace('/', '_')
print(jid_file)
db_file = config.get_pathname_to_database(jid_file)
print(db_file)
update_interval = await config.get_setting_value(db_file, 'interval')
print(update_interval)
update_interval = 60 * int(update_interval)
print(update_interval)
last_update_time = await sqlite.get_last_update_time(db_file)
print(last_update_time)
if last_update_time:
print('if')
last_update_time = float(last_update_time)
diff = time.time() - last_update_time
if diff < update_interval:
next_update_time = update_interval - diff
print('next_update_time')
print(next_update_time)
print(next_update_time/60/60)
await asyncio.sleep(next_update_time) # FIXME!
print('after await sleep')
# print("jid :", jid, "\n"
# "time :", time.time(), "\n"
# "last_update_time :", last_update_time, "\n"
# "difference :", diff, "\n"
# "update interval :", update_interval, "\n"
# "next_update_time :", next_update_time, "\n"
# )
# elif diff > val:
# next_update_time = val
print('await (if)')
await sqlite.update_last_update_time(db_file)
else:
print('await (else)')
await sqlite.set_last_update_time(db_file)
print("await is done for", jid)
await xmpp_send_update(self, jid)
await start_tasks_xmpp(self, jid, ['status'])
await refresh_task(self, jid, task_send, 'interval')
async def send_update(self, jid, num=None):
async def xmpp_send_update(self, jid, num=None):
"""
Send news items as messages.
@ -196,21 +213,28 @@ async def send_update(self, jid, num=None):
num : str, optional
Number. The default is None.
"""
logging.info('Sending a news update to JID {}'.format(jid))
print('Sending a news update to JID {}'.format(jid))
jid_file = jid.replace('/', '_')
print(jid_file)
db_file = config.get_pathname_to_database(jid_file)
print(db_file)
enabled = await config.get_setting_value(db_file, 'enabled')
print(enabled)
if enabled:
print('enabled')
if not num:
num = await config.get_setting_value(db_file, 'quantum')
else:
num = int(num)
news_digest = []
print(num)
results = await sqlite.get_unread_entries(db_file, num)
print(results)
news_digest = ''
media = None
chat_type = await get_chat_type(self, jid)
print(jid, num, chat_type)
for result in results:
print(result)
ix = result[0]
title_e = result[1]
url = result[2]
@ -239,6 +263,7 @@ async def send_update(self, jid, num=None):
if media and news_digest:
print('SENDING MESSAGE (if media and news_digest)')
print(news_digest)
print(media)
# Send textual message
XmppMessage.send(self, jid, news_digest, chat_type)
news_digest = ''
@ -249,12 +274,13 @@ async def send_update(self, jid, num=None):
if news_digest:
print('SENDING MESSAGE (if news_digest)')
print(news_digest)
XmppMessage.send(self, jid, news_digest, chat_type)
# TODO Add while loop to assure delivery.
# print(await current_time(), ">>> ACT send_message",jid)
# NOTE Do we need "if statement"? See NOTE at is_muc.
if chat_type in ('chat', 'groupchat'):
# TODO Provide a choice (with or without images)
XmppMessage.send(self, jid, news_digest, chat_type)
# if chat_type in ('chat', 'groupchat'):
# # TODO Provide a choice (with or without images)
# XmppMessage.send(self, jid, news_digest, chat_type)
# See XEP-0367
# if media:
# # message = xmpp.Slixfeed.make_message(
@ -266,7 +292,10 @@ async def send_update(self, jid, num=None):
# TODO Do not refresh task before
# verifying that it was completed.
await refresh_task(self, jid, send_update, 'interval')
# await start_tasks_xmpp(self, jid, ['status'])
# await refresh_task(self, jid, send_update, 'interval')
# interval = await initdb(
# jid,
# sqlite.get_settings_value,
@ -292,6 +321,19 @@ async def send_update(self, jid, num=None):
# await handle_event()
def clean_tasks_xmpp(self, jid, tasks=None):
if not tasks:
tasks = ['interval', 'status', 'check']
logging.info('Stopping tasks {} for JID {}'.format(tasks, jid))
for task in tasks:
# if self.task_manager[jid][task]:
try:
self.task_manager[jid][task].cancel()
except:
logging.debug('No task {} for JID {} (clean_tasks_xmpp)'
.format(task, jid))
async def send_status(self, jid):
"""
Send status message.

16
slixfeed/url.py
View file

@ -3,6 +3,13 @@
"""
FIXME
1) Do not handle base64
https://www.lilithsaintcrow.com/2024/02/love-anonymous/
data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAeAAQAAAAAQ6M16AAAAAnRSTlMAAHaTzTgAAAFmSURBVBgZ7cEBAQAAAIKg/q92SMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgWE3LAAGyZmPPAAAAAElFTkSuQmCC
https://www.lilithsaintcrow.com/2024/02/love-anonymous//image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAeAAQAAAAAQ6M16AAAAAnRSTlMAAHaTzTgAAAFmSURBVBgZ7cEBAQAAAIKg/q92SMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgWE3LAAGyZmPPAAAAAElFTkSuQmCC
TODO
1) ActivityPub URL revealer activitypub_to_http.
@ -40,6 +47,7 @@ def get_hostname(url):
parted_url = urlsplit(url)
return parted_url.netloc
def replace_hostname(url, url_type):
"""
Replace hostname.
@ -120,6 +128,8 @@ def remove_tracking_parameters(url):
url : str
URL.
"""
if url.startswith('data:') and ';base64,' in url:
return url
parted_url = urlsplit(url)
protocol = parted_url.scheme
hostname = parted_url.netloc
@ -192,6 +202,8 @@ def complete_url(source, link):
str
URL.
"""
if link.startswith('data:') and ';base64,' in link:
return link
if link.startswith('www.'):
return 'http://' + link
parted_link = urlsplit(link)
@ -270,6 +282,8 @@ def join_url(source, link):
str
URL.
"""
if link.startswith('data:') and ';base64,' in link:
return link
if link.startswith('www.'):
new_link = 'http://' + link
elif link.startswith('%20') and link.endswith('%20'):
@ -296,6 +310,8 @@ def trim_url(url):
url : str
URL.
"""
if url.startswith('data:') and ';base64,' in url:
return url
parted_url = urlsplit(url)
protocol = parted_url.scheme
hostname = parted_url.netloc

4
slixfeed/version.py
View file

@ -1,2 +1,2 @@
__version__ = '0.1.2'
__version_info__ = (0, 1, 2)
__version__ = '0.1.3'
__version_info__ = (0, 1, 3)

62
slixfeed/xmpp/client.py
View file

@ -108,7 +108,7 @@ class Slixfeed(slixmpp.ClientXMPP):
self.task_manager = {}
# Handlers for ping
self.ping_task_instance = {}
self.task_ping_instance = {}
# Handlers for connection events
self.connection_attempts = 0
@ -124,8 +124,8 @@ class Slixfeed(slixmpp.ClientXMPP):
self.on_changed_status)
self.add_event_handler("presence_available",
self.on_presence_available)
self.add_event_handler("presence_unavailable",
self.on_presence_unavailable)
# self.add_event_handler("presence_unavailable",
# self.on_presence_unavailable)
self.add_event_handler("chatstate_active",
self.on_chatstate_active)
self.add_event_handler("chatstate_composing",
@ -218,7 +218,7 @@ class Slixfeed(slixmpp.ClientXMPP):
await XmppGroupchat.autojoin(self)
profile.set_identity(self, 'client')
await profile.update(self)
task.ping_task(self)
task.task_ping(self)
# Service.commands(self)
# Service.reactions(self)
@ -262,6 +262,8 @@ class Slixfeed(slixmpp.ClientXMPP):
async def on_changed_status(self, presence):
# await task.check_readiness(self, presence)
jid = presence['from'].bare
if jid in self.boundjid.bare:
return
if presence['show'] in ('away', 'dnd', 'xa'):
task.clean_tasks_xmpp(self, jid, ['interval'])
await task.start_tasks_xmpp(self, jid, ['status', 'check'])
@ -270,6 +272,7 @@ class Slixfeed(slixmpp.ClientXMPP):
async def on_presence_subscribe(self, presence):
jid = presence['from'].bare
if not self.client_roster[jid]['to']:
# XmppPresence.subscription(self, jid, 'subscribe')
XmppPresence.subscription(self, jid, 'subscribed')
await XmppRoster.add(self, jid)
status_message = '✒️ Share online status to receive updates'
@ -280,8 +283,9 @@ class Slixfeed(slixmpp.ClientXMPP):
'chat')
async def on_presence_subscribed(self, presence):
def on_presence_subscribed(self, presence):
jid = presence['from'].bare
# XmppPresence.subscription(self, jid, 'subscribed')
message_subject = 'RSS News Bot'
message_body = ('Greetings! I am {}, the news anchor.\n'
'My job is to bring you the latest '
@ -297,13 +301,18 @@ class Slixfeed(slixmpp.ClientXMPP):
# await task.start_tasks(self, presence)
# NOTE Already done inside the start-task function
jid = presence['from'].bare
if jid in self.boundjid.bare:
return
print('JID available:', jid)
# FIXME TODO Find out what is the source responsible for a couple presences with empty message
# NOTE This is a temporary solution
await asyncio.sleep(10)
await task.start_tasks_xmpp(self, jid)
self.add_event_handler("presence_unavailable",
self.on_presence_unavailable)
async def on_presence_unsubscribed(self, presence):
def on_presence_unsubscribed(self, presence):
jid = presence['from'].bare
message_body = 'You have been unsubscribed.'
# status_message = '🖋️ Subscribe to receive updates'
@ -312,64 +321,81 @@ class Slixfeed(slixmpp.ClientXMPP):
XmppPresence.subscription(self, jid, 'unsubscribed')
# XmppPresence.send(self, jid, status_message,
# presence_type='unsubscribed')
await XmppRoster.remove(self, jid)
XmppRoster.remove(self, jid)
async def on_presence_unavailable(self, presence):
def on_presence_unavailable(self, presence):
jid = presence['from'].bare
print('JID unavailable:', jid)
# await task.stop_tasks(self, jid)
task.clean_tasks_xmpp(self, jid)
# NOTE Albeit nice to ~have~ see, this would constantly
# send presence messages to server to no end.
status_message = 'Farewell'
XmppPresence.send(self, jid, status_message,
presence_type='unavailable')
self.del_event_handler("presence_unavailable",
self.on_presence_unavailable)
# TODO
# Send message that database will be deleted within 30 days
# Check whether JID is in bookmarks or roster
# If roster, remove contact JID into file
# If bookmarks, remove groupchat JID into file
async def on_presence_error(self, presence):
print("on_presence_error")
print(presence)
def on_presence_error(self, presence):
jid = presence["from"].bare
print('JID error:', jid)
task.clean_tasks_xmpp(self, jid)
async def on_reactions(self, message):
def on_reactions(self, message):
print(message['from'])
print(message['reactions']['values'])
async def on_chatstate_active(self, message):
jid = message['from'].bare
if jid in self.boundjid.bare:
return
if message['type'] in ('chat', 'normal'):
jid = message['from'].bare
# task.clean_tasks_xmpp(self, jid, ['status'])
await task.start_tasks_xmpp(self, jid, ['status'])
async def on_chatstate_composing(self, message):
def on_chatstate_composing(self, message):
if message['type'] in ('chat', 'normal'):
jid = message['from'].bare
# task.clean_tasks_xmpp(self, jid, ['status'])
status_message='💡 Press "help" for manual, or "info" for information.'
status_message = ('💡 Send "help" for manual, or "info" for '
'information.')
XmppPresence.send(self, jid, status_message)
async def on_chatstate_gone(self, message):
jid = message['from'].bare
if jid in self.boundjid.bare:
return
if message['type'] in ('chat', 'normal'):
jid = message['from'].bare
# task.clean_tasks_xmpp(self, jid, ['status'])
await task.start_tasks_xmpp(self, jid, ['status'])
async def on_chatstate_inactive(self, message):
jid = message['from'].bare
if jid in self.boundjid.bare:
return
if message['type'] in ('chat', 'normal'):
jid = message['from'].bare
# task.clean_tasks_xmpp(self, jid, ['status'])
await task.start_tasks_xmpp(self, jid, ['status'])
async def on_chatstate_paused(self, message):
jid = message['from'].bare
if jid in self.boundjid.bare:
return
if message['type'] in ('chat', 'normal'):
jid = message['from'].bare
# task.clean_tasks_xmpp(self, jid, ['status'])
await task.start_tasks_xmpp(self, jid, ['status'])

29
slixfeed/xmpp/component.py
View file

@ -13,7 +13,7 @@ TODO
import asyncio
import logging
# import os
from random import randrange
# from random import randrange
import slixmpp
import slixfeed.task as task
from time import sleep
@ -31,11 +31,11 @@ import slixfeed.sqlite as sqlite
from slixfeed.xmpp.bookmark import XmppBookmark
from slixfeed.xmpp.connect import XmppConnect
# NOTE MUC is possible for component
from slixfeed.xmpp.muc import XmppGroupchat
# from slixfeed.xmpp.muc import XmppGroupchat
from slixfeed.xmpp.message import XmppMessage
import slixfeed.xmpp.process as process
import slixfeed.xmpp.profile as profile
from slixfeed.xmpp.roster import XmppRoster
# from slixfeed.xmpp.roster import XmppRoster
# import slixfeed.xmpp.service as service
from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.utility import get_chat_type
@ -74,7 +74,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
self.task_manager = {}
# Handlers for ping
self.ping_task_instance = {}
self.task_ping_instance = {}
# Handlers for connection events
self.connection_attempts = 0
@ -170,7 +170,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
# await XmppGroupchat.autojoin(self)
profile.set_identity(self, 'service')
await profile.update(self)
task.ping_task(self)
task.task_ping(self)
# Service.commands(self)
# Service.reactions(self)
@ -179,7 +179,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
self.service_reactions()
async def on_session_resumed(self, event):
def on_session_resumed(self, event):
self.send_presence()
self['xep_0115'].update_caps()
# await XmppGroupchat.autojoin(self)
@ -220,13 +220,13 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
await task.start_tasks_xmpp(self, jid, ['status', 'check'])
async def on_presence_subscribe(self, presence):
def on_presence_subscribe(self, presence):
jid = presence['from'].bare
# XmppPresence.request(self, jid)
XmppPresence.subscription(self, jid, 'subscribe')
async def on_presence_subscribed(self, presence):
def on_presence_subscribed(self, presence):
jid = presence['from'].bare
message_subject = 'RSS News Bot'
message_body = ('Greetings! I am {}, the news anchor.\n'
@ -249,7 +249,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
await task.start_tasks_xmpp(self, jid)
async def on_presence_unsubscribed(self, presence):
def on_presence_unsubscribed(self, presence):
jid = presence['from'].bare
message_body = 'You have been unsubscribed.'
# status_message = '🖋️ Subscribe to receive updates'
@ -260,7 +260,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
# presence_type='unsubscribed')
async def on_presence_unavailable(self, presence):
def on_presence_unavailable(self, presence):
jid = presence['from'].bare
# await task.stop_tasks(self, jid)
task.clean_tasks_xmpp(self, jid)
@ -271,14 +271,14 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
# Check whether JID is in bookmarks or roster
# If roster, remove contact JID into file
# If bookmarks, remove groupchat JID into file
async def on_presence_error(self, presence):
def on_presence_error(self, presence):
print("on_presence_error")
print(presence)
jid = presence["from"].bare
task.clean_tasks_xmpp(self, jid)
async def on_reactions(self, message):
def on_reactions(self, message):
print(message['from'])
print(message['reactions']['values'])
@ -290,11 +290,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP):
await task.start_tasks_xmpp(self, jid, ['status'])
async def on_chatstate_composing(self, message):
def on_chatstate_composing(self, message):
if message['type'] in ('chat', 'normal'):
jid = message['from'].bare
# task.clean_tasks_xmpp(self, jid, ['status'])
status_message='💡 Press "help" for manual, or "info" for information.'
status_message = ('💡 Send "help" for manual, or "info" for '
'information.')
XmppPresence.send(self, jid, status_message)

59
slixfeed/xmpp/message.py
View file

@ -17,6 +17,7 @@ from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.upload import XmppUpload
from slixfeed.xmpp.utility import get_chat_type
import time
import xml.sax.saxutils as saxutils
"""
@ -24,28 +25,6 @@ NOTE
See XEP-0367: Message Attaching
FIXME
ERROR:asyncio:Task exception was never retrieved
future: <Task finished name='Task-3410' coro=<send_update() done, defined at /home/admin/.venv/lib/python3.11/site-packages/slixfeed/task.py:181> exception=ParseError('not well-formed (invalid token): line 1, column 198')>
Traceback (most recent call last):
File "/home/jojo/.venv/lib/python3.11/site-packages/slixfeed/task.py", line 237, in send_update
XmppMessage.send_oob(self, jid, media, chat_type)
File "/home/jojo/.venv/lib/python3.11/site-packages/slixfeed/xmpp/message.py", line 56, in send_oob
message = self.make_message(mto=jid,
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/basexmpp.py", line 517, in make_message
message['html']['body'] = mhtml
~~~~~~~~~~~~~~~^^^^^^^^
File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/xmlstream/stanzabase.py", line 792, in __setitem__
getattr(self, set_method)(value, **kwargs)
File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/plugins/xep_0071/stanza.py", line 38, in set_body
xhtml = ET.fromstring(content)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/xml/etree/ElementTree.py", line 1338, in XML
parser.feed(text)
xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 1, column 198
"""
class XmppMessage:
@ -71,21 +50,29 @@ class XmppMessage:
mnick=self.alias)
# NOTE We might want to add more characters
# def escape_to_xml(raw_string):
# escape_map = {
# '"' : '&quot;',
# "'" : '&apos;'
# }
# return saxutils.escape(raw_string, escape_map)
def send_oob(self, jid, url, chat_type):
try:
html = (
f'<body xmlns="http://www.w3.org/1999/xhtml">'
f'<a href="{url}">{url}</a></body>')
message = self.make_message(mto=jid,
mfrom=self.boundjid.bare,
mbody=url,
mhtml=html,
mtype=chat_type)
message['oob']['url'] = url
message.send()
except:
logging.error('ERROR!')
logging.error(jid, url, chat_type, html)
url = saxutils.escape(url)
# try:
html = (
f'<body xmlns="http://www.w3.org/1999/xhtml">'
f'<a href="{url}">{url}</a></body>')
message = self.make_message(mto=jid,
mfrom=self.boundjid.bare,
mbody=url,
mhtml=html,
mtype=chat_type)
message['oob']['url'] = url
message.send()
# except:
# logging.error('ERROR!')
# logging.error(jid, url, chat_type, html)
# FIXME Solve this function

21
slixfeed/xmpp/process.py
View file

@ -534,8 +534,7 @@ async def message(self, message):
if query:
if len(query) > 3:
db_file = config.get_pathname_to_database(jid_file)
result = await sqlite.search_feeds(db_file, query)
response = action.list_feeds_by_query(query, result)
response = action.list_feeds_by_query(db_file, query)
else:
response = 'Enter at least 4 characters to search'
else:
@ -546,7 +545,7 @@ async def message(self, message):
case 'goodbye':
if message['type'] == 'groupchat':
await XmppGroupchat.leave(self, jid)
await XmppBookmark.remove(self, muc_jid)
await XmppBookmark.remove(self, jid)
else:
response = 'This command is valid in groupchat only.'
XmppMessage.send_reply(self, message, response)
@ -631,7 +630,7 @@ async def message(self, message):
# num = message_text[5:]
# await task.send_update(self, jid, num)
await task.send_update(self, jid)
await task.xmpp_send_update(self, jid)
# task.clean_tasks_xmpp(self, jid, ['interval', 'status'])
# await task.start_tasks_xmpp(self, jid, ['status', 'interval'])
@ -842,8 +841,11 @@ async def message(self, message):
try:
await sqlite.set_enabled_status(db_file, feed_id, 0)
await sqlite.mark_feed_as_read(db_file, feed_id)
response = ('Updates are now disabled for news source {}.'
.format(feed_id))
name = sqlite.get_feed_title(db_file, feed_id)[0]
addr = sqlite.get_feed_url(db_file, feed_id)[0]
response = ('> {}\n'
'Updates are now disabled for news source "{}"'
.format(addr, name))
except:
response = 'No news source with index {}.'.format(feed_id)
XmppMessage.send_reply(self, message, response)
@ -853,8 +855,11 @@ async def message(self, message):
db_file = config.get_pathname_to_database(jid_file)
try:
await sqlite.set_enabled_status(db_file, feed_id, 1)
response = ('Updates are now enabled for news source {}.'
.format(feed_id))
name = sqlite.get_feed_title(db_file, feed_id)[0]
addr = sqlite.get_feed_url(db_file, feed_id)[0]
response = ('> {}\n'
'Updates are now enabled for news source "{}"'
.format(addr, name))
except:
response = 'No news source with index {}.'.format(ix)
XmppMessage.send_reply(self, message, response)

33
slixfeed/xmpp/roster.py
View file

@ -12,22 +12,6 @@ TODO
class XmppRoster:
async def remove(self, jid):
"""
Remove JID to roster.
Parameters
----------
jid : str
Jabber ID.
Returns
-------
None.
"""
self.update_roster(jid, subscription="remove")
async def add(self, jid):
"""
Add JID to roster.
@ -45,5 +29,20 @@ class XmppRoster:
"""
await self.get_roster()
if jid not in self.client_roster.keys():
self.update_roster(jid, subscription="both")
self.update_roster(jid, subscription='both')
def remove(self, jid):
"""
Remove JID from roster.
Parameters
----------
jid : str
Jabber ID.
Returns
-------
None.
"""
self.update_roster(jid, subscription='remove')