From 94348334491738de5d8de3e5758b51b0f2bdc115 Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Sat, 10 Feb 2024 17:53:53 +0000 Subject: [PATCH] WIP: Closer to fix double message. See task.py --- slixfeed/__main__.py | 28 ++---- slixfeed/action.py | 22 ++--- slixfeed/assets/commands.toml | 10 ++- slixfeed/assets/information.toml | 18 ++-- slixfeed/config.py | 4 + slixfeed/crawl.py | 8 ++ slixfeed/fetch.py | 17 +++- slixfeed/sqlite.py | 138 ++++++++++++++++++------------ slixfeed/task.py | 142 ++++++++++++++++++++----------- slixfeed/url.py | 16 ++++ slixfeed/version.py | 4 +- slixfeed/xmpp/client.py | 62 ++++++++++---- slixfeed/xmpp/component.py | 29 ++++--- slixfeed/xmpp/message.py | 59 +++++-------- slixfeed/xmpp/process.py | 21 +++-- slixfeed/xmpp/roster.py | 33 ++++--- 16 files changed, 369 insertions(+), 242 deletions(-) diff --git a/slixfeed/__main__.py b/slixfeed/__main__.py index 51ab358..238ea86 100644 --- a/slixfeed/__main__.py +++ b/slixfeed/__main__.py @@ -8,9 +8,9 @@ FIXME Consequently, it might result in database lock error upon feed removal attempt -TODO +2) Communicate to messages of new contacts (not subscribed and not in roster) -1) SQL prepared statements; +TODO 2) Machine Learning for scrapping Title, Link, Summary and Timstamp; Scrape element (example: Liferea) @@ -22,15 +22,9 @@ TODO Perhaps not, as it would require to check every feed for this setting. Maybe a separate bot; -4) Support categories; +5) OMEMO; -5) XMPP commands; - -6) Bot as service; - -7) OMEMO; - -8) Logging; +6) Logging; https://docs.python.org/3/howto/logging.html 9) Readability @@ -43,12 +37,6 @@ TODO Store 5 upcoming summaries. This would help making the database files smaller. -11) Support protocol Gopher - See project /michael-lazar/pygopherd - See project /gopherball/gb - -12) Support ActivityPub @person@domain (see Tip Of The Day). - 13) Tip Of The Day. Did you know that you can follow you favorite Mastodon feeds by just sending the URL address? @@ -57,21 +45,15 @@ TODO Mastodon, Misskey, Pixelfed, Pleroma, Socialhome, Soapbox. 14) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger - -15) See project /offpunk/offblocklist.py 16) Search messages of government regulated publishers, and promote other sources. Dear reader, we couldn't get news from XYZ as they don't provide RSS feeds. However, you might want to get news from (1) (2) and (3) instead! -17) Make the program portable (directly use the directory assets) -- Thorsten - -18) The operator account will be given reports from the bot about its +17) The operator account will be given reports from the bot about its activities every X minutes. When a suspicious activity is detected, it will be reported immediately. -19) Communicate to messages of new contacts (not subscribed and not in roster) - """ # vars and their meanings: diff --git a/slixfeed/action.py b/slixfeed/action.py index 015f8dd..cae053c 100644 --- a/slixfeed/action.py +++ b/slixfeed/action.py @@ -86,6 +86,7 @@ except ImportError: "Package readability was not found.\n" "Arc90 Lab algorithm is disabled.") + def manual(filename, section=None, command=None): config_dir = config.get_default_config_directory() with open(config_dir + '/' + filename, mode="rb") as commands: @@ -111,7 +112,8 @@ def manual(filename, section=None, command=None): return cmd_list -async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, session=None): +async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, + session=None): if val: # response = ( # 'Updates will be sent every {} minutes.' @@ -123,7 +125,7 @@ async def xmpp_change_interval(self, key, val, jid, jid_file, message=None, sess await sqlite.set_settings_value(db_file, [key, val]) # NOTE Perhaps this should be replaced # by functions clean and start - await task.refresh_task(self, jid, task.send_update, key, val) + await task.refresh_task(self, jid, task.task_send, key, val) response = ('Updates will be sent every {} minutes.' .format(val)) else: @@ -344,17 +346,17 @@ def list_search_results(query, results): return message -def list_feeds_by_query(query, results): +def list_feeds_by_query(db_file, query): + results = sqlite.search_feeds(db_file, query) message = ( - "Feeds containing '{}':\n\n```" - ).format(query) + 'Feeds containing "{}":\n\n```' + .format(query)) for result in results: message += ( - "\nName : {} [{}]" - "\nURL : {}" - "\n" - ).format( - str(result[0]), str(result[1]), str(result[2])) + '\nName : {} [{}]' + '\nURL : {}' + '\n' + .format(str(result[0]), str(result[1]), str(result[2]))) if len(results): message += "\n```\nTotal of {} feeds".format(len(results)) else: diff --git a/slixfeed/assets/commands.toml b/slixfeed/assets/commands.toml index d563f48..03b315e 100644 --- a/slixfeed/assets/commands.toml +++ b/slixfeed/assets/commands.toml @@ -141,15 +141,19 @@ interval Set interval update to every given . """ length = """ -length +length Set maximum length of news item description. (0 for no limit) """ +media = """ +media [off|on] +Attach media (i.e. audio, image, video) to messages when available. +""" quantum = """ quantum Set amount of updates per message by given . """ random = """ -random +random [off|on] Send messages by random order instead of date. """ @@ -190,7 +194,7 @@ Disable bot and stop updates. [preview] read = """ read -Display most recent 5 titles of given . +Display recent 5 titles of given . """ read_num = """ read diff --git a/slixfeed/assets/information.toml b/slixfeed/assets/information.toml index a9ba061..6df51ad 100644 --- a/slixfeed/assets/information.toml +++ b/slixfeed/assets/information.toml @@ -3,10 +3,10 @@ Slixfeed A Syndication bot for the XMPP communication network. -Slixfeed aims to be an easy to use and fully-featured news \ -aggregator bot for XMPP. It provides a convenient access to Blogs, \ -News websites and even Fediverse instances, along with filtering \ -functionality. +Slixfeed is a news broker which aims to be an easy to use and fully-\ +featured news aggregator bot. It provides a convenient access to \ +Blogs, News websites and even Fediverse instances, along with \ +filtering functionality. Slixfeed is primarily designed for XMPP (aka Jabber). \ Visit https://xmpp.org/software/ for more information. @@ -61,7 +61,7 @@ No operator was specified for this instance. platforms = """ Supported platforms: XMPP -Platforms to be added in future: ActivityPub, Briar, Email, IRC, LXMF, Matrix, MQTT, Nostr, Tox. +Platforms to be added in future: ActivityPub, Briar, Email, IRC, LXMF, Matrix, MQTT, Nostr, Session, Tox. For ideal experience, we recommend using XMPP. """ @@ -78,10 +78,15 @@ Protocols to be added in future: Dat, FTP, Gemini, Gopher, IPFS. resources = """ Slixfeed https://gitgud.io/sjehuda/slixfeed + Slixmpp https://slixmpp.readthedocs.io/ + feedparser https://pythonhosted.org/feedparser + +XMPP +https://xmpp.org/about/ """ terms = """ @@ -113,13 +118,14 @@ imattau (atomtopubsub), \ Jaussoin Timothée (Movim, France), \ Justin Karneges (Psi, California), \ Kevin Smith (Swift IM, Wales), \ +Lars Windolf (Liferea, Germany), \ Luis Henrique Mello (SalixOS, Brazil), \ magicfelix, \ Markus Muttilainen (SalixOS), \ Martin (Debian, Germany), \ Mathieu Pasquet (slixmpp, France), \ Maxime Buquet (slixmpp, France), \ -Phillip Watkins (United Kingdom, SalixOS), \ +Phillip Watkins (SalixOS, United Kingdom), \ Pierrick Le Brun (SalixOS, France), \ Raphael Groner (Fedora, Germany), \ Remko Tronçon (Psi , Belgium), \ diff --git a/slixfeed/config.py b/slixfeed/config.py index 33631f9..3a9fdea 100644 --- a/slixfeed/config.py +++ b/slixfeed/config.py @@ -19,6 +19,10 @@ TODO 6) Use TOML https://ruudvanasseldonk.com/2023/01/11/the-yaml-document-from-hell +7) Make the program portable (directly use the directory assets) -- Thorsten + +7.1) Read missing files from base directories or either set error message. + """ import configparser diff --git a/slixfeed/crawl.py b/slixfeed/crawl.py index 05d1921..ce3df7f 100644 --- a/slixfeed/crawl.py +++ b/slixfeed/crawl.py @@ -3,6 +3,14 @@ """ +FIXME + +1) https://wiki.pine64.org + File "/slixfeed/crawl.py", line 178, in feed_mode_guess + address = join_url(url, parted_url.path.split('/')[1] + path) + ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^ + IndexError: list index out of range + TODO 1.1) Attempt to scan more paths: /blog/, /news/ etc., including root / diff --git a/slixfeed/fetch.py b/slixfeed/fetch.py index 8779a6f..9ccab4c 100644 --- a/slixfeed/fetch.py +++ b/slixfeed/fetch.py @@ -21,6 +21,14 @@ TODO 4) Replace sqlite.remove_nonexistent_entries by sqlite.check_entry_exist Same check, just reverse. +5) Support protocol Gopher + See project /michael-lazar/pygopherd + See project /gopherball/gb + +6) Support ActivityPub @person@domain (see Tip Of The Day). + +7) See project /offpunk/offblocklist.py + """ from aiohttp import ClientError, ClientSession, ClientTimeout @@ -41,18 +49,25 @@ except: "BitTorrent is disabled.") +# class FetchDat: # async def dat(): +# class FetchFtp: # async def ftp(): - + +# class FetchGemini: # async def gemini(): +# class FetchGopher: # async def gopher(): +# class FetchHttp: # async def http(): +# class FetchIpfs: # async def ipfs(): + def http_response(url): """ Download response headers. diff --git a/slixfeed/sqlite.py b/slixfeed/sqlite.py index 8601a0f..6515398 100644 --- a/slixfeed/sqlite.py +++ b/slixfeed/sqlite.py @@ -10,6 +10,11 @@ TODO All other functions to receive cursor. 2) Merge function add_metadata into function import_feeds. + +3) SQL prepared statements. + +4) Support categories; + """ from asyncio import Lock @@ -66,6 +71,51 @@ def create_tables(db_file): Path to database file. """ with create_connection(db_file) as conn: + archive_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS archive ( + id INTEGER NOT NULL, + title TEXT NOT NULL, + link TEXT NOT NULL, + enclosure TEXT, + entry_id TEXT NOT NULL, + feed_id INTEGER NOT NULL, + timestamp TEXT, + read INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + categories_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS categories ( + id INTEGER NOT NULL, + name TEXT, + PRIMARY KEY ("id") + ); + """ + ) + entries_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries ( + id INTEGER NOT NULL, + title TEXT NOT NULL, + link TEXT NOT NULL, + enclosure TEXT, + entry_id TEXT NOT NULL, + feed_id INTEGER NOT NULL, + timestamp TEXT, + read INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) feeds_table_sql = ( """ CREATE TABLE IF NOT EXISTS feeds ( @@ -76,18 +126,19 @@ def create_tables(db_file): ); """ ) - feeds_statistics_table_sql = ( + # TODO Rethink! + # Albeit, probably, more expensive, we might want to have feed_id + # as foreign key, as it is with feeds_properties and feeds_state + feeds_categories_table_sql = ( """ - CREATE TABLE IF NOT EXISTS statistics ( + CREATE TABLE IF NOT EXISTS feeds_categories ( id INTEGER NOT NULL, - feed_id INTEGER NOT NULL UNIQUE, - offline INTEGER, - entries INTEGER, - entries INTEGER, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + category_id INTEGER NOT NULL UNIQUE, + feed_id INTEGER, + FOREIGN KEY ("category_id") REFERENCES "categories" ("id") ON UPDATE CASCADE ON DELETE CASCADE, - PRIMARY KEY ("id") + PRIMARY KEY (id) ); """ ) @@ -127,53 +178,32 @@ def create_tables(db_file): ); """ ) + feeds_statistics_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS statistics ( + id INTEGER NOT NULL, + feed_id INTEGER NOT NULL UNIQUE, + offline INTEGER, + entries INTEGER, + entries INTEGER, + FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) # TODO # Consider parameter unique: # entry_id TEXT NOT NULL UNIQUE, # Will eliminate function: # check_entry_exist - entries_table_sql = ( + filters_table_sql = ( """ - CREATE TABLE IF NOT EXISTS entries ( - id INTEGER NOT NULL, - title TEXT NOT NULL, - link TEXT NOT NULL, - enclosure TEXT, - entry_id TEXT NOT NULL, - feed_id INTEGER NOT NULL, - timestamp TEXT, - read INTEGER NOT NULL DEFAULT 0, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY ("id") - ); - """ - ) - archive_table_sql = ( - """ - CREATE TABLE IF NOT EXISTS archive ( - id INTEGER NOT NULL, - title TEXT NOT NULL, - link TEXT NOT NULL, - enclosure TEXT, - entry_id TEXT NOT NULL, - feed_id INTEGER NOT NULL, - timestamp TEXT, - read INTEGER NOT NULL DEFAULT 0, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY ("id") - ); - """ - ) - status_table_sql = ( - """ - CREATE TABLE IF NOT EXISTS status ( + CREATE TABLE IF NOT EXISTS filters ( id INTEGER NOT NULL, key TEXT NOT NULL, - value INTEGER, + value TEXT, PRIMARY KEY ("id") ); """ @@ -188,12 +218,12 @@ def create_tables(db_file): ); """ ) - filters_table_sql = ( + status_table_sql = ( """ - CREATE TABLE IF NOT EXISTS filters ( + CREATE TABLE IF NOT EXISTS status ( id INTEGER NOT NULL, key TEXT NOT NULL, - value TEXT, + value INTEGER, PRIMARY KEY ("id") ); """ @@ -863,17 +893,17 @@ async def archive_entry(db_file, ix): ) -def get_feed_title(db_file, ix): +def get_feed_title(db_file, feed_id): with create_connection(db_file) as conn: cur = conn.cursor() sql = ( """ SELECT name FROM feeds - WHERE id = :ix + WHERE id = :feed_id """ ) - par = (ix,) + par = (feed_id,) title = cur.execute(sql, par).fetchone() return title @@ -1520,7 +1550,7 @@ async def last_entries(db_file, num): return results -async def search_feeds(db_file, query): +def search_feeds(db_file, query): """ Query feeds. diff --git a/slixfeed/task.py b/slixfeed/task.py index 6d73582..53978cd 100644 --- a/slixfeed/task.py +++ b/slixfeed/task.py @@ -5,12 +5,18 @@ FIXME +0) URGENT!!! Place "await asyncio.sleep(next_update_time)" ***inside*** the + task, and not outside as it is now! + + 1) Function check_readiness or event "changed_status" is causing for triple status messages and also false ones that indicate of lack of feeds. TODO +0) Move functions send_status and send_update to module action + 1) Deprecate "add" (see above) and make it interactive. Slixfeed: Do you still want to add this URL to subscription list? See: case _ if message_lowercase.startswith("add"): @@ -70,13 +76,13 @@ loop = asyncio.get_event_loop() # task_ping = asyncio.create_task(ping(self, jid=None)) -def ping_task(self): - # global ping_task_instance +def task_ping(self): + # global task_ping_instance try: - self.ping_task_instance.cancel() + self.task_ping_instance.cancel() except: logging.info('No ping task to cancel.') - self.ping_task_instance = asyncio.create_task(XmppConnect.ping(self)) + self.task_ping_instance = asyncio.create_task(XmppConnect.ping(self)) """ @@ -118,7 +124,7 @@ async def start_tasks_xmpp(self, jid, tasks=None): self.task_manager[jid][task].cancel() except: logging.info('No task {} for JID {} (start_tasks_xmpp)' - .format(task, jid)) + .format(task, jid)) logging.info('Starting tasks {} for JID {}'.format(tasks, jid)) for task in tasks: # print("task:", task) @@ -133,34 +139,8 @@ async def start_tasks_xmpp(self, jid, tasks=None): self.task_manager[jid]['status'] = asyncio.create_task( send_status(self, jid)) case 'interval': - jid_file = jid.replace('/', '_') - db_file = config.get_pathname_to_database(jid_file) - update_interval = await config.get_setting_value(db_file, - 'interval') - update_interval = 60 * int(update_interval) - last_update_time = await sqlite.get_last_update_time(db_file) - if last_update_time: - last_update_time = float(last_update_time) - diff = time.time() - last_update_time - if diff < update_interval: - next_update_time = update_interval - diff - await asyncio.sleep(next_update_time) - - # print("jid :", jid, "\n" - # "time :", time.time(), "\n" - # "last_update_time :", last_update_time, "\n" - # "difference :", diff, "\n" - # "update interval :", update_interval, "\n" - # "next_update_time :", next_update_time, "\n" - # ) - - # elif diff > val: - # next_update_time = val - await sqlite.update_last_update_time(db_file) - else: - await sqlite.set_last_update_time(db_file) self.task_manager[jid]['interval'] = asyncio.create_task( - send_update(self, jid)) + task_send(self, jid)) # for task in self.task_manager[jid].values(): # print("task_manager[jid].values()") # print(self.task_manager[jid].values()) @@ -172,20 +152,57 @@ async def start_tasks_xmpp(self, jid, tasks=None): # await task -def clean_tasks_xmpp(self, jid, tasks=None): - if not tasks: - tasks = ['interval', 'status', 'check'] - logging.info('Stopping tasks {} for JID {}'.format(tasks, jid)) - for task in tasks: - # if self.task_manager[jid][task]: - try: - self.task_manager[jid][task].cancel() - except: - logging.debug('No task {} for JID {} (clean_tasks_xmpp)' - .format(task, jid)) +async def task_send(self, jid): + print("task_send for", jid) + try: + self.task_manager[jid]['interval'].cancel() + except: + logging.info('No task interval for JID {} (start_tasks_xmpp)' + .format(jid)) + jid_file = jid.replace('/', '_') + print(jid_file) + db_file = config.get_pathname_to_database(jid_file) + print(db_file) + update_interval = await config.get_setting_value(db_file, 'interval') + print(update_interval) + update_interval = 60 * int(update_interval) + print(update_interval) + last_update_time = await sqlite.get_last_update_time(db_file) + print(last_update_time) + if last_update_time: + print('if') + last_update_time = float(last_update_time) + diff = time.time() - last_update_time + if diff < update_interval: + next_update_time = update_interval - diff + print('next_update_time') + print(next_update_time) + print(next_update_time/60/60) + await asyncio.sleep(next_update_time) # FIXME! + print('after await sleep') + + # print("jid :", jid, "\n" + # "time :", time.time(), "\n" + # "last_update_time :", last_update_time, "\n" + # "difference :", diff, "\n" + # "update interval :", update_interval, "\n" + # "next_update_time :", next_update_time, "\n" + # ) + + # elif diff > val: + # next_update_time = val + print('await (if)') + await sqlite.update_last_update_time(db_file) + else: + print('await (else)') + await sqlite.set_last_update_time(db_file) + print("await is done for", jid) + await xmpp_send_update(self, jid) + await start_tasks_xmpp(self, jid, ['status']) + await refresh_task(self, jid, task_send, 'interval') -async def send_update(self, jid, num=None): +async def xmpp_send_update(self, jid, num=None): """ Send news items as messages. @@ -196,21 +213,28 @@ async def send_update(self, jid, num=None): num : str, optional Number. The default is None. """ - logging.info('Sending a news update to JID {}'.format(jid)) + print('Sending a news update to JID {}'.format(jid)) jid_file = jid.replace('/', '_') + print(jid_file) db_file = config.get_pathname_to_database(jid_file) + print(db_file) enabled = await config.get_setting_value(db_file, 'enabled') + print(enabled) if enabled: + print('enabled') if not num: num = await config.get_setting_value(db_file, 'quantum') else: num = int(num) - news_digest = [] + print(num) results = await sqlite.get_unread_entries(db_file, num) + print(results) news_digest = '' media = None chat_type = await get_chat_type(self, jid) + print(jid, num, chat_type) for result in results: + print(result) ix = result[0] title_e = result[1] url = result[2] @@ -239,6 +263,7 @@ async def send_update(self, jid, num=None): if media and news_digest: print('SENDING MESSAGE (if media and news_digest)') print(news_digest) + print(media) # Send textual message XmppMessage.send(self, jid, news_digest, chat_type) news_digest = '' @@ -249,12 +274,13 @@ async def send_update(self, jid, num=None): if news_digest: print('SENDING MESSAGE (if news_digest)') print(news_digest) + XmppMessage.send(self, jid, news_digest, chat_type) # TODO Add while loop to assure delivery. # print(await current_time(), ">>> ACT send_message",jid) # NOTE Do we need "if statement"? See NOTE at is_muc. - if chat_type in ('chat', 'groupchat'): - # TODO Provide a choice (with or without images) - XmppMessage.send(self, jid, news_digest, chat_type) + # if chat_type in ('chat', 'groupchat'): + # # TODO Provide a choice (with or without images) + # XmppMessage.send(self, jid, news_digest, chat_type) # See XEP-0367 # if media: # # message = xmpp.Slixfeed.make_message( @@ -266,7 +292,10 @@ async def send_update(self, jid, num=None): # TODO Do not refresh task before # verifying that it was completed. - await refresh_task(self, jid, send_update, 'interval') + + # await start_tasks_xmpp(self, jid, ['status']) + # await refresh_task(self, jid, send_update, 'interval') + # interval = await initdb( # jid, # sqlite.get_settings_value, @@ -292,6 +321,19 @@ async def send_update(self, jid, num=None): # await handle_event() +def clean_tasks_xmpp(self, jid, tasks=None): + if not tasks: + tasks = ['interval', 'status', 'check'] + logging.info('Stopping tasks {} for JID {}'.format(tasks, jid)) + for task in tasks: + # if self.task_manager[jid][task]: + try: + self.task_manager[jid][task].cancel() + except: + logging.debug('No task {} for JID {} (clean_tasks_xmpp)' + .format(task, jid)) + + async def send_status(self, jid): """ Send status message. diff --git a/slixfeed/url.py b/slixfeed/url.py index 4295427..03869eb 100644 --- a/slixfeed/url.py +++ b/slixfeed/url.py @@ -3,6 +3,13 @@ """ +FIXME + +1) Do not handle base64 + https://www.lilithsaintcrow.com/2024/02/love-anonymous/ + data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAeAAQAAAAAQ6M16AAAAAnRSTlMAAHaTzTgAAAFmSURBVBgZ7cEBAQAAAIKg/q92SMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgWE3LAAGyZmPPAAAAAElFTkSuQmCC + https://www.lilithsaintcrow.com/2024/02/love-anonymous//image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAeAAQAAAAAQ6M16AAAAAnRSTlMAAHaTzTgAAAFmSURBVBgZ7cEBAQAAAIKg/q92SMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADgWE3LAAGyZmPPAAAAAElFTkSuQmCC + TODO 1) ActivityPub URL revealer activitypub_to_http. @@ -40,6 +47,7 @@ def get_hostname(url): parted_url = urlsplit(url) return parted_url.netloc + def replace_hostname(url, url_type): """ Replace hostname. @@ -120,6 +128,8 @@ def remove_tracking_parameters(url): url : str URL. """ + if url.startswith('data:') and ';base64,' in url: + return url parted_url = urlsplit(url) protocol = parted_url.scheme hostname = parted_url.netloc @@ -192,6 +202,8 @@ def complete_url(source, link): str URL. """ + if link.startswith('data:') and ';base64,' in link: + return link if link.startswith('www.'): return 'http://' + link parted_link = urlsplit(link) @@ -270,6 +282,8 @@ def join_url(source, link): str URL. """ + if link.startswith('data:') and ';base64,' in link: + return link if link.startswith('www.'): new_link = 'http://' + link elif link.startswith('%20') and link.endswith('%20'): @@ -296,6 +310,8 @@ def trim_url(url): url : str URL. """ + if url.startswith('data:') and ';base64,' in url: + return url parted_url = urlsplit(url) protocol = parted_url.scheme hostname = parted_url.netloc diff --git a/slixfeed/version.py b/slixfeed/version.py index dffadd2..e88b087 100644 --- a/slixfeed/version.py +++ b/slixfeed/version.py @@ -1,2 +1,2 @@ -__version__ = '0.1.2' -__version_info__ = (0, 1, 2) +__version__ = '0.1.3' +__version_info__ = (0, 1, 3) diff --git a/slixfeed/xmpp/client.py b/slixfeed/xmpp/client.py index 6f1aabf..484df5b 100644 --- a/slixfeed/xmpp/client.py +++ b/slixfeed/xmpp/client.py @@ -108,7 +108,7 @@ class Slixfeed(slixmpp.ClientXMPP): self.task_manager = {} # Handlers for ping - self.ping_task_instance = {} + self.task_ping_instance = {} # Handlers for connection events self.connection_attempts = 0 @@ -124,8 +124,8 @@ class Slixfeed(slixmpp.ClientXMPP): self.on_changed_status) self.add_event_handler("presence_available", self.on_presence_available) - self.add_event_handler("presence_unavailable", - self.on_presence_unavailable) + # self.add_event_handler("presence_unavailable", + # self.on_presence_unavailable) self.add_event_handler("chatstate_active", self.on_chatstate_active) self.add_event_handler("chatstate_composing", @@ -218,7 +218,7 @@ class Slixfeed(slixmpp.ClientXMPP): await XmppGroupchat.autojoin(self) profile.set_identity(self, 'client') await profile.update(self) - task.ping_task(self) + task.task_ping(self) # Service.commands(self) # Service.reactions(self) @@ -262,6 +262,8 @@ class Slixfeed(slixmpp.ClientXMPP): async def on_changed_status(self, presence): # await task.check_readiness(self, presence) jid = presence['from'].bare + if jid in self.boundjid.bare: + return if presence['show'] in ('away', 'dnd', 'xa'): task.clean_tasks_xmpp(self, jid, ['interval']) await task.start_tasks_xmpp(self, jid, ['status', 'check']) @@ -270,6 +272,7 @@ class Slixfeed(slixmpp.ClientXMPP): async def on_presence_subscribe(self, presence): jid = presence['from'].bare if not self.client_roster[jid]['to']: + # XmppPresence.subscription(self, jid, 'subscribe') XmppPresence.subscription(self, jid, 'subscribed') await XmppRoster.add(self, jid) status_message = '✒️ Share online status to receive updates' @@ -280,8 +283,9 @@ class Slixfeed(slixmpp.ClientXMPP): 'chat') - async def on_presence_subscribed(self, presence): + def on_presence_subscribed(self, presence): jid = presence['from'].bare + # XmppPresence.subscription(self, jid, 'subscribed') message_subject = 'RSS News Bot' message_body = ('Greetings! I am {}, the news anchor.\n' 'My job is to bring you the latest ' @@ -297,13 +301,18 @@ class Slixfeed(slixmpp.ClientXMPP): # await task.start_tasks(self, presence) # NOTE Already done inside the start-task function jid = presence['from'].bare + if jid in self.boundjid.bare: + return + print('JID available:', jid) # FIXME TODO Find out what is the source responsible for a couple presences with empty message # NOTE This is a temporary solution await asyncio.sleep(10) await task.start_tasks_xmpp(self, jid) + self.add_event_handler("presence_unavailable", + self.on_presence_unavailable) - async def on_presence_unsubscribed(self, presence): + def on_presence_unsubscribed(self, presence): jid = presence['from'].bare message_body = 'You have been unsubscribed.' # status_message = '🖋️ Subscribe to receive updates' @@ -312,64 +321,81 @@ class Slixfeed(slixmpp.ClientXMPP): XmppPresence.subscription(self, jid, 'unsubscribed') # XmppPresence.send(self, jid, status_message, # presence_type='unsubscribed') - await XmppRoster.remove(self, jid) + XmppRoster.remove(self, jid) - async def on_presence_unavailable(self, presence): + def on_presence_unavailable(self, presence): jid = presence['from'].bare + print('JID unavailable:', jid) # await task.stop_tasks(self, jid) task.clean_tasks_xmpp(self, jid) + # NOTE Albeit nice to ~have~ see, this would constantly + # send presence messages to server to no end. + status_message = 'Farewell' + XmppPresence.send(self, jid, status_message, + presence_type='unavailable') + self.del_event_handler("presence_unavailable", + self.on_presence_unavailable) + # TODO # Send message that database will be deleted within 30 days # Check whether JID is in bookmarks or roster # If roster, remove contact JID into file # If bookmarks, remove groupchat JID into file - async def on_presence_error(self, presence): - print("on_presence_error") - print(presence) + def on_presence_error(self, presence): jid = presence["from"].bare + print('JID error:', jid) task.clean_tasks_xmpp(self, jid) - async def on_reactions(self, message): + def on_reactions(self, message): print(message['from']) print(message['reactions']['values']) async def on_chatstate_active(self, message): + jid = message['from'].bare + if jid in self.boundjid.bare: + return if message['type'] in ('chat', 'normal'): - jid = message['from'].bare # task.clean_tasks_xmpp(self, jid, ['status']) await task.start_tasks_xmpp(self, jid, ['status']) - async def on_chatstate_composing(self, message): + def on_chatstate_composing(self, message): if message['type'] in ('chat', 'normal'): jid = message['from'].bare # task.clean_tasks_xmpp(self, jid, ['status']) - status_message='💡 Press "help" for manual, or "info" for information.' + status_message = ('💡 Send "help" for manual, or "info" for ' + 'information.') XmppPresence.send(self, jid, status_message) async def on_chatstate_gone(self, message): + jid = message['from'].bare + if jid in self.boundjid.bare: + return if message['type'] in ('chat', 'normal'): - jid = message['from'].bare # task.clean_tasks_xmpp(self, jid, ['status']) await task.start_tasks_xmpp(self, jid, ['status']) async def on_chatstate_inactive(self, message): + jid = message['from'].bare + if jid in self.boundjid.bare: + return if message['type'] in ('chat', 'normal'): - jid = message['from'].bare # task.clean_tasks_xmpp(self, jid, ['status']) await task.start_tasks_xmpp(self, jid, ['status']) async def on_chatstate_paused(self, message): + jid = message['from'].bare + if jid in self.boundjid.bare: + return if message['type'] in ('chat', 'normal'): - jid = message['from'].bare # task.clean_tasks_xmpp(self, jid, ['status']) await task.start_tasks_xmpp(self, jid, ['status']) diff --git a/slixfeed/xmpp/component.py b/slixfeed/xmpp/component.py index d57d1ea..01565b5 100644 --- a/slixfeed/xmpp/component.py +++ b/slixfeed/xmpp/component.py @@ -13,7 +13,7 @@ TODO import asyncio import logging # import os -from random import randrange +# from random import randrange import slixmpp import slixfeed.task as task from time import sleep @@ -31,11 +31,11 @@ import slixfeed.sqlite as sqlite from slixfeed.xmpp.bookmark import XmppBookmark from slixfeed.xmpp.connect import XmppConnect # NOTE MUC is possible for component -from slixfeed.xmpp.muc import XmppGroupchat +# from slixfeed.xmpp.muc import XmppGroupchat from slixfeed.xmpp.message import XmppMessage import slixfeed.xmpp.process as process import slixfeed.xmpp.profile as profile -from slixfeed.xmpp.roster import XmppRoster +# from slixfeed.xmpp.roster import XmppRoster # import slixfeed.xmpp.service as service from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.utility import get_chat_type @@ -74,7 +74,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): self.task_manager = {} # Handlers for ping - self.ping_task_instance = {} + self.task_ping_instance = {} # Handlers for connection events self.connection_attempts = 0 @@ -170,7 +170,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): # await XmppGroupchat.autojoin(self) profile.set_identity(self, 'service') await profile.update(self) - task.ping_task(self) + task.task_ping(self) # Service.commands(self) # Service.reactions(self) @@ -179,7 +179,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): self.service_reactions() - async def on_session_resumed(self, event): + def on_session_resumed(self, event): self.send_presence() self['xep_0115'].update_caps() # await XmppGroupchat.autojoin(self) @@ -220,13 +220,13 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): await task.start_tasks_xmpp(self, jid, ['status', 'check']) - async def on_presence_subscribe(self, presence): + def on_presence_subscribe(self, presence): jid = presence['from'].bare # XmppPresence.request(self, jid) XmppPresence.subscription(self, jid, 'subscribe') - async def on_presence_subscribed(self, presence): + def on_presence_subscribed(self, presence): jid = presence['from'].bare message_subject = 'RSS News Bot' message_body = ('Greetings! I am {}, the news anchor.\n' @@ -249,7 +249,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): await task.start_tasks_xmpp(self, jid) - async def on_presence_unsubscribed(self, presence): + def on_presence_unsubscribed(self, presence): jid = presence['from'].bare message_body = 'You have been unsubscribed.' # status_message = '🖋️ Subscribe to receive updates' @@ -260,7 +260,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): # presence_type='unsubscribed') - async def on_presence_unavailable(self, presence): + def on_presence_unavailable(self, presence): jid = presence['from'].bare # await task.stop_tasks(self, jid) task.clean_tasks_xmpp(self, jid) @@ -271,14 +271,14 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): # Check whether JID is in bookmarks or roster # If roster, remove contact JID into file # If bookmarks, remove groupchat JID into file - async def on_presence_error(self, presence): + def on_presence_error(self, presence): print("on_presence_error") print(presence) jid = presence["from"].bare task.clean_tasks_xmpp(self, jid) - async def on_reactions(self, message): + def on_reactions(self, message): print(message['from']) print(message['reactions']['values']) @@ -290,11 +290,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): await task.start_tasks_xmpp(self, jid, ['status']) - async def on_chatstate_composing(self, message): + def on_chatstate_composing(self, message): if message['type'] in ('chat', 'normal'): jid = message['from'].bare # task.clean_tasks_xmpp(self, jid, ['status']) - status_message='💡 Press "help" for manual, or "info" for information.' + status_message = ('💡 Send "help" for manual, or "info" for ' + 'information.') XmppPresence.send(self, jid, status_message) diff --git a/slixfeed/xmpp/message.py b/slixfeed/xmpp/message.py index 9ce1833..be94f42 100644 --- a/slixfeed/xmpp/message.py +++ b/slixfeed/xmpp/message.py @@ -17,6 +17,7 @@ from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.upload import XmppUpload from slixfeed.xmpp.utility import get_chat_type import time +import xml.sax.saxutils as saxutils """ @@ -24,28 +25,6 @@ NOTE See XEP-0367: Message Attaching -FIXME - -ERROR:asyncio:Task exception was never retrieved -future: exception=ParseError('not well-formed (invalid token): line 1, column 198')> -Traceback (most recent call last): - File "/home/jojo/.venv/lib/python3.11/site-packages/slixfeed/task.py", line 237, in send_update - XmppMessage.send_oob(self, jid, media, chat_type) - File "/home/jojo/.venv/lib/python3.11/site-packages/slixfeed/xmpp/message.py", line 56, in send_oob - message = self.make_message(mto=jid, - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/basexmpp.py", line 517, in make_message - message['html']['body'] = mhtml - ~~~~~~~~~~~~~~~^^^^^^^^ - File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/xmlstream/stanzabase.py", line 792, in __setitem__ - getattr(self, set_method)(value, **kwargs) - File "/home/jojo/.venv/lib/python3.11/site-packages/slixmpp/plugins/xep_0071/stanza.py", line 38, in set_body - xhtml = ET.fromstring(content) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/lib/python3.11/xml/etree/ElementTree.py", line 1338, in XML - parser.feed(text) -xml.etree.ElementTree.ParseError: not well-formed (invalid token): line 1, column 198 - """ class XmppMessage: @@ -71,21 +50,29 @@ class XmppMessage: mnick=self.alias) + # NOTE We might want to add more characters + # def escape_to_xml(raw_string): + # escape_map = { + # '"' : '"', + # "'" : ''' + # } + # return saxutils.escape(raw_string, escape_map) def send_oob(self, jid, url, chat_type): - try: - html = ( - f'' - f'{url}') - message = self.make_message(mto=jid, - mfrom=self.boundjid.bare, - mbody=url, - mhtml=html, - mtype=chat_type) - message['oob']['url'] = url - message.send() - except: - logging.error('ERROR!') - logging.error(jid, url, chat_type, html) + url = saxutils.escape(url) + # try: + html = ( + f'' + f'{url}') + message = self.make_message(mto=jid, + mfrom=self.boundjid.bare, + mbody=url, + mhtml=html, + mtype=chat_type) + message['oob']['url'] = url + message.send() + # except: + # logging.error('ERROR!') + # logging.error(jid, url, chat_type, html) # FIXME Solve this function diff --git a/slixfeed/xmpp/process.py b/slixfeed/xmpp/process.py index f32caeb..af8a9e5 100644 --- a/slixfeed/xmpp/process.py +++ b/slixfeed/xmpp/process.py @@ -534,8 +534,7 @@ async def message(self, message): if query: if len(query) > 3: db_file = config.get_pathname_to_database(jid_file) - result = await sqlite.search_feeds(db_file, query) - response = action.list_feeds_by_query(query, result) + response = action.list_feeds_by_query(db_file, query) else: response = 'Enter at least 4 characters to search' else: @@ -546,7 +545,7 @@ async def message(self, message): case 'goodbye': if message['type'] == 'groupchat': await XmppGroupchat.leave(self, jid) - await XmppBookmark.remove(self, muc_jid) + await XmppBookmark.remove(self, jid) else: response = 'This command is valid in groupchat only.' XmppMessage.send_reply(self, message, response) @@ -631,7 +630,7 @@ async def message(self, message): # num = message_text[5:] # await task.send_update(self, jid, num) - await task.send_update(self, jid) + await task.xmpp_send_update(self, jid) # task.clean_tasks_xmpp(self, jid, ['interval', 'status']) # await task.start_tasks_xmpp(self, jid, ['status', 'interval']) @@ -842,8 +841,11 @@ async def message(self, message): try: await sqlite.set_enabled_status(db_file, feed_id, 0) await sqlite.mark_feed_as_read(db_file, feed_id) - response = ('Updates are now disabled for news source {}.' - .format(feed_id)) + name = sqlite.get_feed_title(db_file, feed_id)[0] + addr = sqlite.get_feed_url(db_file, feed_id)[0] + response = ('> {}\n' + 'Updates are now disabled for news source "{}"' + .format(addr, name)) except: response = 'No news source with index {}.'.format(feed_id) XmppMessage.send_reply(self, message, response) @@ -853,8 +855,11 @@ async def message(self, message): db_file = config.get_pathname_to_database(jid_file) try: await sqlite.set_enabled_status(db_file, feed_id, 1) - response = ('Updates are now enabled for news source {}.' - .format(feed_id)) + name = sqlite.get_feed_title(db_file, feed_id)[0] + addr = sqlite.get_feed_url(db_file, feed_id)[0] + response = ('> {}\n' + 'Updates are now enabled for news source "{}"' + .format(addr, name)) except: response = 'No news source with index {}.'.format(ix) XmppMessage.send_reply(self, message, response) diff --git a/slixfeed/xmpp/roster.py b/slixfeed/xmpp/roster.py index 532770e..d1b9b84 100644 --- a/slixfeed/xmpp/roster.py +++ b/slixfeed/xmpp/roster.py @@ -12,22 +12,6 @@ TODO class XmppRoster: - async def remove(self, jid): - """ - Remove JID to roster. - - Parameters - ---------- - jid : str - Jabber ID. - - Returns - ------- - None. - """ - self.update_roster(jid, subscription="remove") - - async def add(self, jid): """ Add JID to roster. @@ -45,5 +29,20 @@ class XmppRoster: """ await self.get_roster() if jid not in self.client_roster.keys(): - self.update_roster(jid, subscription="both") + self.update_roster(jid, subscription='both') + + def remove(self, jid): + """ + Remove JID from roster. + + Parameters + ---------- + jid : str + Jabber ID. + + Returns + ------- + None. + """ + self.update_roster(jid, subscription='remove')