Restructure code;

Add more classes and modules;
Restore database maintenance;
Remove JSON support;
Remove Beautiful Soup.
This commit is contained in:
Schimon Jehudah, Adv. 2024-06-13 18:53:53 +03:00
parent 286900af63
commit 64727d207f
31 changed files with 2401 additions and 2851 deletions

View file

@ -40,12 +40,11 @@ keywords = [
# urls = {Homepage = "https://gitgud.io/sjehuda/slixfeed"} # urls = {Homepage = "https://gitgud.io/sjehuda/slixfeed"}
dependencies = [ dependencies = [
"aiohttp", "aiohttp",
"bs4", # "daemonize",
"feedparser", "feedparser",
"lxml", "lxml",
# "pysocks", # "pysocks",
"python-dateutil", "python-dateutil",
"pyyaml",
"requests", "requests",
"slixmpp", "slixmpp",
"tomli", # Python 3.10 "tomli", # Python 3.10

View file

@ -37,7 +37,7 @@ TODO
13) Tip Of The Day. 13) Tip Of The Day.
Did you know that you can follow you favorite Mastodon feeds by just Did you know that you can follow you favorite Mastodon feeds by just
sending the URL address? sending the URL address?
Supported fediverse websites are: Supported ActivityPub (i.e. fediverse) instances are:
Akkoma, Firefish (Calckey), Friendica, HubZilla, Akkoma, Firefish (Calckey), Friendica, HubZilla,
Mastodon, Misskey, Pixelfed, Pleroma, Socialhome, Soapbox. Mastodon, Misskey, Pixelfed, Pleroma, Socialhome, Soapbox.
@ -57,30 +57,20 @@ TODO
# jid = Jabber ID (XMPP) # jid = Jabber ID (XMPP)
# res = response (HTTP) # res = response (HTTP)
from argparse import ArgumentParser from argparse import ArgumentParser
from getpass import getpass
import sys
import configparser
# import filehandler
# from slixfeed.file import get_default_confdir
from getpass import getpass
import logging import logging
import os import sys
# from datetime import date
# import time
# from eliot import start_action, to_file # from eliot import start_action, to_file
# # to_file(open('slixfeed.log', 'w')) # # to_file(open('slixfeed.log', 'w'))
# # with start_action(action_type='set_date()', jid=jid): # # with start_action(action_type='set_date()', jid=jid):
# # with start_action(action_type='message()', msg=msg): # # with start_action(action_type='message()', msg=msg):
#import slixfeed.smtp
#import slixfeed.irc
#import slixfeed.matrix
import slixfeed.config as config import slixfeed.config as config
from slixfeed.log import Logger
from slixfeed.version import __version__ from slixfeed.version import __version__
logger = Logger(__name__)
# import socks # import socks
# import socket # import socket
@ -89,7 +79,7 @@ from slixfeed.version import __version__
def main(): def main():
config_dir = config.get_default_config_directory() config_dir = config.get_default_config_directory()
logging.info('Reading configuration from {}'.format(config_dir)) logger.info('Reading configuration from {}'.format(config_dir))
print('Reading configuration from {}'.format(config_dir)) print('Reading configuration from {}'.format(config_dir))
network_settings = config.get_values('settings.toml', 'network') network_settings = config.get_values('settings.toml', 'network')
print('User agent:', network_settings['user_agent'] or 'Slixfeed/0.1') print('User agent:', network_settings['user_agent'] or 'Slixfeed/0.1')

File diff suppressed because it is too large Load diff

View file

@ -3,14 +3,14 @@ info = """
Slixfeed is a news broker bot for syndicated news which aims to be \ Slixfeed is a news broker bot for syndicated news which aims to be \
an easy to use and fully-featured news aggregating bot. an easy to use and fully-featured news aggregating bot.
Slixfeed provides a convenient access to Blogs, News websites and \ Slixfeed provides a convenient access to Blogs, News sites and \
even Fediverse instances, along with filtering and other privacy \ even Fediverse instances, along with filtering and other privacy \
driven functionalities. driven functionalities.
Slixfeed is designed primarily for the XMPP communication network \ Slixfeed is designed primarily for the XMPP communication network \
(aka Jabber). (aka Jabber).
https://gitgud.io/sjehuda/slixfeed https://git.xmpp-it.net/sch/Slixfeed
""" """
[note] [note]
@ -160,6 +160,15 @@ https://mov.im
Poezio Poezio
https://poez.io https://poez.io
Psi
https://psi-im.org
Psi+
https://psi-plus.com
Profanity
https://profanity-im.github.io
""" """
[services] [services]

View file

@ -6,6 +6,7 @@ archive = 50 # Maximum items to archive (0 - 500)
check = 120 # Source check interval (recommended 90; minimum 10) check = 120 # Source check interval (recommended 90; minimum 10)
enabled = 1 # Work status (Value 0 to disable) enabled = 1 # Work status (Value 0 to disable)
filter = 0 # Enable filters (Value 1 to enable) filter = 0 # Enable filters (Value 1 to enable)
finished = 0 # Send an extra message which indicates of the amount of time of a done task (Value 1 to enable)
interval = 300 # Update interval (Minimum value 10) interval = 300 # Update interval (Minimum value 10)
length = 300 # Maximum length of summary (Value 0 to disable) length = 300 # Maximum length of summary (Value 0 to disable)
media = 0 # Display media (audio, image, video) when available media = 0 # Display media (audio, image, video) when available
@ -20,13 +21,16 @@ random = 0 # Pick random item from database
# * feed_title = Title of news source # * feed_title = Title of news source
# * ix = Index of item # * ix = Index of item
formatting = """ formatting = """
{title} {ix}. {title}
> {summary} > {summary}
{link} {link}
{feed_title} [{ix}] {feed_title} [{feed_id}]
""" """
[ipc]
bsd = 0 # IPC (BSD/UDS) POSIX sockets
# Utilized in case of missing protocol support. # Utilized in case of missing protocol support.
[bridge] [bridge]
gopher = "" gopher = ""
@ -37,8 +41,8 @@ tor = ""
yggdrasil = "" yggdrasil = ""
[network] [network]
http_proxy = "" http_proxy = "" # Example: http://127.0.0.1:8118
user_agent = "Slixfeed/0.1" user_agent = "Slixfeed/0.1" # Default Slixfeed/0.1
clearnet = 0 # Enable policed DNS system (not recommended) clearnet = 0 # Enable policed DNS system (not recommended)
i2p = 1 # Enable I2P mixnet system (safer) i2p = 1 # Enable I2P mixnet system (safer)
ipfs = 1 # Enable IPFS DHT system (safer) ipfs = 1 # Enable IPFS DHT system (safer)

39
slixfeed/bittorrent.py Normal file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import slixfeed.fetch as fetch
from slixfeed.log import Logger
import sys
from urllib.parse import parse_qs, urlsplit
logger = Logger(__name__)
class BitTorrent:
# TODO Add support for eDonkey, Gnutella, Soulseek
async def get_magnet(link):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: {}'.format(function_name, link))
parted_link = urlsplit(link)
queries = parse_qs(parted_link.query)
query_xt = queries["xt"][0]
if query_xt.startswith("urn:btih:"):
filename = queries["dn"][0]
checksum = query_xt[len("urn:btih:"):]
torrent = await fetch.magnet(link)
logger.debug('Attempting to retrieve {} ({})'
.format(filename, checksum))
if not torrent:
logger.debug('Attempting to retrieve {} from HTTP caching service'
.format(filename))
urls = [
'https://watercache.libertycorp.org/get/{}/{}',
'https://itorrents.org/torrent/{}.torrent?title={}',
'https://firecache.libertycorp.org/get/{}/{}',
'http://fcache63sakpihd44kxdduy6kgpdhgejgp323wci435zwy6kiylcnfad.onion/get/{}/{}'
]
for url in urls:
torrent = fetch.http(url.format(checksum, filename))
if torrent:
break
return torrent

View file

@ -11,9 +11,9 @@ FIXME
TODO TODO
1) Website-specific filter (i.e. audiobookbay). 1) Site-specific filter (i.e. audiobookbay).
2) Exclude websites from being subjected to filtering (e.g. metapedia). 2) Exclude sites from being subjected to filtering (e.g. metapedia).
3) Filter phrases: 3) Filter phrases:
Refer to sqlitehandler.search_entries for implementation. Refer to sqlitehandler.search_entries for implementation.
@ -32,7 +32,7 @@ TODO
""" """
import configparser import configparser
import logging from slixfeed.log import Logger
import os import os
# from random import randrange # from random import randrange
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
@ -43,6 +43,8 @@ try:
except: except:
import tomli as tomllib import tomli as tomllib
logger = Logger(__name__)
# TODO Consider a class ConfigDefault for default values to be initiate at most # TODO Consider a class ConfigDefault for default values to be initiate at most
# basic level possible and a class ConfigJID for each JID (i.e. db_file) to be # basic level possible and a class ConfigJID for each JID (i.e. db_file) to be
# also initiated at same level or at least at event call, then check whether # also initiated at same level or at least at event call, then check whether
@ -75,6 +77,8 @@ class Config:
else: else:
await sqlite.set_setting_value(db_file, key_val) await sqlite.set_setting_value(db_file, key_val)
# TODO Segregate Jabber ID settings from Slixfeed wide settings.
# self.settings, self.settings_xmpp, self.settings_irc etc.
def get_setting_value(settings, jid_bare, key): def get_setting_value(settings, jid_bare, key):
if jid_bare in settings and key in settings[jid_bare]: if jid_bare in settings and key in settings[jid_bare]:
value = settings[jid_bare][key] value = settings[jid_bare][key]
@ -248,21 +252,21 @@ def get_value(filename, section, keys):
for key in keys: for key in keys:
if key in section_res: if key in section_res:
value = section_res[key] value = section_res[key]
logging.debug("Found value {} for key {}".format(value, key)) logger.debug("Found value {} for key {}".format(value, key))
else: else:
value = '' value = ''
logging.debug("Missing key:", key) logger.debug("Missing key:", key)
result.extend([value]) result.extend([value])
elif isinstance(keys, str): elif isinstance(keys, str):
key = keys key = keys
if key in section_res: if key in section_res:
result = section_res[key] result = section_res[key]
logging.debug("Found value {} for key {}".format(result, key)) logger.debug("Found value {} for key {}".format(result, key))
else: else:
result = '' result = ''
# logging.error("Missing key:", key) # logger.error("Missing key:", key)
if result == None: if result == None:
logging.error( logger.error(
"Check configuration file {}.ini for " "Check configuration file {}.ini for "
"missing key(s) \"{}\" under section [{}].".format( "missing key(s) \"{}\" under section [{}].".format(
filename, keys, section) filename, keys, section)

View file

@ -39,37 +39,39 @@ NOTE
from aiohttp import ClientError, ClientSession, ClientTimeout from aiohttp import ClientError, ClientSession, ClientTimeout
from asyncio import TimeoutError from asyncio import TimeoutError
# from asyncio.exceptions import IncompleteReadError # from asyncio.exceptions import IncompleteReadError
# from bs4 import BeautifulSoup
# from http.client import IncompleteRead # from http.client import IncompleteRead
import logging
# from lxml import html # from lxml import html
# from xml.etree.ElementTree import ElementTree, ParseError # from xml.etree.ElementTree import ElementTree, ParseError
import requests import requests
import slixfeed.config as config import slixfeed.config as config
from slixfeed.log import Logger
logger = Logger(__name__)
try: try:
from magnet2torrent import Magnet2Torrent, FailedToFetchException from magnet2torrent import Magnet2Torrent, FailedToFetchException
except: except:
logging.info( logger.info(
"Package magnet2torrent was not found.\n" "Package magnet2torrent was not found.\n"
"BitTorrent is disabled.") "BitTorrent is disabled.")
# class FetchDat: # class Dat:
# async def dat(): # async def dat():
# class FetchFtp: # class Ftp:
# async def ftp(): # async def ftp():
# class FetchGemini: # class Gemini:
# async def gemini(): # async def gemini():
# class FetchGopher: # class Gopher:
# async def gopher(): # async def gopher():
# class FetchHttp: # class Http:
# async def http(): # async def http():
# class FetchIpfs: # class Ipfs:
# async def ipfs(): # async def ipfs():
@ -103,12 +105,13 @@ def http_response(url):
"User-Agent": user_agent "User-Agent": user_agent
} }
try: try:
# Don't use HEAD request because quite a few websites may deny it # Do not use HEAD request because it appears that too many sites would
# deny it.
# response = requests.head(url, headers=headers, allow_redirects=True) # response = requests.head(url, headers=headers, allow_redirects=True)
response = requests.get(url, headers=headers, allow_redirects=True) response = requests.get(url, headers=headers, allow_redirects=True)
except Exception as e: except Exception as e:
logging.warning('Error in HTTP response') logger.warning('Error in HTTP response')
logging.error(e) logger.error(e)
response = None response = None
return response return response
@ -175,7 +178,7 @@ async def http(url):
'original_url': url, 'original_url': url,
'status_code': None} 'status_code': None}
except Exception as e: except Exception as e:
logging.error(e) logger.error(e)
result = {'error': True, result = {'error': True,
'message': 'Error:' + str(e) if e else 'Error', 'message': 'Error:' + str(e) if e else 'Error',
'original_url': url, 'original_url': url,
@ -188,4 +191,4 @@ async def magnet(link):
try: try:
filename, torrent_data = await m2t.retrieve_torrent() filename, torrent_data = await m2t.retrieve_torrent()
except FailedToFetchException: except FailedToFetchException:
logging.debug("Failed") logger.debug("Failed")

View file

@ -13,8 +13,10 @@ logger.debug('This is a debug message')
import logging import logging
class Logger: class Logger:
def __init__(self, name): def __init__(self, name):
self.logger = logging.getLogger(name) self.logger = logging.getLogger(name)
self.logger.setLevel(logging.WARNING) # DEBUG self.logger.setLevel(logging.WARNING) # DEBUG

View file

@ -1,65 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from slixfeed.log import Logger
import slixfeed.dt as dt
import slixfeed.sqlite as sqlite
import sys
import xml.etree.ElementTree as ETR
logger = Logger(__name__)
class Opml:
# TODO Consider adding element jid as a pointer of import
def export_to_file(jid, filename, results):
# print(jid, filename, results)
function_name = sys._getframe().f_code.co_name
logger.debug('{} jid: {} filename: {}'
.format(function_name, jid, filename))
root = ETR.Element("opml")
root.set("version", "1.0")
head = ETR.SubElement(root, "head")
ETR.SubElement(head, "title").text = "{}".format(jid)
ETR.SubElement(head, "description").text = (
"Set of subscriptions exported by Slixfeed")
ETR.SubElement(head, "generator").text = "Slixfeed"
ETR.SubElement(head, "urlPublic").text = (
"https://gitgud.io/sjehuda/slixfeed")
time_stamp = dt.current_time()
ETR.SubElement(head, "dateCreated").text = time_stamp
ETR.SubElement(head, "dateModified").text = time_stamp
body = ETR.SubElement(root, "body")
for result in results:
outline = ETR.SubElement(body, "outline")
outline.set("text", result[1])
outline.set("xmlUrl", result[2])
# outline.set("type", result[2])
tree = ETR.ElementTree(root)
tree.write(filename)
async def import_from_file(db_file, result):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {}'
.format(function_name, db_file))
if not result['error']:
document = result['content']
root = ETR.fromstring(document)
before = sqlite.get_number_of_items(db_file, 'feeds_properties')
feeds = []
for child in root.findall(".//outline"):
url = child.get("xmlUrl")
title = child.get("text")
# feed = (url, title)
# feeds.extend([feed])
feed = {
'title' : title,
'url' : url,
}
feeds.extend([feed])
await sqlite.import_feeds(db_file, feeds)
await sqlite.add_metadata(db_file)
after = sqlite.get_number_of_items(db_file, 'feeds_properties')
difference = int(after) - int(before)
return difference

View file

@ -1558,29 +1558,38 @@ def is_entry_archived(cur, ix):
result = cur.execute(sql, par).fetchone() result = cur.execute(sql, par).fetchone()
return result return result
async def mark_entry_as_read(cur, ix):
def is_entry_read(db_file, ix):
""" """
Set read status of entry as read. Check whether a given entry is marked as read.
Parameters Parameters
---------- ----------
cur : object db_file : str
Cursor object. Path to database file.
ix : str ix : str
Index of entry. Index of entry.
Returns
-------
result : tuple
Entry ID.
""" """
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: ix: {}' logger.debug('{}: ix: {}'
.format(function_name, ix)) .format(function_name, ix))
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = ( sql = (
""" """
UPDATE entries_state SELECT read
SET read = 1 FROM entries_state
WHERE entry_id = ? WHERE entry_id = ?
""" """
) )
par = (ix,) par = (ix,)
cur.execute(sql, par) result = cur.execute(sql, par).fetchone()
return result
def get_last_update_time_of_feed(db_file, feed_id): def get_last_update_time_of_feed(db_file, feed_id):
@ -1669,45 +1678,6 @@ def get_number_of_unread_entries_by_feed(db_file, feed_id):
return count return count
async def mark_feed_as_read(db_file, feed_id):
"""
Set read status of entries of given feed as read.
Parameters
----------
db_file : str
Path to database file.
feed_id : str
Feed ID.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} feed_id: {}'
.format(function_name, db_file, feed_id))
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = (
"""
SELECT id
FROM entries_properties
WHERE feed_id = ?
"""
)
par = (feed_id,)
ixs = cur.execute(sql, par).fetchall()
sql = (
"""
UPDATE entries_state
SET read = 1
WHERE entry_id = ?
"""
)
for ix in ixs: cur.execute(sql, ix)
# for ix in ixs:
# par = ix # Variable ix is already of type tuple
# cur.execute(sql, par)
async def delete_entry_by_id(db_file, ix): async def delete_entry_by_id(db_file, ix):
""" """
Delete entry by Id. Delete entry by Id.
@ -1923,26 +1893,6 @@ def get_feed_url(db_file, feed_id):
return url return url
async def mark_as_read(db_file, ix):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} ix: {}'
.format(function_name, db_file, ix))
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
# TODO While `async with DBLOCK` does work well from
# outside of functions, it would be better practice
# to place it within the functions.
# NOTE: We can use DBLOCK once for both
# functions, because, due to exclusive
# ID, only one can ever occur.
if is_entry_archived(cur, ix):
await delete_entry(cur, ix)
else:
await mark_entry_as_read(cur, ix)
async def mark_all_as_read(db_file): async def mark_all_as_read(db_file):
""" """
Set read status of all entries as read. Set read status of all entries as read.
@ -1985,6 +1935,89 @@ async def mark_all_as_read(db_file):
for ix in ixs: cur.execute(sql, ix) for ix in ixs: cur.execute(sql, ix)
async def mark_feed_as_read(db_file, feed_id):
"""
Set read status of entries of given feed as read.
Parameters
----------
db_file : str
Path to database file.
feed_id : str
Feed ID.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} feed_id: {}'
.format(function_name, db_file, feed_id))
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = (
"""
SELECT id
FROM entries_properties
WHERE feed_id = ?
"""
)
par = (feed_id,)
ixs = cur.execute(sql, par).fetchall()
sql = (
"""
UPDATE entries_state
SET read = 1
WHERE entry_id = ?
"""
)
for ix in ixs: cur.execute(sql, ix)
# for ix in ixs:
# par = ix # Variable ix is already of type tuple
# cur.execute(sql, par)
async def mark_entry_as_read(cur, ix):
"""
Set read status of entry as read.
Parameters
----------
cur : object
Cursor object.
ix : str
Index of entry.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: ix: {}'
.format(function_name, ix))
sql = (
"""
UPDATE entries_state
SET read = 1
WHERE entry_id = ?
"""
)
par = (ix,)
cur.execute(sql, par)
async def mark_as_read(db_file, ix):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} ix: {}'
.format(function_name, db_file, ix))
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
# TODO While `async with DBLOCK` does work well from
# outside of functions, it would be better practice
# to place it within the functions.
# NOTE: We can use DBLOCK once for both
# functions, because, due to exclusive
# ID, only one can ever occur.
if is_entry_archived(cur, ix):
await delete_entry(cur, ix)
else:
await mark_entry_as_read(cur, ix)
async def delete_entry(cur, ix): async def delete_entry(cur, ix):
""" """
Delete entry. Delete entry.

958
slixfeed/syndication.py Normal file
View file

@ -0,0 +1,958 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
TODO
1) Function scan at "for entry in entries"
Suppress directly calling function "add_entry" (accept db_file)
Pass a list of valid entries to a new function "add_entries"
(accept db_file) which would call function "add_entry" (accept cur).
* accelerate adding of large set of entries at once.
* prevent (or mitigate halt of consequent actions).
* reduce I/O.
2) Call sqlite function from function statistics.
Returning a list of values doesn't' seem to be a good practice.
3) Special statistics for operator:
* Size of database(s);
* Amount of JIDs subscribed;
* Amount of feeds of all JIDs;
* Amount of entries of all JIDs.
"""
import asyncio
from feedparser import parse
import os
import slixfeed.config as config
from slixfeed.config import Config
import slixfeed.crawl as crawl
import slixfeed.dt as dt
import slixfeed.fetch as fetch
from slixfeed.log import Logger
import slixfeed.sqlite as sqlite
from slixfeed.url import join_url, trim_url
from slixfeed.utilities import Html, MD, SQLiteMaintain
from slixmpp.xmlstream import ET
import sys
from urllib.parse import urlsplit
import xml.etree.ElementTree as ETR
logger = Logger(__name__)
class Feed:
# NOTE Consider removal of MD (and any other option HTML and XBEL)
def export_feeds(jid_bare, ext):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_bare: {}: ext: {}'.format(function_name, jid_bare, ext))
cache_dir = config.get_default_cache_directory()
if not os.path.isdir(cache_dir):
os.mkdir(cache_dir)
if not os.path.isdir(cache_dir + '/' + ext):
os.mkdir(cache_dir + '/' + ext)
filename = os.path.join(
cache_dir, ext, 'slixfeed_' + dt.timestamp() + '.' + ext)
db_file = config.get_pathname_to_database(jid_bare)
results = sqlite.get_feeds(db_file)
match ext:
# case 'html':
# response = 'Not yet implemented.'
case 'md':
MD.export_to_markdown(jid_bare, filename, results)
case 'opml':
Opml.export_to_file(jid_bare, filename, results)
# case 'xbel':
# response = 'Not yet implemented.'
return filename
def pack_entry_into_dict(db_file, entry):
entry_id = entry[0]
authors = sqlite.get_authors_by_entry_id(db_file, entry_id)
entry_authors = []
for author in authors:
entry_author = {
'name': author[2],
'email': author[3],
'url': author[4]}
entry_authors.extend([entry_author])
contributors = sqlite.get_contributors_by_entry_id(db_file, entry_id)
entry_contributors = []
for contributor in contributors:
entry_contributor = {
'name': contributor[2],
'email': contributor[3],
'url': contributor[4]}
entry_contributors.extend([entry_contributor])
links = sqlite.get_links_by_entry_id(db_file, entry_id)
entry_links = []
for link in links:
entry_link = {
'url': link[2],
'type': link[3],
'rel': link[4],
'size': link[5]}
entry_links.extend([entry_link])
tags = sqlite.get_tags_by_entry_id(db_file, entry_id)
entry_tags = []
for tag in tags:
entry_tag = {
'term': tag[2],
'scheme': tag[3],
'label': tag[4]}
entry_tags.extend([entry_tag])
contents = sqlite.get_contents_by_entry_id(db_file, entry_id)
entry_contents = []
for content in contents:
entry_content = {
'text': content[2],
'type': content[3],
'base': content[4],
'lang': content[5]}
entry_contents.extend([entry_content])
feed_entry = {
'authors' : entry_authors,
'category' : entry[10],
'comments' : entry[12],
'contents' : entry_contents,
'contributors' : entry_contributors,
'summary_base' : entry[9],
'summary_lang' : entry[7],
'summary_text' : entry[6],
'summary_type' : entry[8],
'enclosures' : entry[13],
'href' : entry[11],
'link' : entry[3],
'links' : entry_links,
'published' : entry[14],
'rating' : entry[13],
'tags' : entry_tags,
'title' : entry[4],
'title_type' : entry[3],
'updated' : entry[15]}
return feed_entry
def create_rfc4287_entry(feed_entry):
node_entry = ET.Element('entry')
node_entry.set('xmlns', 'http://www.w3.org/2005/Atom')
# Title
title = ET.SubElement(node_entry, 'title')
if feed_entry['title']:
if feed_entry['title_type']: title.set('type', feed_entry['title_type'])
title.text = feed_entry['title']
elif feed_entry['summary_text']:
if feed_entry['summary_type']: title.set('type', feed_entry['summary_type'])
title.text = feed_entry['summary_text']
# if feed_entry['summary_base']: title.set('base', feed_entry['summary_base'])
# if feed_entry['summary_lang']: title.set('lang', feed_entry['summary_lang'])
else:
title.text = feed_entry['published']
# Some feeds have identical content for contents and summary
# So if content is present, do not add summary
if feed_entry['contents']:
# Content
for feed_entry_content in feed_entry['contents']:
content = ET.SubElement(node_entry, 'content')
# if feed_entry_content['base']: content.set('base', feed_entry_content['base'])
if feed_entry_content['lang']: content.set('lang', feed_entry_content['lang'])
if feed_entry_content['type']: content.set('type', feed_entry_content['type'])
content.text = feed_entry_content['text']
else:
# Summary
summary = ET.SubElement(node_entry, 'summary') # TODO Try 'content'
# if feed_entry['summary_base']: summary.set('base', feed_entry['summary_base'])
# TODO Check realization of "lang"
if feed_entry['summary_type']: summary.set('type', feed_entry['summary_type'])
if feed_entry['summary_lang']: summary.set('lang', feed_entry['summary_lang'])
summary.text = feed_entry['summary_text']
# Authors
for feed_entry_author in feed_entry['authors']:
author = ET.SubElement(node_entry, 'author')
name = ET.SubElement(author, 'name')
name.text = feed_entry_author['name']
if feed_entry_author['url']:
uri = ET.SubElement(author, 'uri')
uri.text = feed_entry_author['url']
if feed_entry_author['email']:
email = ET.SubElement(author, 'email')
email.text = feed_entry_author['email']
# Contributors
for feed_entry_contributor in feed_entry['contributors']:
contributor = ET.SubElement(node_entry, 'author')
name = ET.SubElement(contributor, 'name')
name.text = feed_entry_contributor['name']
if feed_entry_contributor['url']:
uri = ET.SubElement(contributor, 'uri')
uri.text = feed_entry_contributor['url']
if feed_entry_contributor['email']:
email = ET.SubElement(contributor, 'email')
email.text = feed_entry_contributor['email']
# Category
category = ET.SubElement(node_entry, "category")
category.set('category', feed_entry['category'])
# Tags
for feed_entry_tag in feed_entry['tags']:
tag = ET.SubElement(node_entry, 'category')
tag.set('term', feed_entry_tag['term'])
# Link
link = ET.SubElement(node_entry, "link")
link.set('href', feed_entry['link'])
# Links
for feed_entry_link in feed_entry['links']:
link = ET.SubElement(node_entry, "link")
link.set('href', feed_entry_link['url'])
link.set('type', feed_entry_link['type'])
link.set('rel', feed_entry_link['rel'])
# Date updated
if feed_entry['updated']:
updated = ET.SubElement(node_entry, 'updated')
updated.text = feed_entry['updated']
# Date published
if feed_entry['published']:
published = ET.SubElement(node_entry, 'published')
published.text = feed_entry['published']
return node_entry
def is_feed(url, feed):
"""
Determine whether document is feed or not.
Parameters
----------
feed : dict
Parsed feed.
Returns
-------
val : boolean
True or False.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}'.format(function_name))
value = False
# message = None
if not feed.entries:
if "version" in feed.keys():
# feed["version"]
if feed.version:
value = True
# message = (
# "Empty feed for {}"
# ).format(url)
elif "title" in feed["feed"].keys():
value = True
# message = (
# "Empty feed for {}"
# ).format(url)
else:
value = False
# message = (
# "No entries nor title for {}"
# ).format(url)
elif feed.bozo:
# NOTE Consider valid even when is not-well-formed
value = True
logger.warning('Bozo detected for {}'.format(url))
else:
value = True
# message = (
# "Good feed for {}"
# ).format(url)
return value
async def add_feed(self, jid_bare, db_file, url, identifier):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} url: {}'
.format(function_name, db_file, url))
while True:
feed_id = sqlite.get_feed_id(db_file, url)
if not feed_id:
exist_identifier = sqlite.check_identifier_exist(db_file, identifier)
if not exist_identifier:
result = await fetch.http(url)
message = result['message']
status_code = result['status_code']
if not result['error']:
await sqlite.update_feed_status(db_file, feed_id, status_code)
document = result['content']
feed = parse(document)
# if document and status_code == 200:
if Feed.is_feed(url, feed):
if "title" in feed["feed"].keys():
title = feed["feed"]["title"]
else:
title = urlsplit(url).netloc
if "language" in feed["feed"].keys():
language = feed["feed"]["language"]
else:
language = ''
if "encoding" in feed.keys():
encoding = feed["encoding"]
else:
encoding = ''
if "updated_parsed" in feed["feed"].keys():
updated = feed["feed"]["updated_parsed"]
try:
updated = dt.convert_struct_time_to_iso8601(updated)
except Exception as e:
logger.error(str(e))
updated = ''
else:
updated = ''
version = feed.version
entries_count = len(feed.entries)
await sqlite.insert_feed(db_file,
url,
title,
identifier,
entries=entries_count,
version=version,
encoding=encoding,
language=language,
status_code=status_code,
updated=updated)
feed_valid = 0 if feed.bozo else 1
await sqlite.update_feed_validity(
db_file, feed_id, feed_valid)
if feed.has_key('updated_parsed'):
feed_updated = feed.updated_parsed
try:
feed_updated = dt.convert_struct_time_to_iso8601(feed_updated)
except Exception as e:
logger.error(str(e))
feed_updated = None
else:
feed_updated = None
feed_properties = Feed.get_properties_of_feed(
db_file, feed_id, feed)
await sqlite.update_feed_properties(
db_file, feed_id, feed_properties)
feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
new_entries = Feed.get_properties_of_entries(
jid_bare, db_file, url, feed_id, feed)
if new_entries:
await sqlite.add_entries_and_update_feed_state(
db_file, feed_id, new_entries)
old = Config.get_setting_value(self.settings, jid_bare, 'old')
if not old: await sqlite.mark_feed_as_read(db_file, feed_id)
result_final = {'link' : url,
'index' : feed_id,
'name' : title,
'code' : status_code,
'error' : False,
'message': message,
'exist' : False,
'identifier' : None}
break
else:
# NOTE Do not be tempted to return a compact dictionary.
# That is, dictionary within dictionary
# Return multiple dictionaries in a list or tuple.
result = await crawl.probe_page(url, document)
if not result:
# Get out of the loop with dict indicating error.
result_final = {'link' : url,
'index' : None,
'name' : None,
'code' : status_code,
'error' : True,
'message': message,
'exist' : False,
'identifier' : None}
break
elif isinstance(result, list):
# Get out of the loop and deliver a list of dicts.
result_final = result
break
else:
# Go back up to the while loop and try again.
url = result['link']
else:
await sqlite.update_feed_status(db_file, feed_id, status_code)
result_final = {'link' : url,
'index' : None,
'name' : None,
'code' : status_code,
'error' : True,
'message': message,
'exist' : False,
'identifier' : None}
break
else:
ix = exist_identifier[1]
identifier = exist_identifier[2]
message = ('Identifier "{}" is already allocated.'
.format(identifier))
result_final = {'link' : url,
'index' : ix,
'name' : None,
'code' : None,
'error' : False,
'message': message,
'exist' : False,
'identifier' : identifier}
break
else:
feed_id = feed_id[0]
title = sqlite.get_feed_title(db_file, feed_id)
title = title[0]
message = 'URL already exist.'
result_final = {'link' : url,
'index' : feed_id,
'name' : title,
'code' : None,
'error' : False,
'message': message,
'exist' : True,
'identifier' : None}
break
return result_final
def view_feed(url, feed):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: url: {}'
.format(function_name, url))
if "title" in feed["feed"].keys():
title = feed["feed"]["title"]
else:
title = urlsplit(url).netloc
entries = feed.entries
response = "Preview of {}:\n\n```\n".format(title)
counter = 0
for entry in entries:
counter += 1
if entry.has_key("title"):
title = entry.title
else:
title = "*** No title ***"
if entry.has_key("link"):
# link = complete_url(source, entry.link)
link = join_url(url, entry.link)
link = trim_url(link)
else:
link = "*** No link ***"
if entry.has_key("published"):
date = entry.published
date = dt.rfc2822_to_iso8601(date)
elif entry.has_key("updated"):
date = entry.updated
date = dt.rfc2822_to_iso8601(date)
else:
date = "*** No date ***"
response += ("Title : {}\n"
"Date : {}\n"
"Link : {}\n"
"Count : {}\n"
"\n"
.format(title, date, link, counter))
if counter > 4:
break
response += (
"```\nSource: {}"
).format(url)
return response
def view_entry(url, feed, num):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: url: {} num: {}'
.format(function_name, url, num))
if "title" in feed["feed"].keys():
title = feed["feed"]["title"]
else:
title = urlsplit(url).netloc
entries = feed.entries
num = int(num) - 1
entry = entries[num]
response = "Preview of {}:\n\n```\n".format(title)
if entry.has_key("title"):
title = entry.title
else:
title = '*** No title ***'
if entry.has_key("published"):
date = entry.published
date = dt.rfc2822_to_iso8601(date)
elif entry.has_key("updated"):
date = entry.updated
date = dt.rfc2822_to_iso8601(date)
else:
date = '*** No date ***'
if entry.has_key("summary"):
summary = entry.summary
# Remove HTML tags
if summary:
summary = Html.remove_html_tags(summary)
# TODO Limit text length
summary = summary.replace("\n\n\n", "\n\n")
else:
summary = '*** No summary ***'
else:
summary = '*** No summary ***'
if entry.has_key("link"):
# link = complete_url(source, entry.link)
link = join_url(url, entry.link)
link = trim_url(link)
else:
link = '*** No link ***'
response = ("{}\n"
"\n"
# "> {}\n"
"{}\n"
"\n"
"{}\n"
"\n"
.format(title, summary, link))
return response
# NOTE This function is not being utilized
async def download_feed(self, db_file, feed_url):
"""
Get feed content.
Parameters
----------
db_file : str
Path to database file.
url : str, optional
URL.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} url: {}'
.format(function_name, db_file, feed_url))
if isinstance(feed_url, tuple): feed_url = feed_url[0]
result = await fetch.http(feed_url)
feed_id = sqlite.get_feed_id(db_file, feed_url)
feed_id = feed_id[0]
status_code = result['status_code']
await sqlite.update_feed_status(db_file, feed_id, status_code)
def get_properties_of_feed(db_file, feed_id, feed):
if feed.has_key('updated_parsed'):
feed_updated = feed.updated_parsed
try:
feed_updated = dt.convert_struct_time_to_iso8601(feed_updated)
except:
feed_updated = ''
else:
feed_updated = ''
entries_count = len(feed.entries)
feed_version = feed.version if feed.has_key('version') else ''
feed_encoding = feed.encoding if feed.has_key('encoding') else ''
feed_language = feed.feed.language if feed.feed.has_key('language') else ''
feed_icon = feed.feed.icon if feed.feed.has_key('icon') else ''
feed_image = feed.feed.image.href if feed.feed.has_key('image') else ''
feed_logo = feed.feed.logo if feed.feed.has_key('logo') else ''
feed_ttl = feed.feed.ttl if feed.feed.has_key('ttl') else ''
feed_properties = {
"version" : feed_version,
"encoding" : feed_encoding,
"language" : feed_language,
"rating" : '',
"entries_count" : entries_count,
"icon" : feed_icon,
"image" : feed_image,
"logo" : feed_logo,
"ttl" : feed_ttl,
"updated" : feed_updated,
}
return feed_properties
# TODO get all active feeds of active accounts and scan the feed with the earliest scanned time
# TODO Rename function name (idea: scan_and_populate)
def get_properties_of_entries(jid_bare, db_file, feed_url, feed_id, feed):
"""
Get new entries.
Parameters
----------
db_file : str
Path to database file.
url : str, optional
URL.
"""
# print('MID', feed_url, jid_bare, 'get_properties_of_entries')
function_name = sys._getframe().f_code.co_name
logger.debug('{}: feed_id: {} url: {}'
.format(function_name, feed_id, feed_url))
new_entries = []
for entry in feed.entries:
logger.debug('{}: entry: {}'.format(function_name, entry.link))
if entry.has_key("published"):
entry_published = entry.published
entry_published = dt.rfc2822_to_iso8601(entry_published)
else:
entry_published = ''
if entry.has_key("updated"):
entry_updated = entry.updated
entry_updated = dt.rfc2822_to_iso8601(entry_updated)
else:
entry_updated = dt.now()
if entry.has_key("link"):
# link = complete_url(source, entry.link)
entry_link = join_url(feed_url, entry.link)
entry_link = trim_url(entry_link)
else:
entry_link = feed_url
# title = feed["feed"]["title"]
# title = "{}: *{}*".format(feed["feed"]["title"], entry.title)
entry_title = entry.title if entry.has_key("title") else entry_published
entry_id = entry.id if entry.has_key("id") else entry_link
exist = sqlite.check_entry_exist(db_file, feed_id,
identifier=entry_id,
title=entry_title,
link=entry_link,
published=entry_published)
if not exist:
read_status = 0
# # Filter
# pathname = urlsplit(link).path
# string = (
# "{} {} {}"
# ).format(
# title, summary, pathname)
# if self.settings['default']['filter']:
# print('Filter is now processing data.')
# allow_list = config.is_include_keyword(db_file,
# "allow", string)
# if not allow_list:
# reject_list = config.is_include_keyword(db_file,
# "deny",
# string)
# if reject_list:
# read_status = 1
# logger.debug('Rejected : {}'
# '\n'
# 'Keyword : {}'
# .format(link, reject_list))
if isinstance(entry_published, int):
logger.error('Variable "published" is int: {}'.format(entry_published))
if isinstance(entry_updated, int):
logger.error('Variable "updated" is int: {}'.format(entry_updated))
# Authors
entry_authors =[]
if entry.has_key('authors'):
for author in entry.authors:
author_properties = {
'name' : author.name if author.has_key('name') else '',
'url' : author.href if author.has_key('href') else '',
'email' : author.email if author.has_key('email') else '',
}
entry_authors.extend([author_properties])
elif entry.has_key('author_detail'):
author_properties = {
'name' : entry.author_detail.name if entry.author_detail.has_key('name') else '',
'url' : entry.author_detail.href if entry.author_detail.has_key('href') else '',
'email' : entry.author_detail.email if entry.author_detail.has_key('email') else '',
}
entry_authors.extend([author_properties])
elif entry.has_key('author'):
author_properties = {
'name' : entry.author,
'url' : '',
'email' : '',
}
entry_authors.extend([author_properties])
# Contributors
entry_contributors = []
if entry.has_key('contributors'):
for contributor in entry.contributors:
contributor_properties = {
'name' : contributor.name if contributor.has_key('name') else '',
'url' : contributor.href if contributor.has_key('href') else '',
'email' : contributor.email if contributor.has_key('email') else '',
}
entry_contributors.extend([contributor_properties])
# Tags
entry_tags = []
if entry.has_key('tags'):
for tag in entry.tags:
tag_properties = {
'term' : tag.term if tag.has_key('term') else '',
'scheme' : tag.scheme if tag.has_key('scheme') else '',
'label' : tag.label if tag.has_key('label') else '',
}
entry_tags.extend([tag_properties])
# Content
entry_contents = []
if entry.has_key('content'):
for content in entry.content:
text = content.value if content.has_key('value') else ''
type = content.type if content.has_key('type') else ''
lang = content.lang if content.has_key('lang') else ''
base = content.base if content.has_key('base') else ''
entry_content = {
'text' : text,
'lang' : lang,
'type' : type,
'base' : base,
}
entry_contents.extend([entry_content])
# Links and Enclosures
entry_links = []
if entry.has_key('links'):
for link in entry.links:
link_properties = {
'url' : link.href if link.has_key('href') else '',
'rel' : link.rel if link.has_key('rel') else '',
'type' : link.type if link.has_key('type') else '',
'length' : '',
}
entry_links.extend([link_properties])
# Element media:content is utilized by Mastodon
if entry.has_key('media_content'):
for link in entry.media_content:
link_properties = {
'url' : link['url'] if 'url' in link else '',
'rel' : 'enclosure',
'type' : link['type'] if 'type' in link else '',
# 'medium' : link['medium'] if 'medium' in link else '',
'length' : link['filesize'] if 'filesize' in link else '',
}
entry_links.extend([link_properties])
if entry.has_key('media_thumbnail'):
for link in entry.media_thumbnail:
link_properties = {
'url' : link['url'] if 'url' in link else '',
'rel' : 'enclosure',
'type' : '',
# 'medium' : 'image',
'length' : '',
}
entry_links.extend([link_properties])
# Category
entry_category = entry.category if entry.has_key('category') else ''
# Comments
entry_comments = entry.comments if entry.has_key('comments') else ''
# href
entry_href = entry.href if entry.has_key('href') else ''
# Link: Same as entry.links[0].href in most if not all cases
entry_link = entry.link if entry.has_key('link') else ''
# Rating
entry_rating = entry.rating if entry.has_key('rating') else ''
# Summary
entry_summary_text = entry.summary if entry.has_key('summary') else ''
if entry.has_key('summary_detail'):
entry_summary_type = entry.summary_detail.type if entry.summary_detail.has_key('type') else ''
entry_summary_lang = entry.summary_detail.lang if entry.summary_detail.has_key('lang') else ''
entry_summary_base = entry.summary_detail.base if entry.summary_detail.has_key('base') else ''
else:
entry_summary_type = ''
entry_summary_lang = ''
entry_summary_base = ''
# Title
entry_title = entry.title if entry.has_key('title') else ''
if entry.has_key('title_detail'):
entry_title_type = entry.title_detail.type if entry.title_detail.has_key('type') else ''
else:
entry_title_type = ''
###########################################################
# media_type = e_link.type[:e_link.type.index("/")]
# if (e_link.rel == "enclosure" and
# media_type in ("audio", "image", "video")):
# media_link = e_link.href
# media_link = join_url(url, e_link.href)
# media_link = trim_url(media_link)
###########################################################
entry_properties = {
"identifier": entry_id,
"link": entry_link,
"href": entry_href,
"title": entry_title,
"title_type": entry_title_type,
'summary_text' : entry_summary_text,
'summary_lang' : entry_summary_lang,
'summary_type' : entry_summary_type,
'summary_base' : entry_summary_base,
'category' : entry_category,
"comments": entry_comments,
"rating": entry_rating,
"published": entry_published,
"updated": entry_updated,
"read_status": read_status
}
new_entries.extend([{
"entry_properties" : entry_properties,
"entry_authors" : entry_authors,
"entry_contributors" : entry_contributors,
"entry_contents" : entry_contents,
"entry_links" : entry_links,
"entry_tags" : entry_tags
}])
# await sqlite.add_entry(
# db_file, title, link, entry_id,
# url, date, read_status)
# await sqlite.set_date(db_file, url)
return new_entries
class FeedTask:
# TODO Take this function out of
# <class 'slixmpp.clientxmpp.ClientXMPP'>
async def check_updates(self, jid_bare):
"""
Start calling for update check up.
Parameters
----------
jid : str
Jabber ID.
"""
# print('Scanning for updates for JID {}'.format(jid_bare))
logger.info('Scanning for updates for JID {}'.format(jid_bare))
while True:
db_file = config.get_pathname_to_database(jid_bare)
urls = sqlite.get_active_feeds_url(db_file)
for url in urls:
url = url[0]
print('sta : ' + url)
# print('STA',url)
# # Skip Reddit
# if 'reddit.com' in str(url).lower():
# print('Reddit Atom Syndication feeds are not supported by Slixfeed.')
# print('Skipping URL:', url)
# continue
result = await fetch.http(url)
status_code = result['status_code']
feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
if not result['error']:
await sqlite.update_feed_status(db_file, feed_id, status_code)
document = result['content']
feed = parse(document)
feed_valid = 0 if feed.bozo else 1
await sqlite.update_feed_validity(db_file, feed_id, feed_valid)
feed_properties = Feed.get_properties_of_feed(
db_file, feed_id, feed)
await sqlite.update_feed_properties(
db_file, feed_id, feed_properties)
new_entries = Feed.get_properties_of_entries(
jid_bare, db_file, url, feed_id, feed)
if new_entries:
print('{}: {} new_entries: {} ({})'.format(jid_bare, len(new_entries), url, feed_id))
await sqlite.add_entries_and_update_feed_state(db_file, feed_id, new_entries)
await SQLiteMaintain.remove_nonexistent_entries(self, jid_bare, db_file, url, feed)
# await SQLiteMaintain.remove_nonexistent_entries(self, jid_bare, db_file, url, feed)
print('end : ' + url)
# await asyncio.sleep(50)
val = Config.get_setting_value(self.settings, jid_bare, 'check')
await asyncio.sleep(60 * float(val))
# Schedule to call this function again in 90 minutes
# loop.call_at(
# loop.time() + 60 * 90,
# loop.create_task,
# self.check_updates(jid)
# )
def restart_task(self, jid_bare):
if jid_bare == self.boundjid.bare:
return
if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info('Stopping task "check" for JID {}'.format(jid_bare))
try:
self.task_manager[jid_bare]['check'].cancel()
except:
logger.info('No task "check" for JID {} (FeedTask.check_updates)'
.format(jid_bare))
logger.info('Starting tasks "check" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['check'] = asyncio.create_task(
FeedTask.check_updates(self, jid_bare))
class Opml:
# TODO Consider adding element jid as a pointer of import
def export_to_file(jid, filename, results):
# print(jid, filename, results)
function_name = sys._getframe().f_code.co_name
logger.debug('{} jid: {} filename: {}'
.format(function_name, jid, filename))
root = ETR.Element("opml")
root.set("version", "1.0")
head = ETR.SubElement(root, "head")
ETR.SubElement(head, "title").text = "{}".format(jid)
ETR.SubElement(head, "description").text = (
"Set of subscriptions exported by Slixfeed")
ETR.SubElement(head, "generator").text = "Slixfeed"
ETR.SubElement(head, "urlPublic").text = (
"https://slixfeed.woodpeckersnest.space/")
time_stamp = dt.current_time()
ETR.SubElement(head, "dateCreated").text = time_stamp
ETR.SubElement(head, "dateModified").text = time_stamp
body = ETR.SubElement(root, "body")
for result in results:
outline = ETR.SubElement(body, "outline")
outline.set("text", result[1])
outline.set("xmlUrl", result[2])
# outline.set("type", result[2])
tree = ETR.ElementTree(root)
tree.write(filename)
async def import_from_file(db_file, result):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {}'
.format(function_name, db_file))
if not result['error']:
document = result['content']
root = ETR.fromstring(document)
before = sqlite.get_number_of_items(db_file, 'feeds_properties')
feeds = []
for child in root.findall(".//outline"):
url = child.get("xmlUrl")
title = child.get("text")
# feed = (url, title)
# feeds.extend([feed])
feed = {
'title' : title,
'url' : url,
}
feeds.extend([feed])
await sqlite.import_feeds(db_file, feeds)
await sqlite.add_metadata(db_file)
after = sqlite.get_number_of_items(db_file, 'feeds_properties')
difference = int(after) - int(before)
return difference

View file

@ -38,7 +38,7 @@ NOTE
Apparently, it is possible to view self presence. Apparently, it is possible to view self presence.
This means that there is no need to store presences in order to switch or restore presence. This means that there is no need to store presences in order to switch or restore presence.
check_readiness check_readiness
<presence from="slixfeed@canchat.org/xAPgJLHtMMHF" xml:lang="en" id="ab35c07b63a444d0a7c0a9a0b272f301" to="slixfeed@canchat.org/xAPgJLHtMMHF"><status>📂 Send a URL from a blog or a news website.</status><x xmlns="vcard-temp:x:update"><photo /></x></presence> <presence from="slixfeed@canchat.org/xAPgJLHtMMHF" xml:lang="en" id="ab35c07b63a444d0a7c0a9a0b272f301" to="slixfeed@canchat.org/xAPgJLHtMMHF"><status>📂 Send a URL from a blog or a news site.</status><x xmlns="vcard-temp:x:update"><photo /></x></presence>
JID: self.boundjid.bare JID: self.boundjid.bare
MUC: self.alias MUC: self.alias
@ -68,319 +68,27 @@ except Exception as exc:
""" """
import asyncio import asyncio
from feedparser import parse
import logging
import os import os
import slixfeed.action as action
import slixfeed.config as config import slixfeed.config as config
from slixfeed.config import Config from slixfeed.log import Logger
# from slixfeed.dt import current_time
import slixfeed.dt as dt
import slixfeed.fetch as fetch
import slixfeed.sqlite as sqlite
# from xmpp import Slixfeed
from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.message import XmppMessage
from slixfeed.xmpp.connect import XmppConnect
from slixfeed.xmpp.utility import get_chat_type
import time
# main_task = [] logger = Logger(__name__)
# jid_tasker = {}
# task_manager = {}
loop = asyncio.get_event_loop()
# def init_tasks(self):
# global task_ping
# # if task_ping is None or task_ping.done():
# # task_ping = asyncio.create_task(ping(self, jid=None))
# try:
# task_ping.cancel()
# except:
# logging.info('No ping task to cancel')
# task_ping = asyncio.create_task(ping(self, jid=None))
class Task: class Task:
def start(self, jid_full, tasks=None):
asyncio.create_task()
def cancel(self, jid_full, tasks=None): def start(self, jid_bare, callback):
pass callback(self, jid_bare)
def stop(self, jid_bare, task):
if (jid_bare in self.task_manager and
def task_ping(self): task in self.task_manager[jid_bare]):
# global task_ping_instance
try:
self.task_ping_instance.cancel()
except:
logging.info('No ping task to cancel.')
self.task_ping_instance = asyncio.create_task(XmppConnect.ping(self))
"""
FIXME
Tasks don't begin at the same time.
This is noticeable when calling "check" before "status".
await taskhandler.start_tasks(
self,
jid,
["check", "status"]
)
"""
async def start_tasks_xmpp_pubsub(self, jid_bare, tasks=None):
try:
self.task_manager[jid_bare]
except KeyError as e:
self.task_manager[jid_bare] = {}
logging.debug('KeyError:', str(e))
logging.info('Creating new task manager for JID {}'.format(jid_bare))
if not tasks:
tasks = ['check', 'publish']
logging.info('Stopping tasks {} for JID {}'.format(tasks, jid_bare))
for task in tasks:
# if self.task_manager[jid][task]:
try:
self.task_manager[jid_bare][task].cancel() self.task_manager[jid_bare][task].cancel()
except:
logging.info('No task {} for JID {} (start_tasks_xmpp_chat)'
.format(task, jid_bare))
logging.info('Starting tasks {} for JID {}'.format(tasks, jid_bare))
for task in tasks:
match task:
case 'publish':
self.task_manager[jid_bare]['publish'] = asyncio.create_task(
task_publish(self, jid_bare))
case 'check':
self.task_manager[jid_bare]['check'] = asyncio.create_task(
check_updates(self, jid_bare))
async def task_publish(self, jid_bare):
db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self.settings, jid_bare, db_file)
while True:
await action.xmpp_pubsub_send_unread_items(self, jid_bare)
await asyncio.sleep(60 * 180)
async def start_tasks_xmpp_chat(self, jid_bare, tasks=None):
"""
NOTE
For proper activation of tasks involving task 'interval', it is essential
to place task 'interval' as the last to start due to await asyncio.sleep()
which otherwise would postpone tasks that would be set after task 'interval'
"""
if jid_bare == self.boundjid.bare:
return
try:
self.task_manager[jid_bare]
except KeyError as e:
self.task_manager[jid_bare] = {}
logging.debug('KeyError:', str(e))
logging.info('Creating new task manager for JID {}'.format(jid_bare))
if not tasks:
tasks = ['status', 'check', 'interval']
logging.info('Stopping tasks {} for JID {}'.format(tasks, jid_bare))
for task in tasks:
# if self.task_manager[jid][task]:
try:
self.task_manager[jid_bare][task].cancel()
except:
logging.info('No task {} for JID {} (start_tasks_xmpp_chat)'
.format(task, jid_bare))
logging.info('Starting tasks {} for JID {}'.format(tasks, jid_bare))
for task in tasks:
# print("task:", task)
# print("tasks:")
# print(tasks)
# breakpoint()
match task:
case 'check':
self.task_manager[jid_bare]['check'] = asyncio.create_task(
check_updates(self, jid_bare))
case 'status':
self.task_manager[jid_bare]['status'] = asyncio.create_task(
task_status_message(self, jid_bare))
case 'interval':
self.task_manager[jid_bare]['interval'] = asyncio.create_task(
task_message(self, jid_bare))
# for task in self.task_manager[jid].values():
# print("task_manager[jid].values()")
# print(self.task_manager[jid].values())
# print("task")
# print(task)
# print("jid")
# print(jid)
# breakpoint()
# await task
async def task_status_message(self, jid):
await action.xmpp_send_status_message(self, jid)
refresh_task(self, jid, task_status_message, 'status', '90')
async def task_message(self, jid_bare):
db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self.settings, jid_bare, db_file)
update_interval = Config.get_setting_value(self.settings, jid_bare, 'interval')
update_interval = 60 * int(update_interval)
last_update_time = sqlite.get_last_update_time(db_file)
if last_update_time:
last_update_time = float(last_update_time)
diff = time.time() - last_update_time
if diff < update_interval:
next_update_time = update_interval - diff
await asyncio.sleep(next_update_time) # FIXME!
# print("jid :", jid, "\n"
# "time :", time.time(), "\n"
# "last_update_time :", last_update_time, "\n"
# "difference :", diff, "\n"
# "update interval :", update_interval, "\n"
# "next_update_time :", next_update_time, "\n"
# )
# elif diff > val:
# next_update_time = val
await sqlite.update_last_update_time(db_file)
else: else:
await sqlite.set_last_update_time(db_file) logger.debug('No task {} for JID {} (Task.stop)'
await action.xmpp_chat_send_unread_items(self, jid_bare) .format(task, jid_bare))
refresh_task(self, jid_bare, task_message, 'interval')
await start_tasks_xmpp_chat(self, jid_bare, ['status'])
def clean_tasks_xmpp_chat(self, jid, tasks=None):
if not tasks:
tasks = ['interval', 'status', 'check']
logging.info('Stopping tasks {} for JID {}'.format(tasks, jid))
for task in tasks:
# if self.task_manager[jid][task]:
try:
self.task_manager[jid][task].cancel()
except:
logging.debug('No task {} for JID {} (clean_tasks_xmpp)'
.format(task, jid))
def refresh_task(self, jid_bare, callback, key, val=None):
"""
Apply new setting at runtime.
Parameters
----------
jid : str
Jabber ID.
key : str
Key.
val : str, optional
Value. The default is None.
"""
logging.info('Refreshing task {} for JID {}'.format(callback, jid_bare))
if not val:
db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self.settings, jid_bare, db_file)
val = Config.get_setting_value(self.settings, jid_bare, key)
# if self.task_manager[jid][key]:
if jid_bare in self.task_manager:
try:
self.task_manager[jid_bare][key].cancel()
except:
logging.info('No task of type {} to cancel for '
'JID {} (refresh_task)'.format(key, jid_bare))
# self.task_manager[jid][key] = loop.call_at(
# loop.time() + 60 * float(val),
# loop.create_task,
# (callback(self, jid))
# # send_update(jid)
# )
self.task_manager[jid_bare][key] = loop.create_task(
wait_and_run(self, callback, jid_bare, val)
)
# self.task_manager[jid][key] = loop.call_later(
# 60 * float(val),
# loop.create_task,
# send_update(jid)
# )
# self.task_manager[jid][key] = send_update.loop.call_at(
# send_update.loop.time() + 60 * val,
# send_update.loop.create_task,
# send_update(jid)
# )
async def wait_and_run(self, callback, jid_bare, val):
await asyncio.sleep(60 * float(val))
await callback(self, jid_bare)
# TODO Take this function out of
# <class 'slixmpp.clientxmpp.ClientXMPP'>
async def check_updates(self, jid_bare):
"""
Start calling for update check up.
Parameters
----------
jid : str
Jabber ID.
"""
# print('Scanning for updates for JID {}'.format(jid_bare))
logging.info('Scanning for updates for JID {}'.format(jid_bare))
while True:
db_file = config.get_pathname_to_database(jid_bare)
urls = sqlite.get_active_feeds_url(db_file)
for url in urls:
url = url[0]
# print('STA',url)
# # Skip Reddit
# if 'reddit.com' in str(url).lower():
# print('Reddit Atom Syndication feeds are not supported by Slixfeed.')
# print('Skipping URL:', url)
# continue
result = await fetch.http(url)
status_code = result['status_code']
feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
if not result['error']:
await sqlite.update_feed_status(db_file, feed_id, status_code)
document = result['content']
feed = parse(document)
feed_valid = 0 if feed.bozo else 1
await sqlite.update_feed_validity(db_file, feed_id, feed_valid)
feed_properties = action.get_properties_of_feed(db_file,
feed_id, feed)
await sqlite.update_feed_properties(db_file, feed_id,
feed_properties)
new_entries = action.get_properties_of_entries(
jid_bare, db_file, url, feed_id, feed)
if new_entries: await sqlite.add_entries_and_update_feed_state(
db_file, feed_id, new_entries)
await asyncio.sleep(50)
val = Config.get_setting_value(self.settings, jid_bare, 'check')
await asyncio.sleep(60 * float(val))
# Schedule to call this function again in 90 minutes
# loop.call_at(
# loop.time() + 60 * 90,
# loop.create_task,
# self.check_updates(jid)
# )
""" """
@ -393,6 +101,7 @@ async def select_file(self):
""" """
Initiate actions by JID (Jabber ID). Initiate actions by JID (Jabber ID).
""" """
main_task = []
while True: while True:
db_dir = config.get_default_data_directory() db_dir = config.get_default_data_directory()
if not os.path.isdir(db_dir): if not os.path.isdir(db_dir):

View file

@ -19,11 +19,11 @@ TODO
""" """
from email.utils import parseaddr from email.utils import parseaddr
import logging
import os import os
import random import random
import slixfeed.config as config import slixfeed.config as config
import slixfeed.fetch as fetch import slixfeed.fetch as fetch
from slixfeed.log import Logger
from urllib.parse import ( from urllib.parse import (
parse_qs, parse_qs,
urlencode, urlencode,
@ -33,6 +33,8 @@ from urllib.parse import (
urlunsplit urlunsplit
) )
logger = Logger(__name__)
# NOTE # NOTE
# hostname and protocol are listed as one in file proxies.toml. # hostname and protocol are listed as one in file proxies.toml.
@ -113,11 +115,11 @@ async def replace_hostname(url, url_type):
config.update_proxies(proxies_file, proxy_name, config.update_proxies(proxies_file, proxy_name,
proxy_type, proxy_url) proxy_type, proxy_url)
except ValueError as e: except ValueError as e:
logging.error([str(e), proxy_url]) logger.error([str(e), proxy_url])
url_new = None url_new = None
else: else:
logging.warning( logger.warning('No proxy URLs for {}. '
"No proxy URLs for {}. Please update proxies.toml" 'Please update proxies.toml'
.format(proxy_name)) .format(proxy_name))
url_new = url url_new = url
break break

347
slixfeed/utilities.py Normal file
View file

@ -0,0 +1,347 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
TODO
1) Function scan at "for entry in entries"
Suppress directly calling function "add_entry" (accept db_file)
Pass a list of valid entries to a new function "add_entries"
(accept db_file) which would call function "add_entry" (accept cur).
* accelerate adding of large set of entries at once.
* prevent (or mitigate halt of consequent actions).
* reduce I/O.
2) Call sqlite function from function statistics.
Returning a list of values doesn't' seem to be a good practice.
3) Special statistics for operator:
* Size of database(s);
* Amount of JIDs subscribed;
* Amount of feeds of all JIDs;
* Amount of entries of all JIDs.
4) Consider to append text to remind to share presence
'✒️ Share online status to receive updates'
5) Request for subscription
if (await XmppUtilities.get_chat_type(self, jid_bare) == 'chat' and
not self.client_roster[jid_bare]['to']):
XmppPresence.subscription(self, jid_bare, 'subscribe')
await XmppRoster.add(self, jid_bare)
status_message = '✒️ Share online status to receive updates'
XmppPresence.send(self, jid_bare, status_message)
message_subject = 'RSS News Bot'
message_body = 'Share online status to receive updates.'
XmppMessage.send_headline(self, jid_bare, message_subject,
message_body, 'chat')
"""
import hashlib
import slixfeed.config as config
from slixfeed.config import Config
from lxml import etree, html
import slixfeed.dt as dt
import slixfeed.fetch as fetch
from slixfeed.log import Logger
import slixfeed.sqlite as sqlite
from slixfeed.url import join_url, complete_url
import sys
try:
import tomllib
except:
import tomli as tomllib
logger = Logger(__name__)
class Documentation:
def manual(filename, section=None, command=None):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: filename: {}'.format(function_name, filename))
config_dir = config.get_default_config_directory()
with open(config_dir + '/' + filename, mode="rb") as commands:
cmds = tomllib.load(commands)
if section == 'all':
cmd_list = ''
for cmd in cmds:
for i in cmds[cmd]:
cmd_list += cmds[cmd][i] + '\n'
elif command and section:
try:
cmd_list = cmds[section][command]
except KeyError as e:
logger.error(e)
cmd_list = None
elif section:
try:
cmd_list = []
for cmd in cmds[section]:
cmd_list.extend([cmd])
except KeyError as e:
logger.error('KeyError:' + str(e))
cmd_list = None
else:
cmd_list = []
for cmd in cmds:
cmd_list.extend([cmd])
return cmd_list
class Html:
async def extract_image_from_html(url):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: url: {}'.format(function_name, url))
result = await fetch.http(url)
if not result['error']:
data = result['content']
tree = html.fromstring(data)
# TODO Exclude banners, class="share" links etc.
images = tree.xpath(
'//img[not('
'contains(@src, "avatar") or '
'contains(@src, "cc-by-sa") or '
'contains(@src, "emoji") or '
'contains(@src, "icon") or '
'contains(@src, "logo") or '
'contains(@src, "letture") or '
'contains(@src, "poweredby_mediawi") or '
'contains(@src, "search") or '
'contains(@src, "share") or '
'contains(@src, "smiley")'
')]/@src')
if len(images):
image = images[0]
image = str(image)
image_url = complete_url(url, image)
return image_url
def remove_html_tags(data):
function_name = sys._getframe().f_code.co_name
logger.debug('{}'.format(function_name))
parser = etree.HTMLParser()
tree = etree.fromstring(data, parser)
data = etree.tostring(tree, encoding='unicode', method='text')
data = data.replace("\n\n", "\n")
return data
# /questions/9662346/python-code-to-remove-html-tags-from-a-string
def _remove_html_tags(text):
import xml.etree.ElementTree
return ''.join(xml.etree.ElementTree.fromstring(text).itertext())
def __remove_html_tags(data):
from bs4 import BeautifulSoup
function_name = sys._getframe().f_code.co_name
logger.debug('{}'.format(function_name))
data = BeautifulSoup(data, "lxml").text
data = data.replace("\n\n", "\n")
return data
class MD:
def export_to_markdown(jid, filename, results):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid: {} filename: {}'
.format(function_name, jid, filename))
with open(filename, 'w') as file:
file.write('# Subscriptions for {}\n'.format(jid))
file.write('## Set of feeds exported with Slixfeed\n')
for result in results:
file.write('- [{}]({})\n'.format(result[1], result[2]))
file.write('\n\n* * *\n\nThis list was saved on {} from xmpp:{} using '
'[Slixfeed](https://slixfeed.woodpeckersnest.space/)\n'
.format(dt.current_date(), jid))
def log_to_markdown(timestamp, filename, jid, message):
"""
Log message to a markdown file.
Parameters
----------
timestamp : str
Time stamp.
filename : str
Jabber ID as name of file.
jid : str
Jabber ID.
message : str
Message content.
Returns
-------
None.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: timestamp: {} filename: {} jid: {} message: {}'.format(function_name, timestamp, filename, jid, message))
with open(filename + '.md', 'a') as file:
# entry = "{} {}:\n{}\n\n".format(timestamp, jid, message)
entry = '## {}\n### {}\n\n{}\n\n'.format(jid, timestamp, message)
file.write(entry)
class SQLiteMaintain:
# TODO
# (1) Check for duplications
# (2) append all duplications to a list
# (3) Send the list to a function in module sqlite.
async def remove_nonexistent_entries(self, jid_bare, db_file, url, feed):
"""
Remove entries that don't exist in a given parsed feed.
Check the entries returned from feed and delete read non
existing entries, otherwise move to table archive, if unread.
Parameters
----------
db_file : str
Path to database file.
url : str
Feed URL.
feed : list
Parsed feed document.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {} url: {}'
.format(function_name, db_file, url))
feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
items = sqlite.get_entries_of_feed(db_file, feed_id)
entries = feed.entries
limit = Config.get_setting_value(self.settings, jid_bare, 'archive')
print(limit)
for item in items:
ix, entry_title, entry_link, entry_id, timestamp = item
read_status = sqlite.is_entry_read(db_file, ix)
read_status = read_status[0]
valid = False
for entry in entries:
title = None
link = None
time = None
# valid = False
# TODO better check and don't repeat code
if entry.has_key("id") and entry_id:
if entry.id == entry_id:
print("compare entry.id == entry_id:", entry.id)
print("compare entry.id == entry_id:", entry_id)
print("============")
valid = True
break
else:
if entry.has_key("title"):
title = entry.title
else:
title = feed["feed"]["title"]
if entry.has_key("link"):
link = join_url(url, entry.link)
else:
link = url
if entry.has_key("published") and timestamp:
print("compare published:", title, link, time)
print("compare published:", entry_title, entry_link, timestamp)
print("============")
time = dt.rfc2822_to_iso8601(entry.published)
if (entry_title == title and
entry_link == link and
timestamp == time):
valid = True
break
else:
if (entry_title == title and
entry_link == link):
print("compare entry_link == link:", title, link)
print("compare entry_title == title:", entry_title, entry_link)
print("============")
valid = True
break
# TODO better check and don't repeat code
if not valid:
# print("id: ", ix)
# if title:
# print("title: ", title)
# print("entry_title: ", entry_title)
# if link:
# print("link: ", link)
# print("entry_link: ", entry_link)
# if entry.id:
# print("last_entry:", entry.id)
# print("entry_id: ", entry_id)
# if time:
# print("time: ", time)
# print("timestamp: ", timestamp)
# print("read: ", read_status)
# breakpoint()
# TODO Send to table archive
# TODO Also make a regular/routine check for sources that
# have been changed (though that can only happen when
# manually editing)
# ix = item[0]
# print(">>> SOURCE: ", source)
# print(">>> INVALID:", entry_title)
# print("title:", entry_title)
# print("link :", entry_link)
# print("id :", entry_id)
if read_status == 1:
await sqlite.delete_entry_by_id(db_file, ix)
# print(">>> DELETING:", entry_title)
else:
# print(">>> ARCHIVING:", entry_title)
await sqlite.archive_entry(db_file, ix)
await sqlite.maintain_archive(db_file, limit)
class Task:
def start(self, jid_bare, callback):
callback(self, jid_bare)
def stop(self, jid_bare, task):
if (jid_bare in self.task_manager and
task in self.task_manager[jid_bare]):
self.task_manager[jid_bare][task].cancel()
else:
logger.debug('No task {} for JID {} (Task.stop)'
.format(task, jid_bare))
class Utilities:
# NOTE Warning: Entry might not have a link
# TODO Handle situation error
def hash_url_to_md5(url):
url_encoded = url.encode()
url_hashed = hashlib.md5(url_encoded)
url_digest = url_hashed.hexdigest()
return url_digest
def pick_a_feed(lang=None):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: lang: {}'
.format(function_name, lang))
config_dir = config.get_default_config_directory()
with open(config_dir + '/' + 'feeds.toml', mode="rb") as feeds:
urls = tomllib.load(feeds)
import random
url = random.choice(urls['feeds'])
return url

View file

@ -1,2 +1,2 @@
__version__ = '0.1.76' __version__ = '0.1.77'
__version_info__ = (0, 1, 76) __version_info__ = (0, 1, 77)

View file

@ -24,36 +24,29 @@ TODO
""" """
import asyncio import asyncio
from feedparser import parse from random import randrange # pending_tasks: Use a list and read the first index (i.e. index 0).
import logging
import os
import slixfeed.action as action
import slixfeed.config as config import slixfeed.config as config
import slixfeed.crawl as crawl
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.dt as dt from slixfeed.log import Logger
import slixfeed.fetch as fetch
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
import slixfeed.task as task from slixfeed.url import (
import slixfeed.url as uri remove_tracking_parameters,
from slixfeed.version import __version__ replace_hostname,
from slixfeed.xmpp.bookmark import XmppBookmark )
from slixfeed.syndication import FeedTask
from slixfeed.utilities import Documentation, Html, MD, Task
from slixfeed.xmpp.commands import XmppCommands from slixfeed.xmpp.commands import XmppCommands
from slixfeed.xmpp.muc import XmppGroupchat
from slixfeed.xmpp.message import XmppMessage from slixfeed.xmpp.message import XmppMessage
from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.publish import XmppPubsub from slixfeed.xmpp.privilege import is_operator, is_moderator
from slixfeed.xmpp.status import XmppStatusTask
from slixfeed.xmpp.upload import XmppUpload from slixfeed.xmpp.upload import XmppUpload
from slixfeed.xmpp.privilege import is_moderator, is_operator, is_access from slixfeed.xmpp.utilities import XmppUtilities
from slixfeed.xmpp.utility import get_chat_type import sys
import time import time
from random import randrange
try: logger = Logger(__name__)
import tomllib
except:
import tomli as tomllib
# for task in main_task: # for task in main_task:
@ -63,7 +56,9 @@ except:
# if not main_task: # if not main_task:
# await select_file() # await select_file()
class Chat:
class XmppChat:
async def process_message(self, message): async def process_message(self, message):
""" """
@ -154,7 +149,7 @@ class Chat:
command = command[1:] command = command[1:]
command_lowercase = command.lower() command_lowercase = command.lower()
logging.debug([str(message['from']), ':', command]) logger.debug([str(message['from']), ':', command])
# Support private message via groupchat # Support private message via groupchat
# See https://codeberg.org/poezio/slixmpp/issues/3506 # See https://codeberg.org/poezio/slixmpp/issues/3506
@ -175,7 +170,7 @@ class Chat:
'Usage: `help <key>`' 'Usage: `help <key>`'
.format(command_list)) .format(command_list))
case 'help all': case 'help all':
command_list = action.manual('commands.toml', section='all') command_list = Documentation.manual('commands.toml', section='all')
response = ('Complete list of commands:\n' response = ('Complete list of commands:\n'
'```\n{}\n```' '```\n{}\n```'
.format(command_list)) .format(command_list))
@ -185,9 +180,8 @@ class Chat:
if len(command) == 2: if len(command) == 2:
command_root = command[0] command_root = command[0]
command_name = command[1] command_name = command[1]
command_list = action.manual('commands.toml', command_list = Documentation.manual(
section=command_root, 'commands.toml', section=command_root, command=command_name)
command=command_name)
if command_list: if command_list:
command_list = ''.join(command_list) command_list = ''.join(command_list)
response = (command_list) response = (command_list)
@ -196,7 +190,7 @@ class Chat:
.format(command_root, command_name)) .format(command_root, command_name))
elif len(command) == 1: elif len(command) == 1:
command = command[0] command = command[0]
command_list = action.manual('commands.toml', command) command_list = Documentation.manual('commands.toml', command)
if command_list: if command_list:
command_list = ' '.join(command_list) command_list = ' '.join(command_list)
response = ('Available command `{}` keys:\n' response = ('Available command `{}` keys:\n'
@ -224,7 +218,7 @@ class Chat:
'I am an RSS News Bot.\n' 'I am an RSS News Bot.\n'
'Send "help" for further instructions.\n' 'Send "help" for further instructions.\n'
.format(self.alias)) .format(self.alias))
case _ if command_lowercase.startswith('add '): case _ if command_lowercase.startswith('add'):
command = command[4:] command = command[4:]
url = command.split(' ')[0] url = command.split(' ')[0]
title = ' '.join(command.split(' ')[1:]) title = ' '.join(command.split(' ')[1:])
@ -284,10 +278,10 @@ class Chat:
else: else:
response = ('This action is restricted. ' response = ('This action is restricted. '
'Type: viewing bookmarks.') 'Type: viewing bookmarks.')
case _ if command_lowercase.startswith('clear '): case _ if command_lowercase.startswith('clear'):
key = command[6:] key = command[6:]
response = await XmppCommands.clear_filter(db_file, key) response = await XmppCommands.clear_filter(db_file, key)
case _ if command_lowercase.startswith('default '): case _ if command_lowercase.startswith('default'):
key = command[8:] key = command[8:]
response = await XmppCommands.restore_default( response = await XmppCommands.restore_default(
self, jid_bare, key=None) self, jid_bare, key=None)
@ -317,15 +311,14 @@ class Chat:
response = ('No action has been taken.' response = ('No action has been taken.'
'\n' '\n'
'Missing keywords.') 'Missing keywords.')
case _ if command_lowercase.startswith('disable '): case _ if command_lowercase.startswith('disable'):
response = await XmppCommands.feed_disable( response = await XmppCommands.feed_disable(
self, db_file, jid_bare, command) self, db_file, jid_bare, command)
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) case _ if command_lowercase.startswith('enable'):
case _ if command_lowercase.startswith('enable '):
response = await XmppCommands.feed_enable( response = await XmppCommands.feed_enable(
self, db_file, command) self, db_file, command)
case _ if command_lowercase.startswith('export '): case _ if command_lowercase.startswith('export'):
ext = command[7:] ext = command[7:]
if ext in ('md', 'opml'): # html xbel if ext in ('md', 'opml'): # html xbel
status_type = 'dnd' status_type = 'dnd'
@ -346,12 +339,11 @@ class Chat:
# 'Feeds exported successfully to {}.\n{}' # 'Feeds exported successfully to {}.\n{}'
# ).format(ex, url) # ).format(ex, url)
# XmppMessage.send_oob_reply_message(message, url, response) # XmppMessage.send_oob_reply_message(message, url, response)
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
XmppMessage.send_oob(self, jid_bare, url, chat_type) XmppMessage.send_oob(self, jid_bare, url, chat_type)
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
else: else:
response = ('Unsupported filetype.\n' response = ('Unsupported filetype.\n'
'Try: md or opml') 'Try: md or opml')
@ -375,8 +367,7 @@ class Chat:
response = XmppCommands.fetch_gemini() response = XmppCommands.fetch_gemini()
case _ if (command_lowercase.startswith('http') and case _ if (command_lowercase.startswith('http') and
command_lowercase.endswith('.opml')): command_lowercase.endswith('.opml')):
key_list = ['status'] Task.stop(self, jid_bare, 'status')
task.clean_tasks_xmpp_chat(self, jid_bare, key_list)
status_type = 'dnd' status_type = 'dnd'
status_message = '📥️ Procesing request to import feeds...' status_message = '📥️ Procesing request to import feeds...'
# pending_tasks_num = len(self.pending_tasks[jid_bare]) # pending_tasks_num = len(self.pending_tasks[jid_bare])
@ -390,14 +381,13 @@ class Chat:
self, db_file, jid_bare, command) self, db_file, jid_bare, command)
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) case _ if command_lowercase.startswith('pubsub list'):
case _ if command_lowercase.startswith('pubsub list '):
jid = command[12:] jid = command[12:]
response = 'List of nodes for {}:\n```\n'.format(jid) response = 'List of nodes for {}:\n```\n'.format(jid)
response = await XmppCommands.pubsub_list(self, jid) response = await XmppCommands.pubsub_list(self, jid)
response += '```' response += '```'
case _ if command_lowercase.startswith('pubsub send '): case _ if command_lowercase.startswith('pubsub send'):
if is_operator(self, jid_bare): if is_operator(self, jid_bare):
info = command[12:] info = command[12:]
info = info.split(' ') info = info.split(' ')
@ -416,7 +406,6 @@ class Chat:
command_lowercase.startswith('itpc:/') or command_lowercase.startswith('itpc:/') or
command_lowercase.startswith('rss:/')): command_lowercase.startswith('rss:/')):
url = command url = command
# task.clean_tasks_xmpp_chat(self, jid_bare, ['status'])
status_type = 'dnd' status_type = 'dnd'
status_message = ('📫️ Processing request to fetch data from {}' status_message = ('📫️ Processing request to fetch data from {}'
.format(url)) .format(url))
@ -429,11 +418,9 @@ class Chat:
status_type=status_type) status_type=status_type)
response = await XmppCommands.fetch_http( response = await XmppCommands.fetch_http(
self, command, db_file, jid_bare) self, command, db_file, jid_bare)
# task.clean_tasks_xmpp_chat(self, jid_bare, ['status'])
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
# except: # except:
# response = ( # response = (
# '> {}\nNews source is in the process ' # '> {}\nNews source is in the process '
@ -445,6 +432,7 @@ class Chat:
if val: if val:
response = await XmppCommands.set_interval( response = await XmppCommands.set_interval(
self, db_file, jid_bare, val) self, db_file, jid_bare, val)
XmppChatTask.restart_task(self, jid_bare)
else: else:
response = 'Current value for interval: ' response = 'Current value for interval: '
response += XmppCommands.get_interval(self, jid_bare) response += XmppCommands.get_interval(self, jid_bare)
@ -469,10 +457,10 @@ class Chat:
response = await XmppCommands.set_old_off( response = await XmppCommands.set_old_off(
self, jid_bare, db_file) self, jid_bare, db_file)
case _ if command_lowercase.startswith('next'): case _ if command_lowercase.startswith('next'):
await XmppCommands.send_next_update(self, jid_bare, command) num = command[5:]
key_list = ['status'] await XmppChatAction.send_unread_items(self, jid_bare, num)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) XmppStatusTask.restart_task(self, jid_bare)
case _ if command_lowercase.startswith('node delete '): case _ if command_lowercase.startswith('node delete'):
if is_operator(self, jid_bare): if is_operator(self, jid_bare):
info = command[12:] info = command[12:]
info = info.split(' ') info = info.split(' ')
@ -480,7 +468,7 @@ class Chat:
else: else:
response = ('This action is restricted. ' response = ('This action is restricted. '
'Type: sending news to PubSub.') 'Type: sending news to PubSub.')
case _ if command_lowercase.startswith('node purge '): case _ if command_lowercase.startswith('node purge'):
if is_operator(self, jid_bare): if is_operator(self, jid_bare):
info = command[11:] info = command[11:]
info = info.split(' ') info = info.split(' ')
@ -505,13 +493,12 @@ class Chat:
response += XmppCommands.get_quantum(self, jid_bare) response += XmppCommands.get_quantum(self, jid_bare)
case 'random': case 'random':
response = XmppCommands.set_random(self, jid_bare, db_file) response = XmppCommands.set_random(self, jid_bare, db_file)
case _ if command_lowercase.startswith('read '): case _ if command_lowercase.startswith('read'):
data = command[5:] data = command[5:]
data = data.split() data = data.split()
url = data[0] url = data[0]
if url: if url:
key_list = ['status'] Task.stop(self, jid_bare, 'status')
task.clean_tasks_xmpp_chat(self, jid_bare, key_list)
status_type = 'dnd' status_type = 'dnd'
status_message = ('📫️ Processing request to fetch data ' status_message = ('📫️ Processing request to fetch data '
'from {}'.format(url)) 'from {}'.format(url))
@ -520,12 +507,11 @@ class Chat:
response = await XmppCommands.feed_read( response = await XmppCommands.feed_read(
self, jid_bare, data, url) self, jid_bare, data, url)
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
else: else:
response = ('No action has been taken.' response = ('No action has been taken.'
'\n' '\n'
'Missing URL.') 'Missing URL.')
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
case _ if command_lowercase.startswith('recent'): case _ if command_lowercase.startswith('recent'):
num = command[7:] num = command[7:]
if not num: num = 5 if not num: num = 5
@ -535,23 +521,19 @@ class Chat:
response += result + '```\n' response += result + '```\n'
else: else:
response = result response = result
case _ if command_lowercase.startswith('remove '): case _ if command_lowercase.startswith('remove'):
ix_url = command[7:] ix_url = command[7:]
ix_url = ix_url.split(' ') ix_url = ix_url.split(' ')
response = await XmppCommands.feed_remove( response = await XmppCommands.feed_remove(
self, jid_bare, db_file, ix_url) self, jid_bare, db_file, ix_url)
# refresh_task(self, jid_bare, send_status, 'status', 20) XmppStatusTask.restart_task(self, jid_bare)
# task.clean_tasks_xmpp_chat(self, jid_bare, ['status']) case _ if command_lowercase.startswith('rename'):
key_list = ['status']
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
case _ if command_lowercase.startswith('rename '):
response = await XmppCommands.feed_rename( response = await XmppCommands.feed_rename(
self, db_file, jid_bare, command) self, db_file, jid_bare, command)
case _ if command_lowercase.startswith('reset'): case _ if command_lowercase.startswith('reset'):
ix_url = command[6:] ix_url = command[6:]
ix_url = ix_url.split(' ') if ix_url: ix_url = ix_url.split(' ')
key_list = ['status'] Task.stop(self, jid_bare, 'status')
task.clean_tasks_xmpp_chat(self, jid_bare, key_list)
status_type = 'dnd' status_type = 'dnd'
status_message = '📫️ Marking entries as read...' status_message = '📫️ Marking entries as read...'
# pending_tasks_num = len(self.pending_tasks[jid_bare]) # pending_tasks_num = len(self.pending_tasks[jid_bare])
@ -562,24 +544,22 @@ class Chat:
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
status_type=status_type) status_type=status_type)
response = await XmppCommands.mark_as_read( response = await XmppCommands.mark_as_read(
self, jid_bare, db_file, ix_url) jid_bare, db_file, ix_url)
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) case _ if command_lowercase.startswith('search'):
case _ if command_lowercase.startswith('search '):
query = command[7:] query = command[7:]
response = XmppCommands.search_items(self, db_file, query) response = XmppCommands.search_items(db_file, query)
case 'start': case 'start':
status_type = 'available' status_type = 'available'
status_message = '📫️ Welcome back!' status_message = '📫️ Welcome back!'
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
status_type=status_type) status_type=status_type)
await asyncio.sleep(5) await asyncio.sleep(5)
key_list = ['check', 'status', 'interval'] tasks = (FeedTask, XmppChatTask, XmppStatusTask)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
response = await XmppCommands.scheduler_start( response = await XmppCommands.scheduler_start(
self, db_file, jid_bare) self, db_file, jid_bare, tasks)
case 'stats': case 'stats':
response = XmppCommands.print_statistics(db_file) response = XmppCommands.print_statistics(db_file)
case 'stop': case 'stop':
@ -617,10 +597,10 @@ class Chat:
# os.mkdir(data_dir) # os.mkdir(data_dir)
# if not os.path.isdir(data_dir + '/logs/'): # if not os.path.isdir(data_dir + '/logs/'):
# os.mkdir(data_dir + '/logs/') # os.mkdir(data_dir + '/logs/')
# action.log_to_markdown( # MD.log_to_markdown(
# dt.current_time(), os.path.join(data_dir, 'logs', jid_bare), # dt.current_time(), os.path.join(data_dir, 'logs', jid_bare),
# jid_bare, command) # jid_bare, command)
# action.log_to_markdown( # MD.log_to_markdown(
# dt.current_time(), os.path.join(data_dir, 'logs', jid_bare), # dt.current_time(), os.path.join(data_dir, 'logs', jid_bare),
# jid_bare, response) # jid_bare, response)
@ -630,3 +610,228 @@ class Chat:
# '{}\n' # '{}\n'
# .format(command, jid_bare, response) # .format(command, jid_bare, response)
# ) # )
class XmppChatAction:
async def send_unread_items(self, jid_bare, num=None):
"""
Send news items as messages.
Parameters
----------
jid : str
Jabber ID.
num : str, optional
Number. The default is None.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid: {} num: {}'.format(function_name, jid_bare, num))
db_file = config.get_pathname_to_database(jid_bare)
show_media = Config.get_setting_value(self.settings, jid_bare, 'media')
if not num:
num = Config.get_setting_value(self.settings, jid_bare, 'quantum')
else:
num = int(num)
results = sqlite.get_unread_entries(db_file, num)
news_digest = ''
media = None
chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
for result in results:
ix = result[0]
title_e = result[1]
url = result[2]
summary = result[3]
feed_id = result[4]
date = result[5]
enclosure = sqlite.get_enclosure_by_entry_id(db_file, ix)
if enclosure: enclosure = enclosure[0]
title_f = sqlite.get_feed_title(db_file, feed_id)
title_f = title_f[0]
news_digest += await XmppChatAction.list_unread_entries(self, result, title_f, jid_bare)
# print(db_file)
# print(result[0])
# breakpoint()
await sqlite.mark_as_read(db_file, ix)
# Find media
# if url.startswith("magnet:"):
# media = action.get_magnet(url)
# elif enclosure.startswith("magnet:"):
# media = action.get_magnet(enclosure)
# elif enclosure:
if show_media:
if enclosure:
media = enclosure
else:
media = await Html.extract_image_from_html(url)
if media and news_digest:
# Send textual message
XmppMessage.send(self, jid_bare, news_digest, chat_type)
news_digest = ''
# Send media
XmppMessage.send_oob(self, jid_bare, media, chat_type)
media = None
if news_digest:
XmppMessage.send(self, jid_bare, news_digest, chat_type)
# TODO Add while loop to assure delivery.
# print(await current_time(), ">>> ACT send_message",jid)
# NOTE Do we need "if statement"? See NOTE at is_muc.
# if chat_type in ('chat', 'groupchat'):
# # TODO Provide a choice (with or without images)
# XmppMessage.send(self, jid, news_digest, chat_type)
# See XEP-0367
# if media:
# # message = xmpp.Slixfeed.make_message(
# # self, mto=jid, mbody=new, mtype=chat_type)
# message = xmpp.Slixfeed.make_message(
# self, mto=jid, mbody=media, mtype=chat_type)
# message['oob']['url'] = media
# message.send()
# TODO Do not refresh task before
# verifying that it was completed.
# XmppStatusTask.restart_task(self, jid_bare)
# XmppCommands.task_start(self, jid_bare, 'interval')
# interval = await initdb(
# jid,
# sqlite.is_setting_key,
# "interval"
# )
# self.task_manager[jid]["interval"] = loop.call_at(
# loop.time() + 60 * interval,
# loop.create_task,
# send_update(jid)
# )
# print(await current_time(), "asyncio.get_event_loop().time()")
# print(await current_time(), asyncio.get_event_loop().time())
# await asyncio.sleep(60 * interval)
# loop.call_later(
# 60 * interval,
# loop.create_task,
# send_update(jid)
# )
# print
# await handle_event()
async def list_unread_entries(self, result, feed_title, jid):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: feed_title: {} jid: {}'
.format(function_name, feed_title, jid))
# TODO Add filtering
# TODO Do this when entry is added to list and mark it as read
# DONE!
# results = []
# if sqlite.is_setting_key(db_file, "deny"):
# while len(results) < num:
# result = cur.execute(sql).fetchone()
# blacklist = sqlite.get_setting_value(db_file, "deny").split(",")
# for i in blacklist:
# if i in result[1]:
# continue
# print("rejected:", result[1])
# print("accepted:", result[1])
# results.extend([result])
# news_list = "You've got {} news items:\n".format(num)
# NOTE Why doesn't this work without list?
# i.e. for result in results
# for result in results.fetchall():
ix = str(result[0])
title = str(result[1]) or '*** No title ***' # [No Title]
# Remove HTML tags
title = Html.remove_html_tags(title) if title else '*** No title ***'
# # TODO Retrieve summary from feed
# # See fetch.view_entry
summary = result[3]
if summary:
summary = Html.remove_html_tags(summary)
# TODO Limit text length
# summary = summary.replace("\n\n\n", "\n\n")
summary = summary.replace('\n', ' ')
summary = summary.replace(' ', ' ')
# summary = summary.replace(' ', ' ')
summary = ' '.join(summary.split())
length = Config.get_setting_value(self.settings, jid, 'length')
length = int(length)
summary = summary[:length] + " […]"
# summary = summary.strip().split('\n')
# summary = ["> " + line for line in summary]
# summary = "\n".join(summary)
else:
summary = '*** No summary ***'
link = result[2]
link = remove_tracking_parameters(link)
link = await replace_hostname(link, "link") or link
feed_id = result[4]
# news_item = ("\n{}\n{}\n{} [{}]\n").format(str(title), str(link),
# str(feed_title), str(ix))
formatting = Config.get_setting_value(self.settings, jid, 'formatting')
news_item = formatting.format(feed_title=feed_title,
title=title,
summary=summary,
link=link,
ix=ix,
feed_id=feed_id)
# news_item = news_item.replace('\\n', '\n')
return news_item
class XmppChatTask:
async def task_message(self, jid_bare):
db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self.settings, jid_bare, db_file)
while True:
update_interval = Config.get_setting_value(self.settings, jid_bare, 'interval')
update_interval = 60 * int(update_interval)
last_update_time = sqlite.get_last_update_time(db_file)
if last_update_time:
last_update_time = float(last_update_time)
diff = time.time() - last_update_time
if diff < update_interval:
next_update_time = update_interval - diff
await asyncio.sleep(next_update_time) # FIXME!
# print("jid :", jid, "\n"
# "time :", time.time(), "\n"
# "last_update_time :", last_update_time, "\n"
# "difference :", diff, "\n"
# "update interval :", update_interval, "\n"
# "next_update_time :", next_update_time, "\n"
# )
# elif diff > val:
# next_update_time = val
await sqlite.update_last_update_time(db_file)
else:
await sqlite.set_last_update_time(db_file)
await XmppChatAction.send_unread_items(self, jid_bare)
def restart_task(self, jid_bare):
if jid_bare == self.boundjid.bare:
return
if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info('Stopping task "interval" for JID {}'.format(jid_bare))
try:
self.task_manager[jid_bare]['interval'].cancel()
except:
logger.info('No task "interval" for JID {} (XmppChatTask.task_message)'
.format(jid_bare))
logger.info('Starting tasks "interval" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['interval'] = asyncio.create_task(
XmppChatTask.task_message(self, jid_bare))

View file

@ -5,12 +5,10 @@
TODO TODO
1) Use loop (with gather) instead of TaskGroup. 1) Assure message delivery before calling a new task.
2) Assure message delivery before calling a new task.
See https://slixmpp.readthedocs.io/en/latest/event_index.html#term-marker_acknowledged See https://slixmpp.readthedocs.io/en/latest/event_index.html#term-marker_acknowledged
3) XHTTML-IM 2) XHTTML-IM
case _ if message_lowercase.startswith("html"): case _ if message_lowercase.startswith("html"):
message['html']=" message['html']="
Parse me! Parse me!
@ -36,8 +34,6 @@ from datetime import datetime
import os import os
from feedparser import parse from feedparser import parse
import slixmpp import slixmpp
import slixfeed.task as task
from slixfeed.url import join_url, trim_url
# from slixmpp.plugins.xep_0363.http_upload import FileTooBig, HTTPError, UploadServiceNotFound # from slixmpp.plugins.xep_0363.http_upload import FileTooBig, HTTPError, UploadServiceNotFound
# from slixmpp.plugins.xep_0402 import BookmarkStorage, Conference # from slixmpp.plugins.xep_0402 import BookmarkStorage, Conference
# from slixmpp.plugins.xep_0048.stanza import Bookmarks # from slixmpp.plugins.xep_0048.stanza import Bookmarks
@ -46,32 +42,34 @@ from slixfeed.url import join_url, trim_url
# import xml.etree.ElementTree as ET # import xml.etree.ElementTree as ET
# from lxml import etree # from lxml import etree
import slixfeed.action as action
import slixfeed.config as config import slixfeed.config as config
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.crawl as crawl import slixfeed.crawl as crawl
import slixfeed.dt as dt import slixfeed.dt as dt
import slixfeed.fetch as fetch import slixfeed.fetch as fetch
from slixfeed.log import Logger from slixfeed.log import Logger
from slixfeed.opml import Opml
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
from slixfeed.syndication import Feed, FeedTask, Opml
import slixfeed.url as uri import slixfeed.url as uri
from slixfeed.utilities import Html, Task, Utilities
from slixfeed.version import __version__ from slixfeed.version import __version__
from slixfeed.xmpp.bookmark import XmppBookmark from slixfeed.xmpp.bookmark import XmppBookmark
from slixfeed.xmpp.chat import Chat from slixfeed.xmpp.chat import XmppChat, XmppChatTask
from slixfeed.xmpp.connect import XmppConnect from slixfeed.xmpp.connect import XmppConnect, XmppConnectTask
from slixfeed.xmpp.ipc import XmppIpcServer from slixfeed.xmpp.ipc import XmppIpcServer
from slixfeed.xmpp.iq import XmppIQ from slixfeed.xmpp.iq import XmppIQ
from slixfeed.xmpp.message import XmppMessage from slixfeed.xmpp.message import XmppMessage
from slixfeed.xmpp.muc import XmppGroupchat from slixfeed.xmpp.muc import XmppMuc
from slixfeed.xmpp.groupchat import XmppGroupchat
from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.privilege import is_moderator, is_operator, is_access from slixfeed.xmpp.privilege import is_operator, is_access
import slixfeed.xmpp.profile as profile import slixfeed.xmpp.profile as profile
from slixfeed.xmpp.publish import XmppPubsub from slixfeed.xmpp.publish import XmppPubsub, XmppPubsubAction, XmppPubsubTask
from slixfeed.xmpp.roster import XmppRoster from slixfeed.xmpp.roster import XmppRoster
# import slixfeed.xmpp.service as service # import slixfeed.xmpp.service as service
from slixfeed.xmpp.status import XmppStatusTask
from slixfeed.xmpp.upload import XmppUpload from slixfeed.xmpp.upload import XmppUpload
from slixfeed.xmpp.utility import get_chat_type from slixfeed.xmpp.utilities import XmppUtilities
import sys import sys
import time import time
@ -80,13 +78,6 @@ try:
except: except:
import tomli as tomllib import tomli as tomllib
main_task = []
jid_tasker = {}
task_manager = {}
loop = asyncio.get_event_loop()
# asyncio.set_event_loop(loop)
# time_now = datetime.now() # time_now = datetime.now()
# time_now = time_now.strftime("%H:%M:%S") # time_now = time_now.strftime("%H:%M:%S")
@ -230,10 +221,6 @@ class XmppClient(slixmpp.ClientXMPP):
self.on_presence_subscribed) self.on_presence_subscribed)
self.add_event_handler("presence_unsubscribed", self.add_event_handler("presence_unsubscribed",
self.on_presence_unsubscribed) self.on_presence_unsubscribed)
# Initialize event loop
# self.loop = asyncio.get_event_loop()
self.add_event_handler('connection_failed', self.add_event_handler('connection_failed',
self.on_connection_failed) self.on_connection_failed)
self.add_event_handler('session_end', self.add_event_handler('session_end',
@ -252,7 +239,7 @@ class XmppClient(slixmpp.ClientXMPP):
message_log = '{}: jid_full: {}' message_log = '{}: jid_full: {}'
logger.debug(message_log.format(function_name, jid_full)) logger.debug(message_log.format(function_name, jid_full))
muc_jid = message['groupchat_invite']['jid'] muc_jid = message['groupchat_invite']['jid']
result = await XmppGroupchat.join(self, muc_jid) result = await XmppMuc.join(self, muc_jid)
if result == 'ban': if result == 'ban':
message_body = '{} is banned from {}'.format(self.alias, muc_jid) message_body = '{} is banned from {}'.format(self.alias, muc_jid)
jid_bare = message['from'].bare jid_bare = message['from'].bare
@ -290,7 +277,7 @@ class XmppClient(slixmpp.ClientXMPP):
message_log = '{}: jid_full: {}' message_log = '{}: jid_full: {}'
logger.debug(message_log.format(function_name, jid_full)) logger.debug(message_log.format(function_name, jid_full))
muc_jid = message['groupchat_invite']['jid'] muc_jid = message['groupchat_invite']['jid']
result = await XmppGroupchat.join(self, muc_jid) result = await XmppMuc.join(self, muc_jid)
if result == 'ban': if result == 'ban':
message_body = '{} is banned from {}'.format(self.alias, muc_jid) message_body = '{} is banned from {}'.format(self.alias, muc_jid)
jid_bare = message['from'].bare jid_bare = message['from'].bare
@ -342,21 +329,19 @@ class XmppClient(slixmpp.ClientXMPP):
await self['xep_0115'].update_caps() await self['xep_0115'].update_caps()
# self.send_presence() # self.send_presence()
await self.get_roster() await self.get_roster()
# XmppCommand.adhoc_commands(self)
# self.service_reactions() # self.service_reactions()
task.task_ping(self) XmppConnectTask.ping(self)
# NOTE This might take more memory due to # results = await XmppPubsub.get_pubsub_services(self)
# function sqlite.get_unread_entries_of_feed # for result in results + [{'jid' : self.boundjid.bare,
results = await XmppPubsub.get_pubsub_services(self) # 'name' : self.alias}]:
for result in results + [{'jid' : self.boundjid.bare, # jid_bare = result['jid']
'name' : self.alias}]: # if jid_bare not in self.settings:
jid_bare = result['jid'] # db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings: # Config.add_settings_jid(self.settings, jid_bare, db_file)
db_file = config.get_pathname_to_database(jid_bare) # await FeedTask.check_updates(self, jid_bare)
Config.add_settings_jid(self.settings, jid_bare, db_file) # XmppPubsubTask.task_publish(self, jid_bare)
await task.start_tasks_xmpp_pubsub(self, jid_bare)
bookmarks = await XmppBookmark.get_bookmarks(self) bookmarks = await XmppBookmark.get_bookmarks(self)
await action.xmpp_muc_autojoin(self, bookmarks) await XmppGroupchat.autojoin(self, bookmarks)
if 'ipc' in self.settings and self.settings['ipc']['bsd']: if 'ipc' in self.settings and self.settings['ipc']['bsd']:
# Start Inter-Process Communication # Start Inter-Process Communication
print('POSIX sockets: Initiating IPC server...') print('POSIX sockets: Initiating IPC server...')
@ -376,7 +361,7 @@ class XmppClient(slixmpp.ClientXMPP):
profile.set_identity(self, 'client') profile.set_identity(self, 'client')
self['xep_0115'].update_caps() self['xep_0115'].update_caps()
bookmarks = await XmppBookmark.get_bookmarks(self) bookmarks = await XmppBookmark.get_bookmarks(self)
await action.xmpp_muc_autojoin(self, bookmarks) await XmppGroupchat.autojoin(self, bookmarks)
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
@ -419,7 +404,7 @@ class XmppClient(slixmpp.ClientXMPP):
XmppPresence.send(self, jid_bare, status_message) XmppPresence.send(self, jid_bare, status_message)
else: else:
# TODO Request for subscription # TODO Request for subscription
# if (await get_chat_type(self, jid_bare) == 'chat' and # if (await XmppUtilities.get_chat_type(self, jid_bare) == 'chat' and
# not self.client_roster[jid_bare]['to']): # not self.client_roster[jid_bare]['to']):
# XmppPresence.subscription(self, jid_bare, 'subscribe') # XmppPresence.subscription(self, jid_bare, 'subscribe')
# await XmppRoster.add(self, jid_bare) # await XmppRoster.add(self, jid_bare)
@ -434,7 +419,7 @@ class XmppClient(slixmpp.ClientXMPP):
self.pending_tasks[jid_bare] = {} self.pending_tasks[jid_bare] = {}
# if jid_full not in self.pending_tasks: # if jid_full not in self.pending_tasks:
# self.pending_tasks[jid_full] = {} # self.pending_tasks[jid_full] = {}
await Chat.process_message(self, message) await XmppChat.process_message(self, message)
# chat_type = message["type"] # chat_type = message["type"]
# message_body = message["body"] # message_body = message["body"]
# message_reply = message.reply # message_reply = message.reply
@ -455,10 +440,14 @@ class XmppClient(slixmpp.ClientXMPP):
if jid_bare in self.boundjid.bare: if jid_bare in self.boundjid.bare:
return return
if presence['show'] in ('away', 'dnd', 'xa'): if presence['show'] in ('away', 'dnd', 'xa'):
key_list = ['interval'] if (jid_bare in self.task_manager and
task.clean_tasks_xmpp_chat(self, jid_bare, key_list) 'interval' in self.task_manager[jid_bare]):
key_list = ['status', 'check'] self.task_manager[jid_bare]['interval'].cancel()
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) else:
logger.debug('No task "interval" for JID {} (on_changed_status)'
.format(jid_bare))
XmppStatusTask.restart_task(self, jid_bare)
FeedTask.restart_task(self, jid_bare)
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
@ -516,16 +505,15 @@ class XmppClient(slixmpp.ClientXMPP):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
message_log = '{}: jid_full: {}' message_log = '{}: jid_full: {}'
logger.debug(message_log.format(function_name, jid_full)) logger.debug(message_log.format(function_name, jid_full))
# TODO Add function to check whether task is already running or not
# await task.start_tasks(self, presence)
# NOTE Already done inside the start-task function
jid_bare = presence['from'].bare jid_bare = presence['from'].bare
if jid_bare in self.boundjid.bare: if jid_bare in self.boundjid.bare:
return return
# FIXME TODO Find out what is the source responsible for a couple presences with empty message # FIXME TODO Find out what is the source responsible for a couple presences with empty message
# NOTE This is a temporary solution # NOTE This is a temporary solution
await asyncio.sleep(10) await asyncio.sleep(10)
await task.start_tasks_xmpp_chat(self, jid_bare) FeedTask.restart_task(self, jid_bare)
XmppChatTask.restart_task(self, jid_bare)
XmppStatusTask.restart_task(self, jid_bare)
self.add_event_handler("presence_unavailable", self.add_event_handler("presence_unavailable",
self.on_presence_unavailable) self.on_presence_unavailable)
time_end = time.time() time_end = time.time()
@ -563,8 +551,8 @@ class XmppClient(slixmpp.ClientXMPP):
message_log = '{}: jid_full: {}' message_log = '{}: jid_full: {}'
logger.debug(message_log.format(function_name, jid_full)) logger.debug(message_log.format(function_name, jid_full))
jid_bare = presence['from'].bare jid_bare = presence['from'].bare
# await task.stop_tasks(self, jid) for task in ('check', 'interval', 'status'):
task.clean_tasks_xmpp_chat(self, jid_bare) Task.stop(self, jid_bare, 'status')
# NOTE Albeit nice to ~have~ see, this would constantly # NOTE Albeit nice to ~have~ see, this would constantly
# send presence messages to server to no end. # send presence messages to server to no end.
@ -591,7 +579,8 @@ class XmppClient(slixmpp.ClientXMPP):
message_log = '{}: jid_full: {}' message_log = '{}: jid_full: {}'
logger.debug(message_log.format(function_name, jid_full)) logger.debug(message_log.format(function_name, jid_full))
jid_bare = presence["from"].bare jid_bare = presence["from"].bare
task.clean_tasks_xmpp_chat(self, jid_bare) for task in ('check', 'interval', 'status'):
Task.stop(self, jid_bare, 'status')
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
@ -618,8 +607,7 @@ class XmppClient(slixmpp.ClientXMPP):
# self.send_presence(pto=jid) # self.send_presence(pto=jid)
# task.clean_tasks_xmpp_chat(self, jid, ['status']) # task.clean_tasks_xmpp_chat(self, jid, ['status'])
await asyncio.sleep(5) await asyncio.sleep(5)
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 10: logger.warning('{} (time: {})'.format(function_name, if difference > 10: logger.warning('{} (time: {})'.format(function_name,
@ -627,6 +615,7 @@ class XmppClient(slixmpp.ClientXMPP):
async def on_chatstate_composing(self, message): async def on_chatstate_composing(self, message):
# print('on_chatstate_composing START')
time_begin = time.time() time_begin = time.time()
jid_full = str(message['from']) jid_full = str(message['from'])
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
@ -642,13 +631,14 @@ class XmppClient(slixmpp.ClientXMPP):
status_message = ('💡 Send "help" for manual, or "info" for ' status_message = ('💡 Send "help" for manual, or "info" for '
'information.') 'information.')
XmppPresence.send(self, jid_bare, status_message) XmppPresence.send(self, jid_bare, status_message)
# print('on_chatstate_composing FINISH')
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
difference)) difference))
async def on_chatstate_gone(self, message): def on_chatstate_gone(self, message):
time_begin = time.time() time_begin = time.time()
jid_full = str(message['from']) jid_full = str(message['from'])
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
@ -658,16 +648,14 @@ class XmppClient(slixmpp.ClientXMPP):
if jid_bare in self.boundjid.bare: if jid_bare in self.boundjid.bare:
return return
if message['type'] in ('chat', 'normal'): if message['type'] in ('chat', 'normal'):
# task.clean_tasks_xmpp_chat(self, jid, ['status']) XmppStatusTask.restart_task(self, jid_bare)
key_list = ['status']
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
difference)) difference))
async def on_chatstate_inactive(self, message): def on_chatstate_inactive(self, message):
time_begin = time.time() time_begin = time.time()
jid_full = str(message['from']) jid_full = str(message['from'])
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
@ -677,16 +665,14 @@ class XmppClient(slixmpp.ClientXMPP):
if jid_bare in self.boundjid.bare: if jid_bare in self.boundjid.bare:
return return
if message['type'] in ('chat', 'normal'): if message['type'] in ('chat', 'normal'):
# task.clean_tasks_xmpp_chat(self, jid, ['status']) XmppStatusTask.restart_task(self, jid_bare)
key_list = ['status']
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
difference)) difference))
async def on_chatstate_paused(self, message): def on_chatstate_paused(self, message):
time_begin = time.time() time_begin = time.time()
jid_full = str(message['from']) jid_full = str(message['from'])
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
@ -696,9 +682,7 @@ class XmppClient(slixmpp.ClientXMPP):
if jid_bare in self.boundjid.bare: if jid_bare in self.boundjid.bare:
return return
if message['type'] in ('chat', 'normal'): if message['type'] in ('chat', 'normal'):
# task.clean_tasks_xmpp_chat(self, jid, ['status']) XmppStatusTask.restart_task(self, jid_bare)
key_list = ['status']
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
time_end = time.time() time_end = time.time()
difference = time_end - time_begin difference = time_end - time_begin
if difference > 1: logger.warning('{} (time: {})'.format(function_name, if difference > 1: logger.warning('{} (time: {})'.format(function_name,
@ -857,7 +841,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
form = self['xep_0004'].make_form('form', 'PubSub') form = self['xep_0004'].make_form('form', 'PubSub')
form['instructions'] = 'Publish news items to PubSub nodes.' form['instructions'] = 'Publish news items to PubSub nodes.'
@ -898,7 +882,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
values = payload['values'] values = payload['values']
form = self['xep_0004'].make_form('form', 'Publish') form = self['xep_0004'].make_form('form', 'Publish')
@ -1095,7 +1079,7 @@ class XmppClient(slixmpp.ClientXMPP):
# xep = None # xep = None
for ix in ixs: for ix in ixs:
await action.xmpp_pubsub_send_selected_entry(self, jid, jid_bare, node_id, ix) await XmppPubsubAction.send_selected_entry(self, jid, node_id, ix)
text_info = 'Posted {} entries.'.format(len(ixs)) text_info = 'Posted {} entries.'.format(len(ixs))
session['allow_prev'] = False session['allow_prev'] = False
session['has_next'] = False session['has_next'] = False
@ -1143,7 +1127,7 @@ class XmppClient(slixmpp.ClientXMPP):
if not result['error']: if not result['error']:
document = result['content'] document = result['content']
feed = parse(document) feed = parse(document)
if action.is_feed(url, feed): if Feed.is_feed(url, feed):
form['instructions'] = 'Select entries to publish.' form['instructions'] = 'Select entries to publish.'
options = form.add_field(desc='Select entries to post.', options = form.add_field(desc='Select entries to post.',
ftype='list-multi', ftype='list-multi',
@ -1408,7 +1392,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
jid = session['from'].bare jid = session['from'].bare
db_file = config.get_pathname_to_database(jid_bare) db_file = config.get_pathname_to_database(jid_bare)
@ -1517,7 +1501,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
form = self['xep_0004'].make_form('form', 'Subscribe') form = self['xep_0004'].make_form('form', 'Subscribe')
# form['instructions'] = 'Add a new custom subscription.' # form['instructions'] = 'Add a new custom subscription.'
@ -1714,7 +1698,7 @@ class XmppClient(slixmpp.ClientXMPP):
# summary = action.get_document_content_as_text(data) # summary = action.get_document_content_as_text(data)
summary = sqlite.get_entry_summary(db_file, ix) summary = sqlite.get_entry_summary(db_file, ix)
summary = summary[0] summary = summary[0]
summary = action.remove_html_tags(summary) if summary else 'No content to show.' summary = Html.remove_html_tags(summary) if summary else 'No content to show.'
form.add_field(ftype="text-multi", form.add_field(ftype="text-multi",
label='Article', label='Article',
value=summary) value=summary)
@ -1821,7 +1805,7 @@ class XmppClient(slixmpp.ClientXMPP):
identifier = hostname + ':' + str(counter) identifier = hostname + ':' + str(counter)
else: else:
break break
result = await action.add_feed(self, jid_bare, db_file, url, result = await Feed.add_feed(self, jid_bare, db_file, url,
identifier) identifier)
if result['error']: if result['error']:
error_count += 1 error_count += 1
@ -1854,7 +1838,7 @@ class XmppClient(slixmpp.ClientXMPP):
identifier = hostname + ':' + str(counter) identifier = hostname + ':' + str(counter)
else: else:
break break
result = await action.add_feed(self, jid_bare, db_file, url, result = await Feed.add_feed(self, jid_bare, db_file, url,
identifier) identifier)
# URL is not a feed and URL has returned to feeds # URL is not a feed and URL has returned to feeds
if isinstance(result, list): if isinstance(result, list):
@ -2037,7 +2021,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
form = self['xep_0004'].make_form('form', 'Discover & Search') form = self['xep_0004'].make_form('form', 'Discover & Search')
form['instructions'] = 'Discover news subscriptions of all kinds' form['instructions'] = 'Discover news subscriptions of all kinds'
@ -2161,7 +2145,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
form = self['xep_0004'].make_form('form', 'Subscriptions') form = self['xep_0004'].make_form('form', 'Subscriptions')
form['instructions'] = ('Browse, view, toggle or remove ' form['instructions'] = ('Browse, view, toggle or remove '
@ -2521,7 +2505,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
form = self['xep_0004'].make_form('form', 'Advanced') form = self['xep_0004'].make_form('form', 'Advanced')
form['instructions'] = 'Extended options' form['instructions'] = 'Extended options'
@ -2905,9 +2889,9 @@ class XmppClient(slixmpp.ClientXMPP):
# form['instructions'] = ('✅️ Feeds have been exported') # form['instructions'] = ('✅️ Feeds have been exported')
exts = values['filetype'] exts = values['filetype']
for ext in exts: for ext in exts:
filename = action.export_feeds(self, jid_bare, ext) filename = Feed.export_feeds(jid_bare, ext)
url = await XmppUpload.start(self, jid_bare, filename) url = await XmppUpload.start(self, jid_bare, filename)
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
XmppMessage.send_oob(self, jid_bare, url, chat_type) XmppMessage.send_oob(self, jid_bare, url, chat_type)
url_field = form.add_field(var=ext.upper(), url_field = form.add_field(var=ext.upper(),
ftype='text-single', ftype='text-single',
@ -2930,12 +2914,12 @@ class XmppClient(slixmpp.ClientXMPP):
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
jid_full = str(session['from']) jid_full = str(session['from'])
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
form = self['xep_0004'].make_form('form', 'Subscribe') form = self['xep_0004'].make_form('form', 'Subscribe')
# NOTE Refresh button would be of use # NOTE Refresh button would be of use
form['instructions'] = 'Featured subscriptions' form['instructions'] = 'Featured subscriptions'
url = action.pick_a_feed() url = Utilities.pick_a_feed()
# options = form.add_field(desc='Click to subscribe.', # options = form.add_field(desc='Click to subscribe.',
# ftype="boolean", # ftype="boolean",
# label='Subscribe to {}?'.format(url['name']), # label='Subscribe to {}?'.format(url['name']),
@ -2948,7 +2932,7 @@ class XmppClient(slixmpp.ClientXMPP):
label='Subscribe', label='Subscribe',
var='subscription') var='subscription')
for i in range(10): for i in range(10):
url = action.pick_a_feed() url = Utilities.pick_a_feed()
options.addOption(url['name'], url['link']) options.addOption(url['name'], url['link'])
# jid_bare = session['from'].bare # jid_bare = session['from'].bare
if '@' in jid_bare: if '@' in jid_bare:
@ -3114,7 +3098,7 @@ class XmppClient(slixmpp.ClientXMPP):
var=jid_bare) var=jid_bare)
session['allow_complete'] = True session['allow_complete'] = True
session['has_next'] = False session['has_next'] = False
session['next'] = self._handle_pubsubs_complete session['next'] = self._handle_pubsub_complete
# session['allow_prev'] = True # session['allow_prev'] = True
session['payload'] = form session['payload'] = form
# session['prev'] = self._handle_advanced # session['prev'] = self._handle_advanced
@ -3256,11 +3240,12 @@ class XmppClient(slixmpp.ClientXMPP):
content = '' content = ''
# TODO Check whether element of type Atom # TODO Check whether element of type Atom
# NOTE Consider pubsub#type of XEP-0462: PubSub Type Filtering
atom_entry = iq['pubsub']['items']['item']['payload'] atom_entry = iq['pubsub']['items']['item']['payload']
for element in atom_entry: for element in atom_entry:
if element.text: if element.text:
content += element.text + '\n\n' content += element.text + '\n\n'
# content += action.remove_html_tags(element.text) + '\n\n' # content += Html.remove_html_tags(element.text) + '\n\n'
if element.attrib: if element.attrib:
for i in element.attrib: for i in element.attrib:
content += element.attrib[i] + '\n\n' content += element.attrib[i] + '\n\n'
@ -3273,6 +3258,7 @@ class XmppClient(slixmpp.ClientXMPP):
session['payload'] = form session['payload'] = form
return session return session
# FIXME Undefined name 'jid_bare'
async def _handle_node_edit(self, payload, session): async def _handle_node_edit(self, payload, session):
jid_full = str(session['from']) jid_full = str(session['from'])
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
@ -3282,6 +3268,7 @@ class XmppClient(slixmpp.ClientXMPP):
jid = values['jid'][0] jid = values['jid'][0]
node = values['node'] node = values['node']
properties = await XmppPubsub.get_node_properties(self, jid, node) properties = await XmppPubsub.get_node_properties(self, jid, node)
form = self['xep_0004'].make_form('form', 'PubSub')
form['instructions'] = 'Editing bookmark' form['instructions'] = 'Editing bookmark'
jid_split = properties['jid'].split('@') jid_split = properties['jid'].split('@')
room = jid_split[0] room = jid_split[0]
@ -3361,7 +3348,7 @@ class XmppClient(slixmpp.ClientXMPP):
return session return session
async def _handle_pubsubs_complete(self, payload, session): async def _handle_pubsub_complete(self, payload, session):
jid_full = str(session['from']) jid_full = str(session['from'])
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
@ -3632,7 +3619,7 @@ class XmppClient(slixmpp.ClientXMPP):
logger.debug('{}: jid_full: {}' logger.debug('{}: jid_full: {}'
.format(function_name, jid_full)) .format(function_name, jid_full))
jid_bare = session['from'].bare jid_bare = session['from'].bare
chat_type = await get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if is_access(self, jid_bare, jid_full, chat_type): if is_access(self, jid_bare, jid_full, chat_type):
db_file = config.get_pathname_to_database(jid_bare) db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings: if jid_bare not in self.settings:
@ -3783,15 +3770,16 @@ class XmppClient(slixmpp.ClientXMPP):
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
status_type=status_type) status_type=status_type)
await asyncio.sleep(5) await asyncio.sleep(5)
key_list = ['check', 'status', 'interval'] FeedTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) XmppChatTask.restart_task(self, jid_bare)
XmppStatusTask.restart_task(self, jid_bare)
if (key == 'enabled' and if (key == 'enabled' and
val == 0 and val == 0 and
str(is_enabled) == 1): str(is_enabled) == 1):
logger.info('Slixfeed has been disabled for {}'.format(jid_bare)) logger.info('Slixfeed has been disabled for {}'.format(jid_bare))
key_list = ['interval', 'status'] for task in ('interval', 'status'):
task.clean_tasks_xmpp_chat(self, jid_bare, key_list) Task.stop(self, jid_bare, 'status')
status_type = 'xa' status_type = 'xa'
status_message = '📪️ Send "Start" to receive updates' status_message = '📪️ Send "Start" to receive updates'
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
@ -3800,22 +3788,6 @@ class XmppClient(slixmpp.ClientXMPP):
await Config.set_setting_value(self.settings, jid_bare, db_file, key, val) await Config.set_setting_value(self.settings, jid_bare, db_file, key, val)
val = self.settings[jid_bare][key] val = self.settings[jid_bare][key]
# if key == 'enabled':
# if str(setting.enabled) == 0:
# status_type = 'available'
# status_message = '📫️ Welcome back!'
# XmppPresence.send(self, jid, status_message,
# status_type=status_type)
# await asyncio.sleep(5)
# await task.start_tasks_xmpp_chat(self, jid, ['check', 'status',
# 'interval'])
# else:
# task.clean_tasks_xmpp_chat(self, jid, ['interval', 'status'])
# status_type = 'xa'
# status_message = '📪️ Send "Start" to receive Jabber updates'
# XmppPresence.send(self, jid, status_message,
# status_type=status_type)
if key in ('enabled', 'media', 'old'): if key in ('enabled', 'media', 'old'):
if val == '1': if val == '1':
val = 'Yes' val = 'Yes'
@ -3828,17 +3800,6 @@ class XmppClient(slixmpp.ClientXMPP):
val = int(val) val = int(val)
val = str(val) val = str(val)
# match value:
# case 'enabled':
# pass
# case 'interval':
# pass
# result = '{}: {}'.format(key.capitalize(), val)
# form.add_field(var=key,
# ftype='fixed',
# label=result)
form = payload form = payload
form['title'] = 'Done' form['title'] = 'Done'
form['instructions'] = 'has been completed!' form['instructions'] = 'has been completed!'

View file

@ -1,28 +1,25 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import asyncio
from feedparser import parse from feedparser import parse
from random import randrange from random import randrange
import slixfeed.action as action
import slixfeed.config as config import slixfeed.config as config
import slixfeed.crawl as crawl
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.crawl as crawl
import slixfeed.dt as dt import slixfeed.dt as dt
import slixfeed.fetch as fetch import slixfeed.fetch as fetch
from slixfeed.opml import Opml from slixfeed.log import Logger
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
import slixfeed.task as task from slixfeed.syndication import Feed, Opml
import slixfeed.url as uri import slixfeed.url as uri
from slixfeed.utilities import Documentation, Utilities
from slixfeed.version import __version__ from slixfeed.version import __version__
from slixfeed.xmpp.bookmark import XmppBookmark from slixfeed.xmpp.bookmark import XmppBookmark
from slixfeed.log import Logger from slixfeed.xmpp.muc import XmppMuc
from slixfeed.xmpp.muc import XmppGroupchat from slixfeed.xmpp.publish import XmppPubsub, XmppPubsubAction
from slixfeed.xmpp.message import XmppMessage
from slixfeed.xmpp.publish import XmppPubsub
from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.upload import XmppUpload from slixfeed.xmpp.status import XmppStatusTask
from slixfeed.xmpp.utility import get_chat_type from slixfeed.xmpp.utilities import XmppUtilities
import sys import sys
try: try:
@ -44,20 +41,20 @@ class XmppCommands:
def print_help(): def print_help():
result = action.manual('commands.toml') result = Documentation.manual('commands.toml')
message = '\n'.join(result) message = '\n'.join(result)
return message return message
def print_help_list(): def print_help_list():
command_list = action.manual('commands.toml', section='all') command_list = Documentation.manual('commands.toml', section='all')
message = ('Complete list of commands:\n' message = ('Complete list of commands:\n'
'```\n{}\n```'.format(command_list)) '```\n{}\n```'.format(command_list))
return message return message
def print_help_specific(command_root, command_name): def print_help_specific(command_root, command_name):
command_list = action.manual('commands.toml', command_list = Documentation.manual('commands.toml',
section=command_root, section=command_root,
command=command_name) command=command_name)
if command_list: if command_list:
@ -69,7 +66,7 @@ class XmppCommands:
def print_help_key(command): def print_help_key(command):
command_list = action.manual('commands.toml', command) command_list = Documentation.manual('commands.toml', command)
if command_list: if command_list:
command_list = ' '.join(command_list) command_list = ' '.join(command_list)
message = ('Available command `{}` keys:\n' message = ('Available command `{}` keys:\n'
@ -146,22 +143,24 @@ class XmppCommands:
document = result['content'] document = result['content']
feed = parse(document) feed = parse(document)
feed_valid = 0 if feed.bozo else 1 feed_valid = 0 if feed.bozo else 1
await sqlite.update_feed_validity(db_file, feed_id, feed_valid) await sqlite.update_feed_validity(
db_file, feed_id, feed_valid)
if feed.has_key('updated_parsed'): if feed.has_key('updated_parsed'):
feed_updated = feed.updated_parsed feed_updated = feed.updated_parsed
try: try:
feed_updated = dt.convert_struct_time_to_iso8601(feed_updated) feed_updated = dt.convert_struct_time_to_iso8601(
feed_updated)
except: except:
feed_updated = None feed_updated = None
else: else:
feed_updated = None feed_updated = None
feed_properties = action.get_properties_of_feed( feed_properties = Feed.get_properties_of_feed(
db_file, feed_id, feed) db_file, feed_id, feed)
await sqlite.update_feed_properties(db_file, feed_id, await sqlite.update_feed_properties(db_file, feed_id,
feed_properties) feed_properties)
feed_id = sqlite.get_feed_id(db_file, url) feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0] feed_id = feed_id[0]
new_entries = action.get_properties_of_entries( new_entries = Feed.get_properties_of_entries(
jid_bare, db_file, url, feed_id, feed) jid_bare, db_file, url, feed_id, feed)
if new_entries: if new_entries:
await sqlite.add_entries_and_update_feed_state( await sqlite.add_entries_and_update_feed_state(
@ -179,8 +178,7 @@ class XmppCommands:
# if old: # if old:
# # task.clean_tasks_xmpp_chat(self, jid_bare, ['status']) # # task.clean_tasks_xmpp_chat(self, jid_bare, ['status'])
# # await send_status(jid) # # await send_status(jid)
# key_list = ['status'] # Task.start(self, jid_bare, 'status')
# await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
# else: # else:
# feed_id = sqlite.get_feed_id(db_file, url) # feed_id = sqlite.get_feed_id(db_file, url)
# feed_id = feed_id[0] # feed_id = feed_id[0]
@ -252,7 +250,7 @@ class XmppCommands:
message = ('Maximum archived items has been set to {} (was: {}).' message = ('Maximum archived items has been set to {} (was: {}).'
.format(val_new, val_old)) .format(val_new, val_old))
except: except:
message = ('No action has been taken. Enter a numeric value only.') message = 'No action has been taken. Enter a numeric value only.'
return message return message
@ -332,8 +330,8 @@ class XmppCommands:
await sqlite.set_filter_value(db_file, ['deny', val]) await sqlite.set_filter_value(db_file, ['deny', val])
def export_feeds(self, jid_bare, ext): def export_feeds(jid_bare, ext):
filename = action.export_feeds(self, jid_bare, ext) filename = Feed.export_feeds(jid_bare, ext)
message = 'Feeds successfuly exported to {}.'.format(ext) message = 'Feeds successfuly exported to {}.'.format(ext)
return filename, message return filename, message
@ -366,14 +364,14 @@ class XmppCommands:
# This is similar to send_next_update # This is similar to send_next_update
async def pubsub_send(self, info, jid): async def pubsub_send(self, info, jid_bare):
# if num: # if num:
# report = await action.xmpp_pubsub_send_unread_items( # report = await action.xmpp_pubsub_send_unread_items(
# self, jid, num) # self, jid, num)
# else: # else:
# report = await action.xmpp_pubsub_send_unread_items( # report = await action.xmpp_pubsub_send_unread_items(
# self, jid) # self, jid)
result = await action.xmpp_pubsub_send_unread_items(self, jid) result = await XmppPubsubAction.send_unread_items(self, jid_bare)
message = '' message = ''
for url in result: for url in result:
if result[url]: if result[url]:
@ -416,11 +414,12 @@ class XmppCommands:
# self.pending_tasks[jid_bare][self.pending_tasks_counter] = status_message # self.pending_tasks[jid_bare][self.pending_tasks_counter] = status_message
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
status_type=status_type) status_type=status_type)
if url.startswith('feed:/') or url.startswith('itpc:/') or url.startswith('rss:/'): if (url.startswith('feed:/') or
url.startswith('itpc:/') or
url.startswith('rss:/')):
url = uri.feed_to_http(url) url = uri.feed_to_http(url)
url = (await uri.replace_hostname(url, 'feed')) or url url = (await uri.replace_hostname(url, 'feed')) or url
result = await action.add_feed(self, jid_bare, result = await Feed.add_feed(self, jid_bare, db_file, url,
db_file, url,
identifier) identifier)
if isinstance(result, list): if isinstance(result, list):
results = result results = result
@ -457,8 +456,7 @@ class XmppCommands:
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
print(self.pending_tasks) print(self.pending_tasks)
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
# except: # except:
# response = ( # response = (
# '> {}\nNews source is in the process ' # '> {}\nNews source is in the process '
@ -494,8 +492,7 @@ class XmppCommands:
else: else:
break break
# try: # try:
result = await action.add_feed(self, jid_bare, db_file, url, result = await Feed.add_feed(self, jid_bare, db_file, url, identifier)
identifier)
if isinstance(result, list): if isinstance(result, list):
results = result results = result
message = ("Syndication feeds found for {}\n\n```\n" message = ("Syndication feeds found for {}\n\n```\n"
@ -547,7 +544,7 @@ class XmppCommands:
elif query: elif query:
message = "No feeds were found for: {}".format(query) message = "No feeds were found for: {}".format(query)
else: else:
url = action.pick_a_feed() url = Utilities.pick_a_feed()
message = ('List of subscriptions is empty. ' message = ('List of subscriptions is empty. '
'To add a feed, send a URL.\n' 'To add a feed, send a URL.\n'
'Featured news: *{}*\n{}' 'Featured news: *{}*\n{}'
@ -569,19 +566,16 @@ class XmppCommands:
self.settings, jid_bare, 'interval') self.settings, jid_bare, 'interval')
await Config.set_setting_value( await Config.set_setting_value(
self.settings, jid_bare, db_file, 'interval', val_new) self.settings, jid_bare, db_file, 'interval', val_new)
# NOTE Perhaps this should be replaced by functions
# clean and start
task.refresh_task(self, jid_bare,
task.task_message, 'interval', val_new)
message = ('Updates will be sent every {} minutes ' message = ('Updates will be sent every {} minutes '
'(was: {}).'.format(val_new, val_old)) '(was: {}).'.format(val_new, val_old))
except: except Exception as e:
logger.error(str(e))
message = ('No action has been taken. Enter a numeric value only.') message = ('No action has been taken. Enter a numeric value only.')
return message return message
async def muc_leave(self, jid_bare): async def muc_leave(self, jid_bare):
XmppGroupchat.leave(self, jid_bare) XmppMuc.leave(self, jid_bare)
await XmppBookmark.remove(self, jid_bare) await XmppBookmark.remove(self, jid_bare)
@ -590,7 +584,7 @@ class XmppCommands:
muc_jid = uri.check_xmpp_uri(command) muc_jid = uri.check_xmpp_uri(command)
if muc_jid: if muc_jid:
# TODO probe JID and confirm it's a groupchat # TODO probe JID and confirm it's a groupchat
result = await XmppGroupchat.join(self, muc_jid) result = await XmppMuc.join(self, muc_jid)
# await XmppBookmark.add(self, jid=muc_jid) # await XmppBookmark.add(self, jid=muc_jid)
if result == 'ban': if result == 'ban':
message = '{} is banned from {}'.format(self.alias, muc_jid) message = '{} is banned from {}'.format(self.alias, muc_jid)
@ -693,15 +687,6 @@ class XmppCommands:
return message return message
async def send_next_update(self, jid_bare, command):
"""Warning! Not to be interfaced with IPC"""
num = command[5:]
if num:
await action.xmpp_chat_send_unread_items(self, jid_bare, num)
else:
await action.xmpp_chat_send_unread_items(self, jid_bare)
def print_options(self, jid_bare): def print_options(self, jid_bare):
message = '' message = ''
for key in self.settings[jid_bare]: for key in self.settings[jid_bare]:
@ -761,8 +746,8 @@ class XmppCommands:
if not result['error']: if not result['error']:
document = result['content'] document = result['content']
feed = parse(document) feed = parse(document)
if action.is_feed(url, feed): if Feed.is_feed(url, feed):
message = action.view_feed(url, feed) message = Feed.view_feed(url, feed)
break break
else: else:
result = await crawl.probe_page(url, document) result = await crawl.probe_page(url, document)
@ -797,8 +782,8 @@ class XmppCommands:
document = result['content'] document = result['content']
status = result['status_code'] status = result['status_code']
feed = parse(document) feed = parse(document)
if action.is_feed(url, feed): if Feed.is_feed(url, feed):
message = action.view_entry(url, feed, num) message = Feed.view_entry(url, feed, num)
break break
else: else:
result = await crawl.probe_page(url, document) result = await crawl.probe_page(url, document)
@ -901,7 +886,7 @@ class XmppCommands:
return message return message
async def mark_as_read(self, jid_bare, db_file, ix_url=None): async def mark_as_read(jid_bare, db_file, ix_url=None):
if ix_url: if ix_url:
sub_marked = [] sub_marked = []
url_invalid = [] url_invalid = []
@ -941,14 +926,12 @@ class XmppCommands:
message += '\nThe following indexes do not exist:\n\n{}\n'.format(ixs) message += '\nThe following indexes do not exist:\n\n{}\n'.format(ixs)
message += '\n```' message += '\n```'
else: else:
message = ('No action has been taken.' await sqlite.mark_all_as_read(db_file)
'\n' message = 'All subscriptions have been marked as read.'
'Missing argument. '
'Enter a subscription URL or index number.')
return message return message
async def search_items(self, db_file, query): async def search_items(db_file, query):
if query: if query:
if len(query) > 3: if len(query) > 3:
results = sqlite.search_entries(db_file, query) results = sqlite.search_entries(db_file, query)
@ -970,10 +953,12 @@ class XmppCommands:
return message return message
async def scheduler_start(self, db_file, jid_bare): # Tasks are classes which are passed to this function
# On an occasion in which they would have returned, variable "tasks" might be called "callback"
async def scheduler_start(self, db_file, jid_bare, tasks):
await Config.set_setting_value(self.settings, jid_bare, db_file, 'enabled', 1) await Config.set_setting_value(self.settings, jid_bare, db_file, 'enabled', 1)
key_list = ['check', 'status', 'interval'] for task in tasks:
await task.start_tasks_xmpp_chat(self, jid_bare, key_list) task.restart_task(self, jid_bare)
message = 'Updates are enabled.' message = 'Updates are enabled.'
return message return message
@ -981,8 +966,13 @@ class XmppCommands:
async def scheduler_stop(self, db_file, jid_bare): async def scheduler_stop(self, db_file, jid_bare):
await Config.set_setting_value( await Config.set_setting_value(
self.settings, jid_bare, db_file, 'enabled', 0) self.settings, jid_bare, db_file, 'enabled', 0)
key_list = ['interval', 'status'] for task in ('interval', 'status'):
task.clean_tasks_xmpp_chat(self, jid_bare, key_list) if (jid_bare in self.task_manager and
task in self.task_manager[jid_bare]):
self.task_manager[jid_bare][task].cancel()
else:
logger.debug('No task {} for JID {} (Task.stop)'
.format(task, jid_bare))
message = 'Updates are disabled.' message = 'Updates are disabled.'
return message return message
@ -1037,8 +1027,7 @@ class XmppCommands:
except: except:
message = ('No action has been taken. No news source with index {}.' message = ('No action has been taken. No news source with index {}.'
.format(feed_id)) .format(feed_id))
key_list = ['status'] XmppStatusTask.restart_task(self, jid_bare)
await task.start_tasks_xmpp_chat(self, jid_bare, key_list)
return message return message
@ -1102,7 +1091,7 @@ class XmppCommands:
async def invite_jid_to_muc(self, jid_bare): async def invite_jid_to_muc(self, jid_bare):
muc_jid = 'slixfeed@chat.woodpeckersnest.space' muc_jid = 'slixfeed@chat.woodpeckersnest.space'
if await get_chat_type(self, jid_bare) == 'chat': if await XmppUtilities.get_chat_type(self, jid_bare) == 'chat':
self.plugin['xep_0045'].invite(muc_jid, jid_bare) self.plugin['xep_0045'].invite(muc_jid, jid_bare)

View file

@ -15,9 +15,11 @@ TODO
import asyncio import asyncio
from slixfeed.dt import current_time from slixfeed.dt import current_time
from slixfeed.log import Logger
from slixmpp.exceptions import IqTimeout, IqError from slixmpp.exceptions import IqTimeout, IqError
from time import sleep from time import sleep
import logging
logger = Logger(__name__)
class XmppConnect: class XmppConnect:
@ -45,21 +47,21 @@ class XmppConnect:
rtt = await self['xep_0199'].ping(jid, rtt = await self['xep_0199'].ping(jid,
ifrom=jid_from, ifrom=jid_from,
timeout=10) timeout=10)
logging.info('Success! RTT: %s', rtt) logger.info('Success! RTT: %s', rtt)
except IqError as e: except IqError as e:
logging.error('Error pinging %s: %s', jid, logger.error('Error pinging %s: %s', jid,
e.iq['error']['condition']) e.iq['error']['condition'])
except IqTimeout: except IqTimeout:
logging.warning('No response from %s', jid) logger.warning('No response from %s', jid)
if not rtt: if not rtt:
logging.warning('Disconnecting...') logger.warning('Disconnecting...')
self.disconnect() self.disconnect()
break break
await asyncio.sleep(60 * 1) await asyncio.sleep(60 * 1)
def recover(self, message): def recover(self, message):
logging.warning(message) logger.warning(message)
print(current_time(), message, 'Attempting to reconnect.') print(current_time(), message, 'Attempting to reconnect.')
self.connection_attempts += 1 self.connection_attempts += 1
# if self.connection_attempts <= self.max_connection_attempts: # if self.connection_attempts <= self.max_connection_attempts:
@ -78,10 +80,20 @@ class XmppConnect:
def inspect(self): def inspect(self):
print('Disconnected\n' print('Disconnected\nReconnecting...')
'Reconnecting...')
try: try:
self.reconnect self.reconnect
except: except:
self.disconnect() self.disconnect()
print('Problem reconnecting') print('Problem reconnecting')
class XmppConnectTask:
def ping(self):
try:
self.task_ping_instance.cancel()
except:
logger.info('No ping task to cancel.')
self.task_ping_instance = asyncio.create_task(XmppConnect.ping(self))

View file

@ -0,0 +1,54 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
TODO
1) Send message to inviter that bot has joined to groupchat.
2) If groupchat requires captcha, send the consequent message.
3) If groupchat error is received, send that error message to inviter.
FIXME
1) Save name of groupchat instead of jid as name
"""
from slixfeed.xmpp.bookmark import XmppBookmark
from slixfeed.xmpp.muc import XmppMuc
from slixfeed.log import Logger
logger = Logger(__name__)
class XmppGroupchat:
async def autojoin(self, bookmarks):
for bookmark in bookmarks:
if bookmark["jid"] and bookmark["autojoin"]:
if not bookmark["nick"]:
bookmark["nick"] = self.alias
logger.error('Alias (i.e. Nicknname) is missing for '
'bookmark {}'.format(bookmark['name']))
alias = bookmark["nick"]
muc_jid = bookmark["jid"]
result = await XmppMuc.join(self, muc_jid, alias)
print(result)
if result == 'ban':
await XmppBookmark.remove(self, muc_jid)
logger.warning('{} is banned from {}'.format(self.alias, muc_jid))
logger.warning('Groupchat {} has been removed from bookmarks'
.format(muc_jid))
else:
logger.info('Autojoin groupchat\n'
'Name : {}\n'
'JID : {}\n'
'Alias : {}\n'
.format(bookmark["name"],
bookmark["jid"],
bookmark["nick"]))
elif not bookmark["jid"]:
logger.error('JID is missing for bookmark {}'
.format(bookmark['name']))

View file

@ -12,7 +12,11 @@ socket (i.e. clients[fd]) from the respective client.
import asyncio import asyncio
import os import os
import slixfeed.config as config import slixfeed.config as config
from slixfeed.syndication import FeedTask
from slixfeed.xmpp.chat import XmppChatTask
from slixfeed.xmpp.commands import XmppCommands from slixfeed.xmpp.commands import XmppCommands
from slixfeed.xmpp.chat import XmppChatAction
from slixfeed.xmpp.status import XmppStatusTask
import socket import socket
class XmppIpcServer: class XmppIpcServer:
@ -86,7 +90,7 @@ class XmppIpcServer:
else: else:
command = data command = data
match command: match command:
case _ if command.startswith('add '): case _ if command.startswith('add'):
command = command[4:] command = command[4:]
url = command.split(' ')[0] url = command.split(' ')[0]
title = ' '.join(command.split(' ')[1:]) title = ' '.join(command.split(' ')[1:])
@ -130,10 +134,10 @@ class XmppIpcServer:
self, muc_jid) self, muc_jid)
case 'bookmarks': case 'bookmarks':
response = await XmppCommands.print_bookmarks(self) response = await XmppCommands.print_bookmarks(self)
case _ if command.startswith('clear '): case _ if command.startswith('clear'):
key = command[6:] key = command[6:]
response = await XmppCommands.clear_filter(db_file, key) response = await XmppCommands.clear_filter(db_file, key)
case _ if command.startswith('default '): case _ if command.startswith('default'):
key = command[8:] key = command[8:]
response = await XmppCommands.restore_default( response = await XmppCommands.restore_default(
self, jid_bare, key=None) self, jid_bare, key=None)
@ -163,10 +167,10 @@ class XmppIpcServer:
response = ('No action has been taken.' response = ('No action has been taken.'
'\n' '\n'
'Missing keywords.') 'Missing keywords.')
case _ if command.startswith('disable '): case _ if command.startswith('disable'):
response = await XmppCommands.feed_disable( response = await XmppCommands.feed_disable(
self, db_file, jid_bare, command) self, db_file, jid_bare, command)
case _ if command.startswith('enable '): case _ if command.startswith('enable'):
response = await XmppCommands.feed_enable( response = await XmppCommands.feed_enable(
self, db_file, command) self, db_file, command)
case _ if command.startswith('export'): case _ if command.startswith('export'):
@ -207,12 +211,12 @@ class XmppIpcServer:
case 'pubsub list': case 'pubsub list':
response = await XmppCommands.pubsub_list( response = await XmppCommands.pubsub_list(
self, jid_bare) self, jid_bare)
case _ if command.startswith('pubsub list '): case _ if command.startswith('pubsub list'):
jid = command[12:] jid = command[12:]
response = 'List of nodes for {}:\n```\n'.format(jid) response = 'List of nodes for {}:\n```\n'.format(jid)
response = await XmppCommands.pubsub_list(self, jid) response = await XmppCommands.pubsub_list(self, jid)
response += '```' response += '```'
case _ if command.startswith('pubsub send '): case _ if command.startswith('pubsub send'):
info = command[12:] info = command[12:]
info = info.split(' ') info = info.split(' ')
jid = info[0] jid = info[0]
@ -233,6 +237,7 @@ class XmppIpcServer:
if val: if val:
response = await XmppCommands.set_interval( response = await XmppCommands.set_interval(
self, db_file, jid_bare, val) self, db_file, jid_bare, val)
XmppChatTask.restart_task(self, jid_bare)
else: else:
response = 'Current value for interval: ' response = 'Current value for interval: '
response += XmppCommands.get_interval(self, jid_bare) response += XmppCommands.get_interval(self, jid_bare)
@ -257,12 +262,13 @@ class XmppIpcServer:
response = await XmppCommands.set_old_off( response = await XmppCommands.set_old_off(
self, jid_bare, db_file) self, jid_bare, db_file)
case _ if command.startswith('next'): case _ if command.startswith('next'):
await XmppCommands.send_next_update(self, jid_bare, command) num = command[5:]
case _ if command.startswith('node delete '): await XmppChatAction.send_unread_items(self, jid_bare, num)
case _ if command.startswith('node delete'):
info = command[12:] info = command[12:]
info = info.split(' ') info = info.split(' ')
response = XmppCommands.node_delete(self, info) response = XmppCommands.node_delete(self, info)
case _ if command.startswith('node purge '): case _ if command.startswith('node purge'):
info = command[11:] info = command[11:]
info = info.split(' ') info = info.split(' ')
response = XmppCommands.node_purge(self, info) response = XmppCommands.node_purge(self, info)
@ -284,7 +290,7 @@ class XmppIpcServer:
self, jid_bare) self, jid_bare)
case 'random': case 'random':
response = XmppCommands.set_random(self, jid_bare, db_file) response = XmppCommands.set_random(self, jid_bare, db_file)
case _ if command.startswith('read '): case _ if command.startswith('read'):
data = command[5:] data = command[5:]
data = data.split() data = data.split()
url = data[0] url = data[0]
@ -305,26 +311,26 @@ class XmppIpcServer:
response += result response += result
else: else:
response = result response = result
case _ if command.startswith('remove '): case _ if command.startswith('remove'):
ix_url = command[7:] ix_url = command[7:]
ix_url = ix_url.split(' ') ix_url = ix_url.split(' ')
response = await XmppCommands.feed_remove( response = await XmppCommands.feed_remove(
self, jid_bare, db_file, ix_url) self, jid_bare, db_file, ix_url)
case _ if command.startswith('rename '): case _ if command.startswith('rename'):
response = await XmppCommands.feed_rename( response = await XmppCommands.feed_rename(
self, db_file, jid_bare, command) self, db_file, jid_bare, command)
case _ if command.startswith('reset'): case _ if command.startswith('reset'):
ix_url = command[6:] ix_url = command[6:]
ix_url = ix_url.split(' ') ix_url = ix_url.split(' ')
response = await XmppCommands.mark_as_read( response = await XmppCommands.mark_as_read(
self, jid_bare, db_file, ix_url) jid_bare, db_file, ix_url)
case _ if command.startswith('search'): case _ if command.startswith('search'):
query = command[7:] query = command[7:]
response = XmppCommands.search_items( response = XmppCommands.search_items(db_file, query)
self, db_file, query)
case 'start': case 'start':
tasks = (FeedTask, XmppChatTask, XmppStatusTask)
response = await XmppCommands.scheduler_start( response = await XmppCommands.scheduler_start(
self, db_file, jid_bare) self, db_file, jid_bare, tasks)
case 'stats': case 'stats':
response = XmppCommands.print_statistics(db_file) response = XmppCommands.print_statistics(db_file)
case 'stop': case 'stop':

View file

@ -1,17 +1,19 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging from slixfeed.log import Logger
from slixmpp.exceptions import IqError, IqTimeout from slixmpp.exceptions import IqError, IqTimeout
logger = Logger(__name__)
class XmppIQ: class XmppIQ:
async def send(self, iq): async def send(self, iq):
try: try:
await iq.send(timeout=15) await iq.send(timeout=15)
except IqTimeout as e: except IqTimeout as e:
logging.error('Error Timeout') logger.error('Error Timeout')
logging.error(str(e)) logger.error(str(e))
except IqError as e: except IqError as e:
logging.error('Error XmppIQ') logger.error('Error XmppIQ')
logging.error(str(e)) logger.error(str(e))

View file

@ -1,24 +1,11 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging from slixfeed.log import Logger
import os
# import slixfeed.action as action
import slixfeed.config as config
from slixfeed.dt import current_time, timestamp
import slixfeed.fetch as fetch
import slixfeed.sqlite as sqlite
import slixfeed.task as task
import slixfeed.url as uri
from slixfeed.xmpp.bookmark import XmppBookmark
# from slixfeed.xmpp.muc import XmppGroupchat
# from slixfeed.xmpp.message import XmppMessage
from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.upload import XmppUpload
from slixfeed.xmpp.utility import get_chat_type
import time
import xml.sax.saxutils as saxutils import xml.sax.saxutils as saxutils
logger = Logger(__name__)
""" """
NOTE NOTE

View file

@ -1,97 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
TODO
1) Send message to inviter that bot has joined to groupchat.
2) If groupchat requires captcha, send the consequent message.
3) If groupchat error is received, send that error message to inviter.
FIXME
1) Save name of groupchat instead of jid as name
"""
import logging
from slixmpp.exceptions import IqError, IqTimeout, PresenceError
class XmppGroupchat:
async def join(self, jid, alias=None, password=None):
# token = await initdb(
# muc_jid,
# sqlite.get_setting_value,
# "token"
# )
# if token != "accepted":
# token = randrange(10000, 99999)
# await initdb(
# muc_jid,
# sqlite.update_setting_value,
# ["token", token]
# )
# self.send_message(
# mto=inviter,
# mfrom=self.boundjid.bare,
# mbody=(
# "Send activation token {} to groupchat xmpp:{}?join."
# ).format(token, muc_jid)
# )
logging.info('Joining groupchat\n'
'JID : {}\n'
.format(jid))
jid_from = str(self.boundjid) if self.is_component else None
if alias == None: self.alias
try:
await self.plugin['xep_0045'].join_muc_wait(jid,
alias,
presence_options = {"pfrom" : jid_from},
password=password,
maxchars=0,
maxstanzas=0,
seconds=0,
since=0,
timeout=30)
result = 'joined ' + jid
except IqError as e:
logging.error('Error XmppIQ')
logging.error(str(e))
logging.error(jid)
result = 'error'
except IqTimeout as e:
logging.error('Timeout XmppIQ')
logging.error(str(e))
logging.error(jid)
result = 'timeout'
except PresenceError as e:
logging.error('Error Presence')
logging.error(str(e))
if (e.condition == 'forbidden' and
e.presence['error']['code'] == '403'):
logging.warning('{} is banned from {}'.format(self.alias, jid))
result = 'ban'
else:
result = 'error'
return result
def leave(self, jid):
jid_from = str(self.boundjid) if self.is_component else None
message = ('This news bot will now leave this groupchat.\n'
'The JID of this news bot is xmpp:{}?message'
.format(self.boundjid.bare))
status_message = ('This bot has left the group. '
'It can be reached directly via {}'
.format(self.boundjid.bare))
self.send_message(mto=jid,
mfrom=self.boundjid,
mbody=message,
mtype='groupchat')
self.plugin['xep_0045'].leave_muc(jid,
self.alias,
status_message,
jid_from)

View file

@ -28,10 +28,12 @@ TODO
import glob import glob
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.config as config import slixfeed.config as config
from slixfeed.log import Logger
from slixmpp.exceptions import IqTimeout, IqError from slixmpp.exceptions import IqTimeout, IqError
import logging
import os import os
logger = Logger(__name__)
# class XmppProfile: # class XmppProfile:
async def update(self): async def update(self):
@ -39,19 +41,19 @@ async def update(self):
try: try:
await set_vcard(self) await set_vcard(self)
except IqTimeout as e: except IqTimeout as e:
logging.error('Profile vCard: Error Timeout') logger.error('Profile vCard: Error Timeout')
logging.error(str(e)) logger.error(str(e))
except IqError as e: except IqError as e:
logging.error('Profile vCard: Error XmppIQ') logger.error('Profile vCard: Error XmppIQ')
logging.error(str(e)) logger.error(str(e))
try: try:
await set_avatar(self) await set_avatar(self)
except IqTimeout as e: except IqTimeout as e:
logging.error('Profile Photo: Error Timeout') logger.error('Profile Photo: Error Timeout')
logging.error(str(e)) logger.error(str(e))
except IqError as e: except IqError as e:
logging.error('Profile Photo: Error XmppIQ') logger.error('Profile Photo: Error XmppIQ')
logging.error(str(e)) logger.error(str(e))
async def set_avatar(self): async def set_avatar(self):
@ -74,7 +76,14 @@ async def set_avatar(self):
with open(image_file, 'rb') as avatar_file: with open(image_file, 'rb') as avatar_file:
avatar = avatar_file.read() avatar = avatar_file.read()
# await self.plugin['xep_0084'].publish_avatar(avatar) # await self.plugin['xep_0084'].publish_avatar(avatar)
try:
await self.plugin['xep_0153'].set_avatar(avatar=avatar) await self.plugin['xep_0153'].set_avatar(avatar=avatar)
except IqTimeout as e:
logger.error('Profile Photo: Error Timeout 222')
logger.error(str(e))
except IqError as e:
logger.error('Profile Photo: Error XmppIQ 222')
logger.error(str(e))
def set_identity(self, category): def set_identity(self, category):

View file

@ -7,8 +7,22 @@ Functions create_node and create_entry are derived from project atomtopubsub.
""" """
import asyncio
import hashlib
import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub
from slixmpp.xmlstream import ET from slixmpp.xmlstream import ET
import slixfeed.config as config
from slixfeed.config import Config
from slixfeed.log import Logger
import slixfeed.sqlite as sqlite
from slixfeed.syndication import Feed
import slixfeed.url as uri
from slixfeed.utilities import Utilities
from slixfeed.xmpp.iq import XmppIQ
import sys
logger = Logger(__name__)
class XmppPubsub: class XmppPubsub:
@ -110,13 +124,9 @@ class XmppPubsub:
form.addField('pubsub#deliver_payloads', form.addField('pubsub#deliver_payloads',
ftype='boolean', ftype='boolean',
value=0) value=0)
# TODO
form.addField('pubsub#type', form.addField('pubsub#type',
ftype='text-single', ftype='text-single',
value='http://www.w3.org/2005/Atom') value='http://www.w3.org/2005/Atom')
return iq return iq
@ -243,3 +253,137 @@ class XmppPubsub:
iq['pubsub']['publish'].append(item) iq['pubsub']['publish'].append(item)
return iq return iq
class XmppPubsubAction:
async def send_selected_entry(self, jid_bare, node_id, entry_id):
function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare))
db_file = config.get_pathname_to_database(jid_bare)
report = {}
if jid_bare == self.boundjid.bare:
node_id = 'urn:xmpp:microblog:0'
node_subtitle = None
node_title = None
else:
feed_id = sqlite.get_feed_id_by_entry_index(db_file, entry_id)
feed_id = feed_id[0]
node_id, node_title, node_subtitle = sqlite.get_feed_properties(db_file, feed_id)
print('THIS IS A TEST')
print(node_id)
print(node_title)
print(node_subtitle)
print('THIS IS A TEST')
xep = None
iq_create_node = XmppPubsub.create_node(
self, jid_bare, node_id, xep, node_title, node_subtitle)
await XmppIQ.send(self, iq_create_node)
entry = sqlite.get_entry_properties(db_file, entry_id)
print('xmpp_pubsub_send_selected_entry',jid_bare)
print(node_id)
entry_dict = Feed.pack_entry_into_dict(db_file, entry)
node_item = Feed.create_rfc4287_entry(entry_dict)
entry_url = entry_dict['link']
item_id = Utilities.hash_url_to_md5(entry_url)
iq_create_entry = XmppPubsub.create_entry(
self, jid_bare, node_id, item_id, node_item)
await XmppIQ.send(self, iq_create_entry)
await sqlite.mark_as_read(db_file, entry_id)
report = entry_url
return report
async def send_unread_items(self, jid_bare):
"""
Parameters
----------
jid_bare : TYPE
Bare Jabber ID.
Returns
-------
report : dict
URL and Number of processed entries.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare))
db_file = config.get_pathname_to_database(jid_bare)
report = {}
subscriptions = sqlite.get_active_feeds_url(db_file)
for url in subscriptions:
url = url[0]
# feed_id = sqlite.get_feed_id(db_file, url)
# feed_id = feed_id[0]
# feed_properties = sqlite.get_feed_properties(db_file, feed_id)
feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
# Publish to node 'urn:xmpp:microblog:0' for own JID
# Publish to node based on feed identifier for PubSub service.
if jid_bare == self.boundjid.bare:
node_id = 'urn:xmpp:microblog:0'
node_subtitle = None
node_title = None
else:
# node_id = feed_properties[2]
# node_title = feed_properties[3]
# node_subtitle = feed_properties[5]
node_id = sqlite.get_feed_identifier(db_file, feed_id)
node_id = node_id[0]
if not node_id:
counter = 0
hostname = uri.get_hostname(url)
hostname = hostname.replace('.','-')
identifier = hostname + ':' + str(counter)
while True:
if sqlite.check_identifier_exist(db_file, identifier):
counter += 1
identifier = hostname + ':' + str(counter)
else:
break
await sqlite.update_feed_identifier(db_file, feed_id, identifier)
node_id = sqlite.get_feed_identifier(db_file, feed_id)
node_id = node_id[0]
node_title = sqlite.get_feed_title(db_file, feed_id)
node_title = node_title[0]
node_subtitle = sqlite.get_feed_subtitle(db_file, feed_id)
node_subtitle = node_subtitle[0]
xep = None
node_exist = await XmppPubsub.get_node_configuration(self, jid_bare, node_id)
if not node_exist:
iq_create_node = XmppPubsub.create_node(
self, jid_bare, node_id, xep, node_title, node_subtitle)
await XmppIQ.send(self, iq_create_node)
entries = sqlite.get_unread_entries_of_feed(db_file, feed_id)
report[url] = len(entries)
for entry in entries:
feed_entry = Feed.pack_entry_into_dict(db_file, entry)
node_entry = Feed.create_rfc4287_entry(feed_entry)
entry_url = feed_entry['link']
item_id = Utilities.hash_url_to_md5(entry_url)
print('PubSub node item was sent to', jid_bare, node_id)
print(entry_url)
print(item_id)
iq_create_entry = XmppPubsub.create_entry(
self, jid_bare, node_id, item_id, node_entry)
await XmppIQ.send(self, iq_create_entry)
ix = entry[0]
await sqlite.mark_as_read(db_file, ix)
return report
class XmppPubsubTask:
async def task_publish(self, jid_bare):
db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self.settings, jid_bare, db_file)
while True:
await XmppPubsubAction.send_unread_items(self, jid_bare)
await asyncio.sleep(60 * 180)

94
slixfeed/xmpp/status.py Normal file
View file

@ -0,0 +1,94 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import asyncio
from slixfeed.config import Config
import slixfeed.config as config
import slixfeed.sqlite as sqlite
from slixfeed.log import Logger
from slixfeed.xmpp.presence import XmppPresence
import sys
logger = Logger(__name__)
class XmppStatus:
def send_status_message(self, jid_bare):
"""
Send status message.
Parameters
----------
jid : str
Jabber ID.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid: {}'.format(function_name, jid_bare))
status_text = '📜️ Slixfeed RSS News Bot'
db_file = config.get_pathname_to_database(jid_bare)
enabled = Config.get_setting_value(self.settings, jid_bare, 'enabled')
if enabled:
jid_task = self.pending_tasks[jid_bare] if jid_bare in self.pending_tasks else None
if jid_task and len(jid_task):
# print('status dnd for ' + jid_bare)
status_mode = 'dnd'
status_text = jid_task[list(jid_task.keys())[0]]
else:
# print('status enabled for ' + jid_bare)
feeds = sqlite.get_number_of_items(db_file, 'feeds_properties')
if not feeds:
# print('status no feeds for ' + jid_bare)
status_mode = 'available'
status_text = '📪️ Send a URL from a blog or a news site'
else:
unread = sqlite.get_number_of_entries_unread(db_file)
if unread:
# print('status unread for ' + jid_bare)
status_mode = 'chat'
status_text = '📬️ There are {} news items'.format(str(unread))
else:
# print('status no news for ' + jid_bare)
status_mode = 'available'
status_text = '📭️ No news'
else:
# print('status disabled for ' + jid_bare)
status_mode = 'xa'
status_text = '📪️ Send "Start" to receive updates'
XmppPresence.send(self, jid_bare, status_text, status_type=status_mode)
class XmppStatusTask:
async def task_status(self, jid_bare):
while True:
XmppStatus.send_status_message(self, jid_bare)
await asyncio.sleep(60 * 90)
def restart_task(self, jid_bare):
if jid_bare == self.boundjid.bare:
return
if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info('Stopping task "status" for JID {}'.format(jid_bare))
try:
self.task_manager[jid_bare]['status'].cancel()
except:
logger.info('No task "status" for JID {} (XmppStatusTask.start_task)'
.format(jid_bare))
logger.info('Starting tasks "status" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['status'] = asyncio.create_task(
XmppStatusTask.task_status(self, jid_bare))
def stop_task(self, jid_bare):
if (jid_bare in self.task_manager and
'status' in self.task_manager[jid_bare]):
self.task_manager[jid_bare]['status'].cancel()
else:
logger.debug('No task "status" for JID {}'
.format(jid_bare))

View file

@ -6,15 +6,17 @@ Based on http_upload.py example from project slixmpp
https://codeberg.org/poezio/slixmpp/src/branch/master/examples/http_upload.py https://codeberg.org/poezio/slixmpp/src/branch/master/examples/http_upload.py
""" """
import logging from slixfeed.log import Logger
from slixmpp.exceptions import IqTimeout, IqError from slixmpp.exceptions import IqTimeout, IqError
from slixmpp.plugins.xep_0363.http_upload import HTTPError from slixmpp.plugins.xep_0363.http_upload import HTTPError
logger = Logger(__name__)
# import sys # import sys
class XmppUpload: class XmppUpload:
async def start(self, jid, filename, domain=None): async def start(self, jid, filename, domain=None):
logging.info('Uploading file %s...', filename) logger.info('Uploading file %s...', filename)
try: try:
upload_file = self['xep_0363'].upload_file upload_file = self['xep_0363'].upload_file
# if self.encrypted and not self['xep_0454']: # if self.encrypted and not self['xep_0454']:
@ -31,19 +33,21 @@ class XmppUpload:
url = await upload_file( url = await upload_file(
filename, domain, timeout=10, filename, domain, timeout=10,
) )
logging.info('Upload successful!') logger.info('Upload successful!')
logging.info('Sending file to %s', jid) logger.info('Sending file to %s', jid)
except HTTPError: except HTTPError:
url = ( url = ('Error: It appears that this server does not support '
"Error: It appears that this server doesn't support " 'HTTP File Upload.')
"HTTP File Upload." logger.error('It appears that this server does not support '
) 'HTTP File Upload.')
logging.error(
"It appears that this server doesn't support HTTP File Upload."
)
# raise HTTPError( # raise HTTPError(
# "This server doesn't appear to support HTTP File Upload" # "This server doesn't appear to support HTTP File Upload"
# ) # )
except IqTimeout: except IqError as e:
raise TimeoutError('Could not send message in time') logger.error('Could not send message')
logger.error(e)
except IqTimeout as e:
# raise TimeoutError('Could not send message in time')
logger.error('Could not send message in time')
logger.error(e)
return url return url

View file

@ -0,0 +1,60 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from slixfeed.log import Logger
from slixmpp.exceptions import IqError, IqTimeout
logger = Logger(__name__)
# class XmppChat
# class XmppUtility:
class XmppUtilities:
async def get_chat_type(self, jid):
"""
Check chat (i.e. JID) type.
If iqresult["disco_info"]["features"] contains XML namespace
of 'http://jabber.org/protocol/muc', then it is a 'groupchat'.
Unless it has forward slash, which would indicate that it is
a chat which is conducted through a groupchat.
Otherwise, determine type 'chat'.
Parameters
----------
jid : str
Jabber ID.
Returns
-------
result : str
'chat' or 'groupchat' or 'error'.
"""
try:
iqresult = await self["xep_0030"].get_info(jid=jid)
features = iqresult["disco_info"]["features"]
# identity = iqresult['disco_info']['identities']
# if 'account' in indentity:
# if 'conference' in indentity:
if ('http://jabber.org/protocol/muc' in features) and not ('/' in jid):
result = "groupchat"
# TODO elif <feature var='jabber:iq:gateway'/>
# NOTE Is it needed? We do not interact with gateways or services
else:
result = "chat"
logger.info('Jabber ID: {}\n'
'Chat Type: {}'.format(jid, result))
except (IqError, IqTimeout) as e:
logger.warning('Chat type could not be determined for {}'.format(jid))
logger.error(e)
result = 'error'
# except BaseException as e:
# logger.error('BaseException', str(e))
# finally:
# logger.info('Chat type is:', chat_type)
return result

View file

@ -1,56 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from slixmpp.exceptions import IqError, IqTimeout
import logging
# class XmppChat
# class XmppUtility:
# TODO Rename to get_jid_type
async def get_chat_type(self, jid):
"""
Check chat (i.e. JID) type.
If iqresult["disco_info"]["features"] contains XML namespace
of 'http://jabber.org/protocol/muc', then it is a 'groupchat'.
Unless it has forward slash, which would indicate that it is
a chat which is conducted through a groupchat.
Otherwise, determine type 'chat'.
Parameters
----------
jid : str
Jabber ID.
Returns
-------
result : str
'chat' or 'groupchat' or 'error'.
"""
try:
iqresult = await self["xep_0030"].get_info(jid=jid)
features = iqresult["disco_info"]["features"]
# identity = iqresult['disco_info']['identities']
# if 'account' in indentity:
# if 'conference' in indentity:
if ('http://jabber.org/protocol/muc' in features) and not ('/' in jid):
result = "groupchat"
# TODO elif <feature var='jabber:iq:gateway'/>
# NOTE Is it needed? We do not interact with gateways or services
else:
result = "chat"
logging.info('Jabber ID: {}\n'
'Chat Type: {}'.format(jid, result))
except (IqError, IqTimeout) as e:
logging.warning('Chat type could not be determined for {}'.format(jid))
logging.error(e)
result = 'error'
# except BaseException as e:
# logging.error('BaseException', str(e))
# finally:
# logging.info('Chat type is:', chat_type)
return result