sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: sch:master
Branches Tags
sch:master
thecoffemaker:master
..
compare: thecoffemaker:master
Branches Tags
thecoffemaker:master
sch:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

45 changed files with 9622 additions and 2639 deletions
Show stats Expand all files Collapse all files

22
README.md
View file

@ -20,15 +20,14 @@ Slixfeed is primarily designed for XMPP (aka Jabber), yet it is built to be exte
## Features ## Features
- **Visual interface** - Interactive interface for XMPP using Ad-Hoc Commands,
- **Ease** - Slixfeed automatically scans (i.e. crawls) for syndication feeds of given URL. - **Ease** - Slixfeed automatically scans (i.e. crawls) for syndication feeds of given URL.
- **Encryption** - Messages are encrypted with the OMEMO standard.
- **Export** - Download articles as ePUB, HTML, Markdown and PDF. - **Export** - Download articles as ePUB, HTML, Markdown and PDF.
- **Filtering** - Filter news items using lists of allow and deny. - **Filtering** - Filter news items using lists of allow and deny.
- **Multimedia** - Display audios pictures and videos inline. - **Multimedia** - Display audios pictures and videos inline.
- **Privacy** - Redirect to alternative back-ends, such as Invidious, Librarian, Nitter, for increased privacy, productivity and security. - **Privacy** - Redirect to alternative back-ends, such as Invidious, Librarian, Nitter, for increased privacy, productivity and security.
- **Portable** - Export and import feeds with a standard OPML file. - **Portable** - Export and import feeds with a standard OPML file.
- **Simultaneous** - Slixfeed is designed to handle multiple contacts, including groupchats, Simultaneously. - **Simultaneous** - Slixfeed is designed to handle multiple contacts, including groupchats, Simultaneously.
- **Visual interface** - Interactive interface for XMPP using Ad-Hoc Commands,
## Preview ## Preview
@ -57,18 +56,7 @@ It is possible to install Slixfeed using pip and pipx.
``` ```
$ python3 -m venv .venv $ python3 -m venv .venv
$ source .venv/bin/activate $ source .venv/bin/activate
``` $ pip install git+https://gitgud.io/sjehuda/slixfeed
##### Install
```
$ pip install git+https://git.xmpp-it.net/sch/Slixfeed
```
##### Install (OMEMO)
```
$ pip install git+https://git.xmpp-it.net/sch/Slixfeed[omemo]
``` ```
#### pipx #### pipx
@ -76,14 +64,14 @@ $ pip install git+https://git.xmpp-it.net/sch/Slixfeed[omemo]
##### Install ##### Install
``` ```
$ pipx install git+https://git.xmpp-it.net/sch/Slixfeed $ pipx install git+https://gitgud.io/sjehuda/slixfeed
``` ```
##### Update ##### Update
``` ```
$ pipx uninstall slixfeed $ pipx uninstall slixfeed
$ pipx install git+https://git.xmpp-it.net/sch/Slixfeed $ pipx install git+https://gitgud.io/sjehuda/slixfeed
``` ```
### Start ### Start
@ -99,7 +87,7 @@ $ slixfeed
## Recommended Clients ## Recommended Clients
Slixfeed works with any XMPP chat client; if you want to make use of the visual interface Slixfeed has to offer (i.e. Ad-Hoc Commands), then you are advised to use [Cheogram](https://cheogram.com), [Converse](https://conversejs.org), [Gajim](https://gajim.org), [LeechCraft](https://leechcraft.org/plugins-azoth-xoox), [monocles chat](https://monocles.chat), [Movim](https://mov.im), [Poezio](https://poez.io), [Psi](https://psi-im.org) or [Psi+](https://psi-plus.com). Slixfeed works with any XMPP chat client; if you want to make use of the visual interface Slixfeed has to offer (i.e. Ad-Hoc Commands), then you are advised to use [Cheogram](https://cheogram.com), [Converse](https://conversejs.org), [Gajim](https://gajim.org), [monocles chat](https://monocles.chat), [Movim](https://mov.im), [Poezio](https://poez.io), [Psi](https://psi-im.org) or [Psi+](https://psi-plus.com).
### Support ### Support

11
pyproject.toml
View file

@ -37,28 +37,27 @@ keywords = [
"xml", "xml",
"xmpp", "xmpp",
] ]
# urls = {Homepage = "https://gitgud.io/sjehuda/slixfeed"} # urls = {Homepage = "https://gitgud.io/sjehuda/slixfeed"}
dependencies = [ dependencies = [
"aiofiles",
"aiohttp", "aiohttp",
# "daemonize", # "daemonize",
"feedparser", "feedparser",
"lxml", "lxml",
# "pysocks",
"python-dateutil", "python-dateutil",
"requests",
"slixmpp", "slixmpp",
"tomli", # Python 3.10 "tomli", # Python 3.10
"tomli_w", "tomli_w",
] ]
[project.urls] [project.urls]
Homepage = "https://slixfeed.woodpeckersnest.space" Homepage = "http://slixfeed.i2p/"
Repository = "https://git.xmpp-it.net/sch/Slixfeed" Repository = "https://gitgud.io/sjehuda/slixfeed"
Issues = "https://gitgud.io/sjehuda/slixfeed/issues" Issues = "https://gitgud.io/sjehuda/slixfeed/issues"
[project.optional-dependencies] [project.optional-dependencies]
omemo = ["slixmpp-omemo"]
proxy = ["pysocks"] proxy = ["pysocks"]
# [project.readme] # [project.readme]

82
slixfeed/__main__.py
View file

@ -58,8 +58,6 @@ TODO
# res = response (HTTP) # res = response (HTTP)
from argparse import ArgumentParser from argparse import ArgumentParser
import logging import logging
import os
import shutil
import sys import sys
# from eliot import start_action, to_file # from eliot import start_action, to_file
@ -67,9 +65,8 @@ import sys
# # with start_action(action_type='set_date()', jid=jid): # # with start_action(action_type='set_date()', jid=jid):
# # with start_action(action_type='message()', msg=msg): # # with start_action(action_type='message()', msg=msg):
from slixfeed.config import Settings, Share, Cache import slixfeed.config as config
from slixfeed.log import Logger from slixfeed.log import Logger
from slixfeed.utilities import Toml
from slixfeed.version import __version__ from slixfeed.version import __version__
logger = Logger(__name__) logger = Logger(__name__)
@ -81,44 +78,10 @@ logger = Logger(__name__)
def main(): def main():
directory = os.path.dirname(__file__) config_dir = config.get_default_config_directory()
logger.info('Reading configuration from {}'.format(config_dir))
# Copy data files print('Reading configuration from {}'.format(config_dir))
directory_data = Share.get_directory() network_settings = config.get_values('settings.toml', 'network')
if not os.path.exists(directory_data):
directory_assets = os.path.join(directory, 'assets')
directory_assets_new = shutil.copytree(directory_assets, directory_data)
print(f'Data directory {directory_assets_new} has been created and populated.')
# Copy settings files
directory_settings = Settings.get_directory()
if not os.path.exists(directory_settings):
directory_configs = os.path.join(directory, 'configs')
directory_settings_new = shutil.copytree(directory_configs, directory_settings)
print(f'Settings directory {directory_settings_new} has been created and populated.')
# Create cache directories
directory_cache = Cache.get_directory()
if not os.path.exists(directory_cache):
print(f'Creating a cache directory at {directory_cache}.')
os.mkdir(directory_cache)
for subdirectory in ('md', 'enclosure', 'markdown', 'opml', 'readability'):
subdirectory_cache = os.path.join(directory_cache, subdirectory)
if not os.path.exists(subdirectory_cache):
print(f'Creating a cache subdirectory at {subdirectory_cache}.')
os.mkdir(subdirectory_cache)
filename_settings = os.path.join(directory_settings, 'settings.toml')
settings = Toml.open_file(filename_settings)
network_settings = settings['network']
# Configure account
print('User agent:', network_settings['user_agent'] or 'Slixfeed/0.1') print('User agent:', network_settings['user_agent'] or 'Slixfeed/0.1')
if network_settings['http_proxy']: print('HTTP Proxy:', network_settings['http_proxy']) if network_settings['http_proxy']: print('HTTP Proxy:', network_settings['http_proxy'])
@ -160,6 +123,10 @@ def main():
# Setup logging. # Setup logging.
logging.basicConfig(level=args.loglevel, logging.basicConfig(level=args.loglevel,
format='%(levelname)-8s %(message)s') format='%(levelname)-8s %(message)s')
# # Setup logging.
# logging.basicConfig(level=args.loglevel,
# format='%(levelname)-8s %(message)s')
# # logging.basicConfig(format='[%(levelname)s] %(message)s') # # logging.basicConfig(format='[%(levelname)s] %(message)s')
# logger = logging.getLogger() # logger = logging.getLogger()
# logdbg = logger.debug # logdbg = logger.debug
@ -197,33 +164,28 @@ def main():
# if not alias: # if not alias:
# alias = (input('Alias: ')) or 'Slixfeed' # alias = (input('Alias: ')) or 'Slixfeed'
filename_accounts = os.path.join(directory_settings, 'accounts.toml') account_xmpp = config.get_values('accounts.toml', 'xmpp')
accounts = Toml.open_file(filename_accounts)
accounts_xmpp = accounts['xmpp']
# Try configuration file # Try configuration file
if 'client' in accounts_xmpp: if 'client' in account_xmpp:
from slixfeed.xmpp.client import XmppClient from slixfeed.xmpp.client import XmppClient
jid = account_xmpp['client']['jid']
accounts_xmpp_client = accounts_xmpp['client'] password = account_xmpp['client']['password']
jid = accounts_xmpp_client['jid'] alias = account_xmpp['client']['alias'] if 'alias' in account_xmpp['client'] else None
password = accounts_xmpp_client['password'] hostname = account_xmpp['client']['hostname'] if 'hostname' in account_xmpp['client'] else None
alias = accounts_xmpp_client['alias'] if 'alias' in accounts_xmpp_client else None port = account_xmpp['client']['port'] if 'port' in account_xmpp['client'] else None
hostname = accounts_xmpp_client['hostname'] if 'hostname' in accounts_xmpp_client else None
port = accounts_xmpp_client['port'] if 'port' in accounts_xmpp_client else None
XmppClient(jid, password, hostname, port, alias) XmppClient(jid, password, hostname, port, alias)
# xmpp_client = Slixfeed(jid, password, hostname, port, alias) # xmpp_client = Slixfeed(jid, password, hostname, port, alias)
# xmpp_client.connect((hostname, port)) if hostname and port else xmpp_client.connect() # xmpp_client.connect((hostname, port)) if hostname and port else xmpp_client.connect()
# xmpp_client.process() # xmpp_client.process()
if 'component' in accounts_xmpp: if 'component' in account_xmpp:
from slixfeed.xmpp.component import XmppComponent from slixfeed.xmpp.component import XmppComponent
accounts_xmpp_component = accounts_xmpp['component'] jid = account_xmpp['component']['jid']
jid = accounts_xmpp_component['jid'] secret = account_xmpp['component']['password']
secret = accounts_xmpp_component['password'] alias = account_xmpp['component']['alias'] if 'alias' in account_xmpp['component'] else None
alias = accounts_xmpp_component['alias'] if 'alias' in accounts_xmpp_component else None hostname = account_xmpp['component']['hostname'] if 'hostname' in account_xmpp['component'] else None
hostname = accounts_xmpp_component['hostname'] if 'hostname' in accounts_xmpp_component else None port = account_xmpp['component']['port'] if 'port' in account_xmpp['component'] else None
port = accounts_xmpp_component['port'] if 'port' in accounts_xmpp_component else None
XmppComponent(jid, secret, hostname, port, alias) XmppComponent(jid, secret, hostname, port, alias)
# xmpp_component = SlixfeedComponent(jid, secret, hostname, port, alias) # xmpp_component = SlixfeedComponent(jid, secret, hostname, port, alias)
# xmpp_component.connect() # xmpp_component.connect()

0
slixfeed/assets/__init__.py Normal file
View file

21
slixfeed/configs/about.toml → slixfeed/assets/about.toml
View file

@ -28,9 +28,9 @@ Good luck!
filetypes = "Atom, JSON, RDF, RSS, XML." filetypes = "Atom, JSON, RDF, RSS, XML."
platforms = "XMPP" platforms = "XMPP"
# platforms = "ActivityPub, BitMessage, Briar, DeltaChat, Email, IRC, LXMF, MQTT, Nostr, Session, Tox." # platforms = "ActivityPub, Briar, DeltaChat, Email, IRC, LXMF, MQTT, Nostr, Session, Tox."
comment = "For ideal experience, we recommend using XMPP." # Nostr, Session or DeltaChat comment = "For ideal experience, we recommend using XMPP." # Nostr, Session or DeltaChat
url = "https://git.xmpp-it.net/sch/Slixfeed" url = "https://gitgud.io/sjehuda/slixfeed"
[[about]] [[about]]
name = "slixmpp" name = "slixmpp"
@ -245,7 +245,7 @@ and webhooks.
User ⇄ XMPP client ⇄ XMPP Server ⇄ XMPP Bot ⇄ REST API User ⇄ XMPP client ⇄ XMPP Server ⇄ XMPP Bot ⇄ REST API
"""] """]
interface = "Groupchat" interface = "Groupchat"
url = "https://git.xmpp-it.net/roughnecks/xmpp-bot" url = "https://github.com/nioc/xmpp-bot"
[[legal]] [[legal]]
title = "Legal" title = "Legal"
@ -260,7 +260,7 @@ Slixfeed is distributed in the hope that it will be useful, but WITHOUT ANY \
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR \ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR \
A PARTICULAR PURPOSE. See the MIT License for more details. A PARTICULAR PURPOSE. See the MIT License for more details.
"""] """]
link = "https://git.xmpp-it.net/sch/Slixfeed" link = "https://gitgud.io/sjehuda/slixfeed"
[[license]] [[license]]
title = "License" title = "License"
@ -653,15 +653,10 @@ or on your desktop.
url = "https://conversejs.org" url = "https://conversejs.org"
platform = "HTML (Web)" platform = "HTML (Web)"
[[clients]] # [[clients]]
name = "Gajim" # name = "Gajim"
info = "XMPP client for desktop" # info = "XMPP client for desktop"
info = [""" # url = "https://gajim.org"
Gajim aims to be an easy to use and fully-featured XMPP client. \
It is open source and released under the GNU General Public License (GPL).
"""]
url = "https://gajim.org"
platform = "Any"
# [[clients]] # [[clients]]
# name = "Monal IM" # name = "Monal IM"

0
slixfeed/configs/accounts.toml → slixfeed/assets/accounts.toml
View file

30
slixfeed/configs/commands.toml → slixfeed/assets/commands.toml
View file

@ -213,18 +213,6 @@ read <url> <index>
Display specified entry number from given <url> by given <index>. Display specified entry number from given <url> by given <index>.
""" """
[pubsub]
pubsub = """
pubsub [off|on]
Designate JID as PubSub service.
"""
[send]
send = """
send <pubsub>/<jid> <url> <node>
Send feeds to given JID.
"""
[search] [search]
feeds = """ feeds = """
feeds feeds
@ -248,24 +236,6 @@ stats = """
stats stats
Show general statistics. Show general statistics.
""" """
[list]
blacklist = """
blacklist
List all blacklisted JIDs.
"""
blacklist_action = """
blacklist [add|delete] <jid>
Manage allowed list.
"""
whitelist = """
whitelist
List all whitelisted JIDs.
"""
whitelist_action = """
whitelist [add|delete] <jid>
Manage denied list.
"""
# analyses = """ # analyses = """
# analyses # analyses
# Show report and statistics of feeds. # Show report and statistics of feeds.

2108
slixfeed/assets/feeds.csv Normal file
View file

File diff suppressed because it is too large Load diff

66
slixfeed/configs/feeds.toml → slixfeed/assets/feeds.toml
View file

@ -66,7 +66,7 @@ tags = ["event", "germany", "xmpp"]
[[feeds]] [[feeds]]
lang = "de-de" lang = "de-de"
name = "journal | hasecke" name = "blog | hasecke"
link = "https://www.hasecke.eu/index.xml" link = "https://www.hasecke.eu/index.xml"
tags = ["linux", "p2p", "software", "technology"] tags = ["linux", "p2p", "software", "technology"]
@ -78,7 +78,7 @@ tags = ["computer", "industry", "electronics", "technology"]
[[feeds]] [[feeds]]
lang = "de-de" lang = "de-de"
name = "CCC Event Journal" name = "CCC Event Blog"
link = "https://events.ccc.de/feed" link = "https://events.ccc.de/feed"
tags = ["ccc", "club", "event"] tags = ["ccc", "club", "event"]
@ -188,13 +188,13 @@ tags = ["linux", "debian", "ubuntu", "industry"]
lang = "en" lang = "en"
name = "Dig Deeper" name = "Dig Deeper"
link = "https://diggy.club/atom.xml" link = "https://diggy.club/atom.xml"
tags = ["linux", "health", "computer", "wisdom", "research", "life", "industry"] tags = ["linux", "health", "computer", "wisdom", "life", "industry"]
[[feeds]] [[feeds]]
lang = "en" lang = "en"
name = "Earth Newspaper" name = "Earth Newspaper"
link = "https://earthnewspaper.com/feed/atom/" link = "https://earthnewspaper.com/feed/atom/"
tags = ["technology", "sports", "culture", "world", "war", "politics"] tags = ["technology", "world", "war", "politics"]
[[feeds]] [[feeds]]
lang = "en" lang = "en"
@ -208,12 +208,6 @@ name = "her.st - Do you see it yet?"
link = "https://her.st/feed.xml" link = "https://her.st/feed.xml"
tags = ["lifestyle", "technology", "xmpp", "computer", "code", "llm", "syndication", "minimalism", "linux", "self-hosting", ".net", "go", "python", "philosophy", "psychology", "privacy", "security"] tags = ["lifestyle", "technology", "xmpp", "computer", "code", "llm", "syndication", "minimalism", "linux", "self-hosting", ".net", "go", "python", "philosophy", "psychology", "privacy", "security"]
[[feeds]]
lang = "en"
name = "Hippo (Badri Sunderarajan)"
link = "https://badrihippo.thekambattu.rocks/feed.xml"
tags = ["computer", "mobian", "gerda", "pris", "prav", "kaios", "linux", "phosh", "browser", "telecommunication", "internet", "xmpp"]
[[feeds]] [[feeds]]
lang = "en" lang = "en"
name = "Lagrange Gemini Client" name = "Lagrange Gemini Client"
@ -222,19 +216,13 @@ tags = ["gemini", "gopher", "browser", "telecommunication", "internet"]
[[feeds]] [[feeds]]
lang = "en" lang = "en"
name = "[ngn.tf] | journal" name = "[ngn.tf] | blog"
link = "https://api.ngn.tf/blog/feed.atom" link = "https://api.ngn.tf/blog/feed.atom"
tags = ["computer", "service", "technology", "telecommunication", "xmpp"] tags = ["computer", "service", "technology", "telecommunication", "xmpp"]
[[feeds]] [[feeds]]
lang = "en" lang = "en"
name = "Proycon's Journal" name = "RTP Blog"
link = "https://proycon.anaproy.nl/rss.xml"
tags = ["computer", "technology", "telecommunication", "postmarketos", "music", "piano", "privacy"]
[[feeds]]
lang = "en"
name = "RTP - Right To Privacy Journal"
link = "http://righttoprivacy.i2p/rss/" link = "http://righttoprivacy.i2p/rss/"
tags = ["computer", "service", "technology", "telecommunication", "i2p", "privacy"] tags = ["computer", "service", "technology", "telecommunication", "i2p", "privacy"]
@ -282,7 +270,7 @@ tags = ["christianity", "copy", "freedom", "religion", "software", "technology"]
[[feeds]] [[feeds]]
lang = "en-ca" lang = "en-ca"
name = "JMP's Journal" name = "blog.jmp.chat's blog"
link = "https://blog.jmp.chat/atom.xml" link = "https://blog.jmp.chat/atom.xml"
tags = ["jmp", "service", "sms", "telecommunication", "xmpp"] tags = ["jmp", "service", "sms", "telecommunication", "xmpp"]
@ -330,7 +318,7 @@ tags = ["news", "politics", "privacy", "surveillance"]
[[feeds]] [[feeds]]
lang = "en-gb" lang = "en-gb"
name = "op-co.de journal" name = "op-co.de blog"
link = "https://op-co.de/blog/index.rss" link = "https://op-co.de/blog/index.rss"
tags = ["code", "germany", "jabber", "mastodon", "telecommunication", "xmpp"] tags = ["code", "germany", "jabber", "mastodon", "telecommunication", "xmpp"]
@ -360,7 +348,7 @@ tags = ["art", "economics", "education", "hardware", "research", "technology"]
[[feeds]] [[feeds]]
lang = "en-gb" lang = "en-gb"
name = "Snikket Journal on Snikket Chat" name = "Snikket Blog on Snikket Chat"
link = "https://snikket.org/blog/index.xml" link = "https://snikket.org/blog/index.xml"
tags = ["chat", "jabber", "telecommunication", "xmpp"] tags = ["chat", "jabber", "telecommunication", "xmpp"]
@ -380,7 +368,7 @@ tags = ["design", "diy", "household"]
lang = "en-us" lang = "en-us"
name = "12bytes.org" name = "12bytes.org"
link = "https://12bytes.org/feed.xml" link = "https://12bytes.org/feed.xml"
tags = ["conspiracy", "linux", "computer", "security", "privacy", "culture", "health", "government", "war", "world"] tags = ["conspiracy", "health", "government", "war", "world"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
@ -418,6 +406,12 @@ name = "CODEPINK - Women for Peace"
link = "https://www.codepink.org/news.rss" link = "https://www.codepink.org/news.rss"
tags = ["activism", "peace", "war", "women"] tags = ["activism", "peace", "war", "women"]
[[feeds]]
lang = "en-us"
name = "Ctrl blog"
link = "https://feed.ctrl.blog/latest.atom"
tags = ["computer", "technology"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "Delta Chat - Messenger based on e-mail" name = "Delta Chat - Messenger based on e-mail"
@ -426,16 +420,10 @@ tags = ["email", "telecommunication"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "Disroot Journal" name = "Disroot Blog"
link = "https://disroot.org/en/blog.atom" link = "https://disroot.org/en/blog.atom"
tags = ["decentralization", "privacy"] tags = ["decentralization", "privacy"]
[[feeds]]
lang = "en-us"
name = "Earthing Institute"
link = "https://earthinginstitute.net/feed/atom/"
tags = ["health", "meditation", "yoga"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "F-Droid" name = "F-Droid"
@ -498,7 +486,7 @@ tags = ["news", "politics", "usa", "world"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "Jacob's Unnamed Journal" name = "Jacob's Unnamed Blog"
link = "https://jacobwsmith.xyz/feed.xml" link = "https://jacobwsmith.xyz/feed.xml"
tags = ["book", "community", "culture", "family", "finance", "lifestyle", "market", "usa"] tags = ["book", "community", "culture", "family", "finance", "lifestyle", "market", "usa"]
@ -636,7 +624,7 @@ tags = ["gemini", "internet"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "Public Intelligence Journal" name = "Public Intelligence Blog"
link = "https://phibetaiota.net/feed/" link = "https://phibetaiota.net/feed/"
tags = ["cia", "conspiracy", "health", "government", "war", "world"] tags = ["cia", "conspiracy", "health", "government", "war", "world"]
@ -684,7 +672,7 @@ tags = ["culture", "podcast", "politics", "usa", "vodcast"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "Redecentralize Journal" name = "Redecentralize Blog"
link = "https://redecentralize.org/blog/feed.rss" link = "https://redecentralize.org/blog/feed.rss"
tags = ["podcast", "privacy", "surveillance", "vodcast"] tags = ["podcast", "privacy", "surveillance", "vodcast"]
@ -732,7 +720,7 @@ tags = ["activism", "geoengineering"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "Sweet Home 3D Journal" name = "Sweet Home 3D Blog"
link = "http://www.sweethome3d.com/blog/rss.xml" link = "http://www.sweethome3d.com/blog/rss.xml"
tags = ["3d", "architecture", "design", "game"] tags = ["3d", "architecture", "design", "game"]
@ -780,7 +768,7 @@ tags = ["farming", "food", "gardening", "survival"]
[[feeds]] [[feeds]]
lang = "en-us" lang = "en-us"
name = "The XMPP Journal on XMPP" name = "The XMPP Blog on XMPP"
link = "https://xmpp.org/feeds/all.atom.xml" link = "https://xmpp.org/feeds/all.atom.xml"
tags = ["jabber", "telecommunication", "xmpp"] tags = ["jabber", "telecommunication", "xmpp"]
@ -828,7 +816,7 @@ tags = ["decentralization", "development", "electronics", "networking", "privacy
[[feeds]] [[feeds]]
lang = "es-es" lang = "es-es"
name = "Disroot Journal" name = "Disroot Blog"
link = "https://disroot.org/es/blog.atom" link = "https://disroot.org/es/blog.atom"
tags = ["decentralization", "privacy"] tags = ["decentralization", "privacy"]
@ -858,13 +846,13 @@ tags = ["technology"]
[[feeds]] [[feeds]]
lang = "fr-fr" lang = "fr-fr"
name = "Disroot Journal" name = "Disroot Blog"
link = "https://disroot.org/fr/blog.atom" link = "https://disroot.org/fr/blog.atom"
tags = ["decentralization", "privacy"] tags = ["decentralization", "privacy"]
[[feeds]] [[feeds]]
lang = "fr-fr" lang = "fr-fr"
name = "Frama Journal" name = "Framablog"
link = "https://framablog.org/feed/" link = "https://framablog.org/feed/"
tags = ["fediverse", "framasoft", "open source", "peertube", "privacy", "software", "xmpp"] tags = ["fediverse", "framasoft", "open source", "peertube", "privacy", "software", "xmpp"]
@ -960,7 +948,7 @@ tags = ["computer", "culture", "food", "technology"]
[[feeds]] [[feeds]]
lang = "it-it" lang = "it-it"
name = "Disroot Journal" name = "Disroot Blog"
link = "https://disroot.org/it/blog.atom" link = "https://disroot.org/it/blog.atom"
tags = ["decentralization", "privacy"] tags = ["decentralization", "privacy"]
@ -1014,7 +1002,7 @@ tags = ["computer", "technology", "design"]
[[feeds]] [[feeds]]
lang = "ru-ru" lang = "ru-ru"
name = "Disroot Journal" name = "Disroot Blog"
link = "https://disroot.org/ru/blog.atom" link = "https://disroot.org/ru/blog.atom"
tags = ["decentralization", "privacy"] tags = ["decentralization", "privacy"]

49
slixfeed/assets/image.svg
View file

@ -1,48 +1 @@
<svg height="600" width="600" xmlns="http://www.w3.org/2000/svg" xml:space="preserve"> <svg height="600" width="600" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0zm0 0" style="fill:#ffa000" transform="translate(44 44)"/><path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279zm0 0" style="fill:#ffa000" transform="translate(44 44)"/><path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47Zm0 0" style="fill:#ffa000" transform="translate(44 44)"/></svg>
<defs>
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
<feGaussianBlur in="SourceAlpha" stdDeviation="5" />
<feOffset dx="5" dy="5" result="offsetblur" />
<feFlood flood-color="rgba(0,0,0,0.5)" />
<feComposite in2="offsetblur" operator="in" />
<feMerge>
<feMergeNode />
<feMergeNode in="SourceGraphic" />
</feMerge>
</filter>
<!-- Glass Gradient -->
<linearGradient id="glassGradient" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:rgba(255, 255, 255, 0.3); stop-opacity:1" />
<stop offset="100%" style="stop-color:rgba(255, 255, 255, 0.1); stop-opacity:1" />
</linearGradient>
</defs>
<!-- Black shapes with orange margins -->
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:#ffa000; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:#ffa000; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:#ffa000; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<!-- Glass Shadow Effect Layer -->
<g filter="url(#shadow)">
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
</g>
</svg>
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 2 KiB

After

Width:  |  Height:  |  Size: 448 B

48
slixfeed/assets/image_black.svg
View file

@ -1,48 +0,0 @@
<svg height="600" width="600" xmlns="http://www.w3.org/2000/svg" xml:space="preserve">
<defs>
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
<feGaussianBlur in="SourceAlpha" stdDeviation="5" />
<feOffset dx="5" dy="5" result="offsetblur" />
<feFlood flood-color="rgba(0,0,0,0.5)" />
<feComposite in2="offsetblur" operator="in" />
<feMerge>
<feMergeNode />
<feMergeNode in="SourceGraphic" />
</feMerge>
</filter>
<!-- Glass Gradient -->
<linearGradient id="glassGradient" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:rgba(255, 255, 255, 0.3); stop-opacity:1" />
<stop offset="100%" style="stop-color:rgba(255, 255, 255, 0.1); stop-opacity:1" />
</linearGradient>
</defs>
<!-- Black shapes with orange margins -->
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:#000000; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:#000000; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:#000000; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<!-- Glass Shadow Effect Layer -->
<g filter="url(#shadow)">
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
</g>
</svg>
Side by side

Before

Width:  |  Height:  |  Size: 2 KiB

51
slixfeed/assets/image_semi_transparent.svg
View file

@ -1,51 +0,0 @@
<svg height="600" width="600" xmlns="http://www.w3.org/2000/svg" xml:space="preserve">
<defs>
<!-- Gradient for Glass Effect -->
<linearGradient id="glassGradient" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:rgba(255, 255, 255, 0.5); stop-opacity:1" />
<stop offset="100%" style="stop-color:rgba(0, 0, 0, 0.2); stop-opacity:1" />
</linearGradient>
<linearGradient id="glassHighlight" x1="0%" y1="0%" x2="100%" y2="0%">
<stop offset="0%" style="stop-color:rgba(255, 255, 255, 0.8); stop-opacity:1" />
<stop offset="100%" style="stop-color:rgba(255, 255, 255, 0); stop-opacity:1" />
</linearGradient>
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
<feGaussianBlur in="SourceAlpha" stdDeviation="5" />
<feOffset dx="5" dy="5" result="offsetblur" />
<feFlood flood-color="rgba(0, 0, 0, 0.3)" />
<feComposite in2="offsetblur" operator="in" />
<feMerge>
<feMergeNode />
<feMergeNode in="SourceGraphic" />
</feMerge>
</filter>
</defs>
<!-- Shapes with Glass Effect -->
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassGradient); stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassGradient); stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassGradient); stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<!-- Highlights for Shiny Effect -->
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassHighlight); opacity:0.6;"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassHighlight); opacity:0.6;"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassHighlight); opacity:0.6;"
transform="translate(44 44)" />
</svg>
Side by side

Before

Width:  |  Height:  |  Size: 2.2 KiB

48
slixfeed/assets/image_transparent.svg
View file

@ -1,48 +0,0 @@
<svg height="600" width="600" xmlns="http://www.w3.org/2000/svg" xml:space="preserve">
<defs>
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
<feGaussianBlur in="SourceAlpha" stdDeviation="5" />
<feOffset dx="5" dy="5" result="offsetblur" />
<feFlood flood-color="rgba(0,0,0,0.5)" />
<feComposite in2="offsetblur" operator="in" />
<feMerge>
<feMergeNode />
<feMergeNode in="SourceGraphic" />
</feMerge>
</filter>
<!-- Glass Gradient -->
<linearGradient id="glassGradient" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:rgba(255, 255, 255, 0.3); stop-opacity:1" />
<stop offset="100%" style="stop-color:rgba(255, 255, 255, 0.1); stop-opacity:1" />
</linearGradient>
</defs>
<!-- Black shapes with orange margins -->
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassGradient); stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassGradient); stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassGradient); stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<!-- Glass Shadow Effect Layer -->
<g filter="url(#shadow)">
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
</g>
</svg>
Side by side

Before

Width:  |  Height:  |  Size: 2 KiB

48
slixfeed/assets/image_white.svg
View file

@ -1,48 +0,0 @@
<svg height="600" width="600" xmlns="http://www.w3.org/2000/svg" xml:space="preserve">
<defs>
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
<feGaussianBlur in="SourceAlpha" stdDeviation="5" />
<feOffset dx="5" dy="5" result="offsetblur" />
<feFlood flood-color="rgba(0,0,0,0.5)" />
<feComposite in2="offsetblur" operator="in" />
<feMerge>
<feMergeNode />
<feMergeNode in="SourceGraphic" />
</feMerge>
</filter>
<!-- Glass Gradient -->
<linearGradient id="glassGradient" x1="0%" y1="0%" x2="0%" y2="100%">
<stop offset="0%" style="stop-color:rgba(255, 255, 255, 0.3); stop-opacity:1" />
<stop offset="100%" style="stop-color:rgba(255, 255, 255, 0.1); stop-opacity:1" />
</linearGradient>
</defs>
<!-- Black shapes with orange margins -->
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:#ffffff; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:#ffffff; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:#ffffff; stroke:#e15a00; stroke-width:10; filter:url(#shadow);"
transform="translate(44 44)" />
<!-- Glass Shadow Effect Layer -->
<g filter="url(#shadow)">
<path d="M167 406a60 60 0 1 1-120 0 60 60 0 0 1 120 0z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 186v80c110 0 199 89 199 199h80c0-154-125-279-279-279z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
<path d="M47 47v79c187 0 338 152 338 339h80C465 234 277 47 47 47z"
style="fill:url(#glassGradient); stroke:none;"
transform="translate(44 44)" />
</g>
</svg>
Side by side

Before

Width:  |  Height:  |  Size: 2 KiB

2
slixfeed/configs/information.toml → slixfeed/assets/information.toml
View file

@ -3,7 +3,7 @@ info = """
Slixfeed is a news broker bot for syndicated news which aims to be \ Slixfeed is a news broker bot for syndicated news which aims to be \
an easy to use and fully-featured news aggregating bot. an easy to use and fully-featured news aggregating bot.
Slixfeed provides a convenient access to Blogs, News websites and \ Slixfeed provides a convenient access to Blogs, News sites and \
even Fediverse instances, along with filtering and other privacy \ even Fediverse instances, along with filtering and other privacy \
driven functionalities. driven functionalities.

0
slixfeed/configs/lists.toml → slixfeed/assets/lists.toml
View file

0
slixfeed/configs/proxies.toml → slixfeed/assets/proxies.toml
View file

0
slixfeed/configs/queries.toml → slixfeed/assets/queries.toml
View file

1
slixfeed/configs/settings.toml → slixfeed/assets/settings.toml
View file

@ -11,7 +11,6 @@ interval = 300 # Update interval (Minimum value 10)
length = 300 # Maximum length of summary (Value 0 to disable) length = 300 # Maximum length of summary (Value 0 to disable)
media = 0 # Display media (audio, image, video) when available media = 0 # Display media (audio, image, video) when available
old = 0 # Mark entries of newly added entries as unread old = 0 # Mark entries of newly added entries as unread
omemo = 1 # Encrypt messages with OMEMO
quantum = 3 # Amount of entries per update quantum = 3 # Amount of entries per update
random = 0 # Pick random item from database random = 0 # Pick random item from database

562
slixfeed/config.py
View file

@ -3,6 +3,12 @@
""" """
FIXME
1) Use dict for ConfigDefault
2) Store ConfigJabberID in dicts
TODO TODO
1) Site-specific filter (i.e. audiobookbay). 1) Site-specific filter (i.e. audiobookbay).
@ -15,6 +21,14 @@ TODO
4) Copy file from /etc/slixfeed/ or /usr/share/slixfeed/ 4) Copy file from /etc/slixfeed/ or /usr/share/slixfeed/
5) Merge get_value_default into get_value.
6) Use TOML https://ruudvanasseldonk.com/2023/01/11/the-yaml-document-from-hell
7) Make the program portable (directly use the directory assets) -- Thorsten
7.1) Read missing files from base directories or either set error message.
""" """
import configparser import configparser
@ -31,157 +45,33 @@ except:
logger = Logger(__name__) logger = Logger(__name__)
class Settings: # TODO Consider a class ConfigDefault for default values to be initiate at most
# basic level possible and a class ConfigJID for each JID (i.e. db_file) to be
def get_directory(): # also initiated at same level or at least at event call, then check whether
""" # setting_jid.setting_key has value, otherwise resort to setting_default.setting_key.
Determine the directory path where setting files be stored.
* If $XDG_CONFIG_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
Returns
-------
str
Path to configuration directory.
"""
# config_home = xdg.BaseDirectory.xdg_config_home
config_home = os.environ.get('XDG_CONFIG_HOME')
if config_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
config_home = os.environ.get('APPDATA')
if config_home is None:
return os.path.abspath('.')
else:
return os.path.abspath('.')
else:
config_home = os.path.join(
os.environ.get('HOME'), '.config'
)
return os.path.join(config_home, 'slixfeed')
class Share:
def get_directory():
"""
Determine the directory path where data files be stored.
* If $XDG_DATA_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
Returns
-------
str
Path to database file.
"""
# data_home = xdg.BaseDirectory.xdg_data_home
data_home = os.environ.get('XDG_DATA_HOME')
if data_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
data_home = os.environ.get('APPDATA')
if data_home is None:
return os.path.abspath('.slixfeed/data')
else:
return os.path.abspath('.slixfeed/data')
else:
data_home = os.path.join(
os.environ.get('HOME'), '.local', 'share'
)
return os.path.join(data_home, 'slixfeed')
class Cache:
def get_directory():
"""
Determine the directory path where cache files be stored.
* If $XDG_CACHE_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
Returns
-------
str
Path to cache directory.
"""
# cache_home = xdg.BaseDirectory.xdg_cache_home
cache_home = os.environ.get('XDG_CACHE_HOME')
if cache_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
cache_home = os.environ.get('APPDATA')
if cache_home is None:
return os.path.abspath('.slixfeed/cache')
else:
return os.path.abspath('.slixfeed/cache')
else:
cache_home = os.path.join(
os.environ.get('HOME'), '.cache'
)
return os.path.join(cache_home, 'slixfeed')
class Config: class Config:
def get_directory(): def add_settings_default(settings):
""" settings_default = get_values('settings.toml', 'settings')
Determine the directory path where setting files be stored. settings['default'] = settings_default
* If $XDG_CONFIG_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
Returns
-------
str
Path to configuration directory.
"""
# config_home = xdg.BaseDirectory.xdg_config_home
config_home = os.environ.get('XDG_CONFIG_HOME')
if config_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
config_home = os.environ.get('APPDATA')
if config_home is None:
return os.path.abspath('.')
else:
return os.path.abspath('.')
else:
config_home = os.path.join(
os.environ.get('HOME'), '.config'
)
return os.path.join(config_home, 'slixfeed')
def update_toml_file(filename, data):
with open(filename, 'w') as new_file:
content = tomli_w.dumps(data)
new_file.write(content)
# TODO Open SQLite file once # TODO Open SQLite file once
def add_settings_jid(self, jid_bare, db_file): def add_settings_jid(settings, jid_bare, db_file):
self.settings[jid_bare] = {} settings[jid_bare] = {}
for key in self.defaults['default']: for key in ('archive', 'enabled', 'filter', 'formatting', 'interval',
'length', 'media', 'old', 'quantum'):
value = sqlite.get_setting_value(db_file, key) value = sqlite.get_setting_value(db_file, key)
if value: if value: settings[jid_bare][key] = value[0]
self.settings[jid_bare][key] = value[0]
elif key not in ('check', 'formatting'):
# NOTE This might neglects the need for
# self.defaults of get_setting_value
self.settings[jid_bare][key] = self.defaults['default'][key]
async def set_setting_value(self, jid_bare, db_file, key, val): def get_settings_xmpp(key=None):
result = get_values('accounts.toml', 'xmpp')
result = result[key] if key else result
return result
async def set_setting_value(settings, jid_bare, db_file, key, val):
key = key.lower() key = key.lower()
key_val = [key, val] key_val = [key, val]
self.settings[jid_bare][key] = val settings[jid_bare][key] = val
if sqlite.is_setting_key(db_file, key): if sqlite.is_setting_key(db_file, key):
await sqlite.update_setting_value(db_file, key_val) await sqlite.update_setting_value(db_file, key_val)
else: else:
@ -189,18 +79,262 @@ class Config:
# TODO Segregate Jabber ID settings from Slixfeed wide settings. # TODO Segregate Jabber ID settings from Slixfeed wide settings.
# self.settings, self.settings_xmpp, self.settings_irc etc. # self.settings, self.settings_xmpp, self.settings_irc etc.
def get_setting_value(self, jid_bare, key): def get_setting_value(settings, jid_bare, key):
if jid_bare in self.settings and key in self.settings[jid_bare]: if jid_bare in settings and key in settings[jid_bare]:
value = self.settings[jid_bare][key] value = settings[jid_bare][key]
else: else:
value = self.defaults['default'][key] value = settings['default'][key]
return value
class ConfigNetwork:
def __init__(self, settings):
settings['network'] = {}
for key in ('http_proxy', 'user_agent'):
value = get_value('settings', 'Network', key)
settings['network'][key] = value
class ConfigJabberID:
def __init__(self, settings, jid_bare, db_file):
settings[jid_bare] = {}
for key in ('archive', 'enabled', 'filter', 'formatting', 'interval',
'length', 'media', 'old', 'quantum'):
value = sqlite.get_setting_value(db_file, key)
if value: value = value[0]
print(value)
settings[jid_bare][key] = value
def get_values(filename, key=None):
config_dir = get_default_config_directory()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
if not os.path.isdir(config_dir):
config_dir = os.path.dirname(__file__) + "/assets"
config_file = os.path.join(config_dir, filename)
with open(config_file, mode="rb") as defaults:
result = tomllib.load(defaults)
values = result[key] if key else result
return values
def get_setting_value(db_file, key):
value = sqlite.get_setting_value(db_file, key)
if value:
value = value[0]
else:
value = get_value('settings', 'Settings', key)
# try:
# value = int(value)
# except ValueError as e:
# print('ValueError for value {} (key {}):\n{}'.format(value, key, e))
# if isinstance(value, bool):
# if value:
# value = 1
# else:
# value = 0
return value return value
class Data: # TODO Merge with backup_obsolete
def update_proxies(file, proxy_name, proxy_type, proxy_url, action='remove'):
"""
Add given URL to given list.
Parameters
----------
file : str
Filename.
proxy_name : str
Proxy name.
proxy_type : str
Proxy title.
proxy_url : str
Proxy URL.
action : str
add or remove
Returns
-------
None.
"""
data = open_config_file('proxies.toml')
proxy_list = data['proxies'][proxy_name][proxy_type]
# breakpoint()
print('####################### PROXY ######################')
proxy_index = proxy_list.index(proxy_url)
proxy_list.pop(proxy_index)
with open(file, 'w') as new_file:
content = tomli_w.dumps(data)
new_file.write(content)
def get_directory(): # TODO Merge with update_proxies
def backup_obsolete(file, proxy_name, proxy_type, proxy_url, action='add'):
"""
Add given URL to given list.
Parameters
----------
file : str
Filename.
proxy_name : str
Proxy name.
proxy_type : str
Proxy title.
proxy_url : str
Proxy URL.
action : str
add or remove
Returns
-------
None.
"""
data = open_config_file('proxies_obsolete.toml')
proxy_list = data['proxies'][proxy_name][proxy_type]
proxy_list.extend([proxy_url])
with open(file, 'w') as new_file:
content = tomli_w.dumps(data)
new_file.write(content)
def create_skeleton(file):
with open(file, 'rb') as original_file:
data = tomllib.load(original_file)
data = clear_values(data)
with open('proxies_obsolete.toml', 'w') as new_file:
content = tomli_w.dumps(data)
new_file.write(content)
def clear_values(input):
if isinstance(input, dict):
return {k: clear_values(v) for k, v in input.items()}
elif isinstance(input, list):
return ['']
else:
return ''
# TODO Return dict instead of list
def get_value(filename, section, keys):
"""
Get setting value.
Parameters
----------
filename : str
INI filename.
keys : list or str
A single key as string or multiple keys as list.
section : str
INI Section.
Returns
-------
result : list or str
A single value as string or multiple values as list.
"""
result = None
config_res = configparser.RawConfigParser()
config_dir = get_default_config_directory()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
if not os.path.isdir(config_dir):
config_dir = os.path.dirname(__file__) + "/assets"
config_file = os.path.join(config_dir, filename + ".ini")
config_res.read(config_file)
if config_res.has_section(section):
section_res = config_res[section]
if isinstance(keys, list):
result = []
for key in keys:
if key in section_res:
value = section_res[key]
logger.debug("Found value {} for key {}".format(value, key))
else:
value = ''
logger.debug("Missing key:", key)
result.extend([value])
elif isinstance(keys, str):
key = keys
if key in section_res:
result = section_res[key]
logger.debug("Found value {} for key {}".format(result, key))
else:
result = ''
# logger.error("Missing key:", key)
if result == None:
logger.error(
"Check configuration file {}.ini for "
"missing key(s) \"{}\" under section [{}].".format(
filename, keys, section)
)
else:
return result
# TODO Store config file as an object in runtime, otherwise
# the file will be opened time and time again.
# TODO Copy file from /etc/slixfeed/ or /usr/share/slixfeed/
def get_value_default(filename, section, key):
"""
Get settings default value.
Parameters
----------
key : str
Key: archive, enabled, interval,
length, old, quantum, random.
Returns
-------
result : str
Value.
"""
config_res = configparser.RawConfigParser()
config_dir = get_default_config_directory()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
config_file = os.path.join(config_dir, filename + ".ini")
config_res.read(config_file)
if config_res.has_section(section):
result = config_res[section][key]
return result
# TODO DELETE THIS FUNCTION OR KEEP ONLY THE CODE BELOW NOTE
# IF CODE BELOW NOTE IS KEPT, RENAME FUNCTION TO open_toml
def open_config_file(filename):
"""
Get settings default value.
Parameters
----------
filename : str
Filename of toml file.
Returns
-------
result : list
List of pathnames or keywords.
"""
config_dir = get_default_config_directory()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
if not os.path.isdir(config_dir):
config_dir = os.path.dirname(__file__) + "/assets"
config_file = os.path.join(config_dir, filename)
# NOTE THIS IS THE IMPORTANT CODE
with open(config_file, mode="rb") as defaults:
# default = yaml.safe_load(defaults)
# result = default[key]
result = tomllib.load(defaults)
return result
def get_default_data_directory():
""" """
Determine the directory path where dbfile will be stored. Determine the directory path where dbfile will be stored.
@ -240,47 +374,105 @@ class Data:
return os.path.join(data_home, 'slixfeed') return os.path.join(data_home, 'slixfeed')
def get_values(filename, key=None): def get_default_cache_directory():
config_dir = get_default_config_directory() """
if not os.path.isdir(config_dir): Determine the directory path where dbfile will be stored.
config_dir = '/usr/share/slixfeed/'
if not os.path.isdir(config_dir):
config_dir = os.path.dirname(__file__) + "/assets"
config_file = os.path.join(config_dir, filename)
with open(config_file, mode="rb") as defaults:
result = tomllib.load(defaults)
values = result[key] if key else result
return values
* If $XDG_DATA_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
def get_setting_value(db_file, key): Returns
value = sqlite.get_setting_value(db_file, key) -------
if value: str
value = value[0] Path to cache directory.
"""
# data_home = xdg.BaseDirectory.xdg_data_home
data_home = os.environ.get('XDG_CACHE_HOME')
if data_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
data_home = os.environ.get('APPDATA')
if data_home is None:
return os.path.abspath('.slixfeed/cache')
else: else:
value = get_value('settings', 'Settings', key) return os.path.abspath('.slixfeed/cache')
# try: else:
# value = int(value) data_home = os.path.join(
# except ValueError as e: os.environ.get('HOME'), '.cache'
# print('ValueError for value {} (key {}):\n{}'.format(value, key, e)) )
# if isinstance(value, bool): return os.path.join(data_home, 'slixfeed')
# if value:
# value = 1
# TODO Write a similar function for file.
# NOTE the is a function of directory, noot file.
def get_default_config_directory():
"""
Determine the directory path where configuration will be stored.
* If $XDG_CONFIG_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
Returns
-------
str
Path to configuration directory.
"""
# config_home = xdg.BaseDirectory.xdg_config_home
config_home = os.environ.get('XDG_CONFIG_HOME')
if config_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
config_home = os.environ.get('APPDATA')
if config_home is None:
return os.path.abspath('.')
else:
return os.path.abspath('.')
else:
config_home = os.path.join(
os.environ.get('HOME'), '.config'
)
return os.path.join(config_home, 'slixfeed')
def get_pathname_to_database(jid_file):
"""
Callback function to instantiate action on database.
Parameters
----------
jid_file : str
Filename.
callback : ?
Function name.
message : str, optional
Optional kwarg when a message is a part or
required argument. The default is None.
Returns
-------
object
Coroutine object.
"""
db_dir = get_default_data_directory()
if not os.path.isdir(db_dir):
os.mkdir(db_dir)
if not os.path.isdir(db_dir + "/sqlite"):
os.mkdir(db_dir + "/sqlite")
db_file = os.path.join(db_dir, "sqlite", r"{}.db".format(jid_file))
sqlite.create_tables(db_file)
return db_file
# await set_default_values(db_file)
# if message:
# return await callback(db_file, message)
# else: # else:
# value = 0 # return await callback(db_file)
return value
def clear_values(input): async def add_to_list(newwords, keywords):
if isinstance(input, dict):
return {k: clear_values(v) for k, v in input.items()}
elif isinstance(input, list):
return ['']
else:
return ''
def add_to_list(newwords, keywords):
""" """
Append new keywords to list. Append new keywords to list.
@ -311,7 +503,7 @@ def add_to_list(newwords, keywords):
return val return val
def remove_from_list(newwords, keywords): async def remove_from_list(newwords, keywords):
""" """
Remove given keywords from list. Remove given keywords from list.

1
slixfeed/configs/proxies_obsolete.toml
View file

@ -1 +0,0 @@
proxies = {}

28
slixfeed/configs/selector.toml
View file

@ -1,28 +0,0 @@
enabled = 0
blacklist = [
"christian@iceagefarmer.com",
"david@falkon.org",
"diggy@diggy.club",
"dino@drdino.com",
"doug@baxterisland.com",
"doug@blacklistednews.com",
"emdek@otter-browser.org",
"eric@drdino.com",
"gemini@circumlunar.space",
"hal@reallibertymedia.com",
"henrik@redice.tv",
"jg@cwns.i2p",
"jo@drdino.com",
"joel@thelunaticfarmer.com",
"kent@drdino.com",
"lana@redice.tv",
"larken@larkenrose.com",
"lee@oraclebroadcasting.com",
"mark@enclosedworld.com",
"mark@marksargent.com",
"nick@nightnationreview.com",
"oliver@postmarketos.org",
"robert@cwns.i2p",
"patrick@slackware.com",
]
whitelist = []

56
slixfeed/documentation/adhoc_commands_muc.md
View file

@ -1,56 +0,0 @@
# Set Slixfeed Ad-Hoc Commands in MUC
This documents provides instructions for setting Slixfeed Ad-Hoc Commands on your XMPP server
These instruction are currently applied only to Prosody XMPP server.
We encourage to contribute instructions for other XMPP servers.
## Prosody
First of all install the relative Community Module:
```
$ sudo prosodyctl install --server=https://modules.prosody.im/rocks/ mod_muc_adhoc_bots
```
Then enable the module in your **MUC component** (`/etc/prosody/prosody.cfg.lua`), like this:
```
modules_enabled = {
"muc_mam",
"vcard_muc",
…
"muc_adhoc_bots",
…
"server_contact_info"
}
```
Last part is the bot's configuration, which goes again under the MUC component settings:
```
adhoc_bots = { "bot@jabber.i2p/slixfeed" }
```
Substitute `bot@jabber.i2p/slixfeed` with your bot JID and device name which has to correspond to `accounts.toml` settings for Slixfeed configuration:
```
[xmpp.client]
alias = "Slixfeed"
jid = "bot@jabber.i2p/slixfeed"
```
Reload the Prosody config and then load the module you just enabled under MUC component, or simply restart the XMPP server.
```
$ sudo prosodyctl shell
prosody> config:reload()
prosody> module:load('muc_adhoc_bots', "muc_component.jabber.i2p")
prosody> bye
```
Authors:
- Simone Canaletti (roughnecks)

127
slixfeed/fetch.py
View file

@ -36,17 +36,15 @@ NOTE
""" """
import aiofiles
from aiohttp import ClientError, ClientSession, ClientTimeout from aiohttp import ClientError, ClientSession, ClientTimeout
from asyncio import TimeoutError from asyncio import TimeoutError
# from asyncio.exceptions import IncompleteReadError # from asyncio.exceptions import IncompleteReadError
# from http.client import IncompleteRead # from http.client import IncompleteRead
# from lxml import html # from lxml import html
# from xml.etree.ElementTree import ElementTree, ParseError # from xml.etree.ElementTree import ElementTree, ParseError
#import requests import requests
import slixfeed.config as config
from slixfeed.log import Logger from slixfeed.log import Logger
# import urllib.request
# from urllib.error import HTTPError
logger = Logger(__name__) logger = Logger(__name__)
@ -57,6 +55,7 @@ except:
"Package magnet2torrent was not found.\n" "Package magnet2torrent was not found.\n"
"BitTorrent is disabled.") "BitTorrent is disabled.")
# class Dat: # class Dat:
# async def dat(): # async def dat():
@ -69,114 +68,14 @@ except:
# class Gopher: # class Gopher:
# async def gopher(): # async def gopher():
# class Http:
# async def http():
# class Ipfs: # class Ipfs:
# async def ipfs(): # async def ipfs():
class Http: def http_response(url):
# def fetch_media(url, pathname):
# try:
# urllib.request.urlretrieve(url, pathname)
# status = 1
# except HTTPError as e:
# logger.error(e)
# status = 0
# return status
async def fetch_headers(settings_network, url):
user_agent = (settings_network['user_agent'] or 'Slixfeed/0.1')
headers = {'User-Agent': user_agent}
proxy = (settings_network['http_proxy'] or None)
timeout = ClientTimeout(total=10)
async with ClientSession(headers=headers) as session:
async with session.get(url, proxy=proxy,
# proxy_auth=(proxy_username, proxy_password),
timeout=timeout
) as response:
headers = response.headers
return headers
# print("Headers for URL:", url)
# for header_name, header_value in headers.items():
# print(f"{header_name}: {header_value}")
# TODO Write file to disk. Consider aiofiles
async def fetch_media(settings_network, url, pathname):
"""
Download media content of given URL.
Parameters
----------
url : str
URL.
pathname : list
Pathname (including filename) to save content to.
Returns
-------
msg: list or str
Document or error message.
"""
user_agent = (settings_network['user_agent'] or 'Slixfeed/0.1')
headers = {'User-Agent': user_agent}
proxy = (settings_network['http_proxy'] or None)
timeout = ClientTimeout(total=10)
async with ClientSession(headers=headers) as session:
# async with ClientSession(trust_env=True) as session:
try:
async with session.get(url, proxy=proxy,
# proxy_auth=(proxy_username, proxy_password),
timeout=timeout
) as response:
status = response.status
if status in (200, 201):
f = await aiofiles.open(pathname, mode='wb')
await f.write(await response.read())
await f.close()
try:
result = {'charset': response.charset,
'content_length': response.content_length,
'content_type': response.content_type,
'error': False,
'message': None,
'original_url': url,
'status_code': status,
'response_url': response.url}
except:
result = {'error': True,
'message': 'Could not get document.',
'original_url': url,
'status_code': status,
'response_url': response.url}
else:
result = {'error': True,
'message': 'HTTP Error:' + str(status),
'original_url': url,
'status_code': status,
'response_url': response.url}
except ClientError as e:
result = {'error': True,
'message': 'Error:' + str(e) if e else 'ClientError',
'original_url': url,
'status_code': None}
except TimeoutError as e:
result = {'error': True,
'message': 'Timeout:' + str(e) if e else 'TimeoutError',
'original_url': url,
'status_code': None}
except Exception as e:
logger.error(e)
result = {'error': True,
'message': 'Error:' + str(e) if e else 'Error',
'original_url': url,
'status_code': None}
return result
def http_response(settings_network, url):
""" """
Download response headers. Download response headers.
@ -198,7 +97,10 @@ class Http:
response.status_code response.status_code
response.url response.url
""" """
user_agent = settings_network['user_agent'] or 'Slixfeed/0.1' user_agent = (
config.get_value(
"settings", "Network", "user_agent")
) or 'Slixfeed/0.1'
headers = { headers = {
"User-Agent": user_agent "User-Agent": user_agent
} }
@ -214,7 +116,7 @@ class Http:
return response return response
async def http(settings_network, url): async def http(url):
""" """
Download content of given URL. Download content of given URL.
@ -228,9 +130,10 @@ async def http(settings_network, url):
msg: list or str msg: list or str
Document or error message. Document or error message.
""" """
user_agent = (settings_network['user_agent'] or 'Slixfeed/0.1') network_settings = config.get_values('settings.toml', 'network')
user_agent = (network_settings['user_agent'] or 'Slixfeed/0.1')
headers = {'User-Agent': user_agent} headers = {'User-Agent': user_agent}
proxy = (settings_network['http_proxy'] or None) proxy = (network_settings['http_proxy'] or None)
timeout = ClientTimeout(total=10) timeout = ClientTimeout(total=10)
async with ClientSession(headers=headers) as session: async with ClientSession(headers=headers) as session:
# async with ClientSession(trust_env=True) as session: # async with ClientSession(trust_env=True) as session:

5
slixfeed/log.py
View file

@ -19,8 +19,6 @@ import logging
class Logger: class Logger:
def set_logging_level(level):
logging.basicConfig(level)
def __init__(self, name): def __init__(self, name):
self.logger = logging.getLogger(name) self.logger = logging.getLogger(name)
@ -60,5 +58,4 @@ class Message:
def printer(text): def printer(text):
now = datetime.now() now = datetime.now()
current_time = now.strftime("%H:%M:%S") current_time = now.strftime("%H:%M:%S")
# print('{} {}'.format(current_time, text), end='\r') print('{} {}'.format(current_time, text), end='\r')
print('{} {}'.format(current_time, text))

37
slixfeed/sqlite.py
View file

@ -366,7 +366,7 @@ def create_tables(db_file):
id INTEGER NOT NULL, id INTEGER NOT NULL,
feed_id INTEGER NOT NULL, feed_id INTEGER NOT NULL,
tag_id INTEGER NOT NULL, tag_id INTEGER NOT NULL,
FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id")
ON UPDATE CASCADE ON UPDATE CASCADE
ON DELETE CASCADE, ON DELETE CASCADE,
FOREIGN KEY ("tag_id") REFERENCES "tags" ("id") FOREIGN KEY ("tag_id") REFERENCES "tags" ("id")
@ -2762,39 +2762,6 @@ def get_active_feeds_url(db_file):
return result return result
def get_active_feeds_url_sorted_by_last_scanned(db_file):
"""
Query table feeds for active URLs and sort them by last scanned time.
Parameters
----------
db_file : str
Path to database file.
Returns
-------
result : tuple
URLs of active feeds.
"""
function_name = sys._getframe().f_code.co_name
logger.debug('{}: db_file: {}'
.format(function_name, db_file))
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = (
"""
SELECT feeds_properties.url
FROM feeds_properties
INNER JOIN feeds_preferences ON feeds_properties.id = feeds_preferences.feed_id
INNER JOIN feeds_state ON feeds_properties.id = feeds_state.feed_id
WHERE feeds_preferences.enabled = 1
ORDER BY feeds_state.scanned
"""
)
result = cur.execute(sql).fetchall()
return result
def get_tags(db_file): def get_tags(db_file):
""" """
Query table tags and list items. Query table tags and list items.
@ -3032,7 +2999,7 @@ def check_entry_exist(db_file, feed_id, identifier=None, title=None, link=None,
""" """
SELECT id SELECT id
FROM entries_properties FROM entries_properties
WHERE identifier = :identifier AND feed_id = :feed_id WHERE identifier = :identifier and feed_id = :feed_id
""" """
) )
par = { par = {

100
slixfeed/syndication.py
View file

@ -27,11 +27,12 @@ TODO
import asyncio import asyncio
from feedparser import parse from feedparser import parse
import os import os
import slixfeed.config as config
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.fetch as fetch import slixfeed.fetch as fetch
from slixfeed.log import Logger,Message from slixfeed.log import Logger,Message
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
from slixfeed.utilities import Database, DateAndTime, Html, MD, String, Url, Utilities from slixfeed.utilities import DateAndTime, Html, MD, String, Url, Utilities
from slixmpp.xmlstream import ET from slixmpp.xmlstream import ET
import sys import sys
from urllib.parse import urlsplit from urllib.parse import urlsplit
@ -43,16 +44,17 @@ logger = Logger(__name__)
class Feed: class Feed:
# NOTE Consider removal of MD (and any other option HTML and XBEL) # NOTE Consider removal of MD (and any other option HTML and XBEL)
def export_feeds(dir_data, dir_cache, jid_bare, ext): def export_feeds(jid_bare, ext):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_bare: {}: ext: {}'.format(function_name, jid_bare, ext)) logger.debug('{}: jid_bare: {}: ext: {}'.format(function_name, jid_bare, ext))
if not os.path.isdir(dir_cache): cache_dir = config.get_default_cache_directory()
os.mkdir(dir_cache) if not os.path.isdir(cache_dir):
if not os.path.isdir(dir_cache + '/' + ext): os.mkdir(cache_dir)
os.mkdir(dir_cache + '/' + ext) if not os.path.isdir(cache_dir + '/' + ext):
os.mkdir(cache_dir + '/' + ext)
filename = os.path.join( filename = os.path.join(
dir_cache, ext, 'slixfeed_' + DateAndTime.timestamp() + '.' + ext) cache_dir, ext, 'slixfeed_' + DateAndTime.timestamp() + '.' + ext)
db_file = Database.instantiate(dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
results = sqlite.get_feeds(db_file) results = sqlite.get_feeds(db_file)
match ext: match ext:
# case 'html': # case 'html':
@ -272,7 +274,7 @@ class Feed:
feed_id = sqlite.get_feed_id(db_file, url) feed_id = sqlite.get_feed_id(db_file, url)
if not feed_id: if not feed_id:
if not sqlite.check_identifier_exist(db_file, identifier): if not sqlite.check_identifier_exist(db_file, identifier):
result = await fetch.http(self.settings_network, url) result = await fetch.http(url)
message = result['message'] message = result['message']
status_code = result['status_code'] status_code = result['status_code']
if not result['error']: if not result['error']:
@ -346,9 +348,8 @@ class Feed:
if new_entries: if new_entries:
await sqlite.add_entries_and_update_feed_state( await sqlite.add_entries_and_update_feed_state(
db_file, feed_id, new_entries) db_file, feed_id, new_entries)
old = self.settings[jid_bare]['old'] or self.defaults['default']['old'] old = Config.get_setting_value(self.settings, jid_bare, 'old')
if not old: await sqlite.mark_feed_as_read(db_file, if not old: await sqlite.mark_feed_as_read(db_file, feed_id)
feed_id)
result_final = {'link' : url, result_final = {'link' : url,
'index' : feed_id, 'index' : feed_id,
'name' : title, 'name' : title,
@ -362,8 +363,7 @@ class Feed:
# NOTE Do not be tempted to return a compact dictionary. # NOTE Do not be tempted to return a compact dictionary.
# That is, dictionary within dictionary # That is, dictionary within dictionary
# Return multiple dictionaries in a list or tuple. # Return multiple dictionaries in a list or tuple.
result = await FeedDiscovery.probe_page( result = await FeedDiscovery.probe_page(url, document)
self.settings_network, self.pathnames, url, document)
if not result: if not result:
# Get out of the loop with dict indicating error. # Get out of the loop with dict indicating error.
result_final = {'link' : url, result_final = {'link' : url,
@ -521,7 +521,7 @@ class Feed:
# NOTE This function is not being utilized # NOTE This function is not being utilized
async def download_feed(settings_network, db_file, feed_url): async def download_feed(self, db_file, feed_url):
""" """
Process feed content. Process feed content.
@ -536,7 +536,7 @@ class Feed:
logger.debug('{}: db_file: {} url: {}' logger.debug('{}: db_file: {} url: {}'
.format(function_name, db_file, feed_url)) .format(function_name, db_file, feed_url))
if isinstance(feed_url, tuple): feed_url = feed_url[0] if isinstance(feed_url, tuple): feed_url = feed_url[0]
result = await fetch.http(settings_network, feed_url) result = await fetch.http(feed_url)
feed_id = sqlite.get_feed_id(db_file, feed_url) feed_id = sqlite.get_feed_id(db_file, feed_url)
feed_id = feed_id[0] feed_id = feed_id[0]
status_code = result['status_code'] status_code = result['status_code']
@ -933,7 +933,7 @@ class FeedDiscovery:
# else: # else:
# return await callback(url) # return await callback(url)
async def probe_page(settings_network, pathnames, url, document=None): async def probe_page(url, document=None):
""" """
Parameters Parameters
---------- ----------
@ -948,7 +948,7 @@ class FeedDiscovery:
Single URL as list or selection of URLs as str. Single URL as list or selection of URLs as str.
""" """
if not document: if not document:
response = await fetch.http(settings_network, url) response = await fetch.http(url)
if not response['error']: if not response['error']:
document = response['content'] document = response['content']
try: try:
@ -974,7 +974,7 @@ class FeedDiscovery:
result = None result = None
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
logger.warning(f"Failed to parse URL as feed for {url}.") logger.warning("Failed to parse URL as feed for {}.".format(url))
result = {'link' : None, result = {'link' : None,
'index' : None, 'index' : None,
'name' : None, 'name' : None,
@ -982,23 +982,23 @@ class FeedDiscovery:
'error' : True, 'error' : True,
'exist' : None} 'exist' : None}
if not result: if not result:
logger.debug(f"Feed auto-discovery engaged for {url}") logger.debug("Feed auto-discovery engaged for {}".format(url))
result = FeedDiscovery.feed_mode_auto_discovery(url, tree) result = FeedDiscovery.feed_mode_auto_discovery(url, tree)
if not result: if not result:
logger.debug(f"Feed link scan mode engaged for {url}") logger.debug("Feed link scan mode engaged for {}".format(url))
result = FeedDiscovery.feed_mode_scan(url, tree, pathnames) result = FeedDiscovery.feed_mode_scan(url, tree)
if not result: if not result:
logger.debug(f"Feed arbitrary mode engaged for {url}") logger.debug("Feed arbitrary mode engaged for {}".format(url))
result = FeedDiscovery.feed_mode_guess(url, pathnames) result = FeedDiscovery.feed_mode_guess(url, tree)
if not result: if not result:
logger.debug(f"No feeds were found for {url}") logger.debug("No feeds were found for {}".format(url))
result = None result = None
result = await FeedDiscovery.process_feed_selection(settings_network, url, result) result = await FeedDiscovery.process_feed_selection(url, result)
return result return result
# TODO Improve scan by gradual decreasing of path # TODO Improve scan by gradual decreasing of path
def feed_mode_guess(url, pathnames): def feed_mode_guess(url, tree):
""" """
Lookup for feeds by pathname using HTTP Requests. Lookup for feeds by pathname using HTTP Requests.
@ -1008,8 +1008,8 @@ class FeedDiscovery:
Path to database file. Path to database file.
url : str url : str
URL. URL.
pathnames : list tree : TYPE
pathnames. DESCRIPTION.
Returns Returns
------- -------
@ -1018,17 +1018,18 @@ class FeedDiscovery:
""" """
urls = [] urls = []
parted_url = urlsplit(url) parted_url = urlsplit(url)
paths = config.open_config_file("lists.toml")["pathnames"]
# Check whether URL has path (i.e. not root) # Check whether URL has path (i.e. not root)
# Check parted_url.path to avoid error in case root wasn't given # Check parted_url.path to avoid error in case root wasn't given
# TODO Make more tests # TODO Make more tests
if parted_url.path and parted_url.path.split('/')[1]: if parted_url.path and parted_url.path.split('/')[1]:
pathnames.extend( paths.extend(
[".atom", ".feed", ".rdf", ".rss"] [".atom", ".feed", ".rdf", ".rss"]
) if '.rss' not in pathnames else -1 ) if '.rss' not in paths else -1
# if paths.index('.rss'): # if paths.index('.rss'):
# paths.extend([".atom", ".feed", ".rdf", ".rss"]) # paths.extend([".atom", ".feed", ".rdf", ".rss"])
parted_url_path = parted_url.path if parted_url.path else '/' parted_url_path = parted_url.path if parted_url.path else '/'
for path in pathnames: for path in paths:
address = Url.join_url(url, parted_url_path.split('/')[1] + path) address = Url.join_url(url, parted_url_path.split('/')[1] + path)
if address not in urls: if address not in urls:
urls.extend([address]) urls.extend([address])
@ -1037,7 +1038,7 @@ class FeedDiscovery:
return urls return urls
def feed_mode_scan(url, tree, pathnames): def feed_mode_scan(url, tree):
""" """
Scan page for potential feeds by pathname. Scan page for potential feeds by pathname.
@ -1056,7 +1057,8 @@ class FeedDiscovery:
Message with URLs. Message with URLs.
""" """
urls = [] urls = []
for path in pathnames: paths = config.open_config_file("lists.toml")["pathnames"]
for path in paths:
# xpath_query = "//*[@*[contains(.,'{}')]]".format(path) # xpath_query = "//*[@*[contains(.,'{}')]]".format(path)
# xpath_query = "//a[contains(@href,'{}')]".format(path) # xpath_query = "//a[contains(@href,'{}')]".format(path)
num = 5 num = 5
@ -1138,10 +1140,10 @@ class FeedDiscovery:
# URLs (string) and Feeds (dict) and function that # URLs (string) and Feeds (dict) and function that
# composes text message (string). # composes text message (string).
# Maybe that's not necessary. # Maybe that's not necessary.
async def process_feed_selection(settings_network, url, urls): async def process_feed_selection(url, urls):
feeds = {} feeds = {}
for i in urls: for i in urls:
result = await fetch.http(settings_network, i) result = await fetch.http(i)
if not result['error']: if not result['error']:
document = result['content'] document = result['content']
status_code = result['status_code'] status_code = result['status_code']
@ -1273,10 +1275,10 @@ class FeedTask:
# print('Scanning for updates for JID {}'.format(jid_bare)) # print('Scanning for updates for JID {}'.format(jid_bare))
logger.info('Scanning for updates for JID {}'.format(jid_bare)) logger.info('Scanning for updates for JID {}'.format(jid_bare))
while True: while True:
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
urls = sqlite.get_active_feeds_url_sorted_by_last_scanned(db_file) urls = sqlite.get_active_feeds_url(db_file)
for url in urls: for url in urls:
#Message.printer('Scanning updates for URL {} ...'.format(url)) Message.printer('Scanning updates for URL {} ...'.format(url))
url = url[0] url = url[0]
# print('STA',url) # print('STA',url)
@ -1286,7 +1288,7 @@ class FeedTask:
# print('Skipping URL:', url) # print('Skipping URL:', url)
# continue # continue
result = await fetch.http(self.settings_network, url) result = await fetch.http(url)
status_code = result['status_code'] status_code = result['status_code']
feed_id = sqlite.get_feed_id(db_file, url) feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0] feed_id = feed_id[0]
@ -1333,7 +1335,7 @@ class FeedTask:
new_entries.extend([new_entry]) new_entries.extend([new_entry])
if new_entries: if new_entries:
await sqlite.add_entries_and_update_feed_state(db_file, feed_id, new_entries) await sqlite.add_entries_and_update_feed_state(db_file, feed_id, new_entries)
limit = Config.get_setting_value(self, jid_bare, 'archive') limit = Config.get_setting_value(self.settings, jid_bare, 'archive')
ixs = sqlite.get_entries_id_of_feed(db_file, feed_id) ixs = sqlite.get_entries_id_of_feed(db_file, feed_id)
ixs_invalid = {} ixs_invalid = {}
for ix in ixs: for ix in ixs:
@ -1358,8 +1360,8 @@ class FeedTask:
# TODO return number of archived entries and add if statement to run archive maintainence function # TODO return number of archived entries and add if statement to run archive maintainence function
await sqlite.maintain_archive(db_file, limit) await sqlite.maintain_archive(db_file, limit)
# await sqlite.process_invalid_entries(db_file, ixs) # await sqlite.process_invalid_entries(db_file, ixs)
await asyncio.sleep(60 * 2) await asyncio.sleep(50)
val = Config.get_setting_value(self, jid_bare, 'check') val = Config.get_setting_value(self.settings, jid_bare, 'check')
await asyncio.sleep(60 * float(val)) await asyncio.sleep(60 * float(val))
# Schedule to call this function again in 90 minutes # Schedule to call this function again in 90 minutes
# loop.call_at( # loop.call_at(
@ -1368,18 +1370,10 @@ class FeedTask:
# self.check_updates(jid) # self.check_updates(jid)
# ) # )
# Consider an endless loop. See XmppPubsubTask.loop_task
# def restart_task(self, jid_bare):
async def loop_task(self, jid_bare):
await asyncio.sleep(60)
while True:
logger.info('Looping task "check" for JID {}'.format(jid_bare))
print('Looping task "check" for JID {}'.format(jid_bare))
await FeedTask.check_updates(self, jid_bare)
await asyncio.sleep(60 * 60)
def restart_task(self, jid_bare): def restart_task(self, jid_bare):
if jid_bare == self.boundjid.bare:
return
if jid_bare not in self.task_manager: if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {} self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {}'.format(jid_bare)) logger.info('Creating new task manager for JID {}'.format(jid_bare))

178
slixfeed/utilities.py
View file

@ -46,9 +46,9 @@ import hashlib
from lxml import etree, html from lxml import etree, html
import os import os
import random import random
import slixfeed.config as config
import slixfeed.fetch as fetch import slixfeed.fetch as fetch
from slixfeed.log import Logger from slixfeed.log import Logger
import slixfeed.sqlite as sqlite
import sys import sys
from urllib.parse import ( from urllib.parse import (
parse_qs, parse_qs,
@ -67,99 +67,6 @@ except:
logger = Logger(__name__) logger = Logger(__name__)
class Config:
def get_default_data_directory():
if os.environ.get('HOME'):
data_home = os.path.join(os.environ.get('HOME'), '.local', 'share')
return os.path.join(data_home, 'kaikout')
elif sys.platform == 'win32':
data_home = os.environ.get('APPDATA')
if data_home is None:
return os.path.join(
os.path.dirname(__file__) + '/kaikout_data')
else:
return os.path.join(os.path.dirname(__file__) + '/kaikout_data')
def get_default_config_directory():
"""
Determine the directory path where configuration will be stored.
* If $XDG_CONFIG_HOME is defined, use it;
* else if $HOME exists, use it;
* else if the platform is Windows, use %APPDATA%;
* else use the current directory.
Returns
-------
str
Path to configuration directory.
"""
# config_home = xdg.BaseDirectory.xdg_config_home
config_home = os.environ.get('XDG_CONFIG_HOME')
if config_home is None:
if os.environ.get('HOME') is None:
if sys.platform == 'win32':
config_home = os.environ.get('APPDATA')
if config_home is None:
return os.path.abspath('.')
else:
return os.path.abspath('.')
else:
config_home = os.path.join(
os.environ.get('HOME'), '.config'
)
return os.path.join(config_home, 'kaikout')
def get_setting_value(db_file, key):
value = sqlite.get_setting_value(db_file, key)
if value:
value = value[0]
else:
value = Config.get_value('settings', 'Settings', key)
return value
def get_values(filename, key=None):
config_dir = Config.get_default_config_directory()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
if not os.path.isdir(config_dir):
config_dir = os.path.dirname(__file__) + "/assets"
config_file = os.path.join(config_dir, filename)
with open(config_file, mode="rb") as defaults:
result = tomllib.load(defaults)
values = result[key] if key else result
return values
class Database:
def instantiate(dir_data, jid_bare):
"""
Instantiate action on database and return its filename location.
Parameters
----------
dir_data : str
Directory.
jid_file : str
Jabber ID.
Returns
-------
db_file
Filename.
"""
db_file = os.path.join(dir_data, 'sqlite', f'{jid_bare}.db')
sqlite.create_tables(db_file)
return db_file
class DateAndTime: class DateAndTime:
#https://feedparser.readthedocs.io/en/latest/date-parsing.html #https://feedparser.readthedocs.io/en/latest/date-parsing.html
@ -183,12 +90,6 @@ class DateAndTime:
return date return date
def convert_seconds_to_yyyy_mm_dd(seconds_time):
date_time = datetime.fromtimestamp(seconds_time)
formatted_date = date_time.strftime('%Y-%m-%d')
return formatted_date
def current_date(): def current_date():
""" """
Print MM DD, YYYY (Weekday Time) timestamp. Print MM DD, YYYY (Weekday Time) timestamp.
@ -278,11 +179,11 @@ class DateAndTime:
class Documentation: class Documentation:
def manual(config_dir, section=None, command=None): def manual(filename, section=None, command=None):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: filename: {}'.format(function_name, config_dir)) logger.debug('{}: filename: {}'.format(function_name, filename))
filename = os.path.join(config_dir, 'commands.toml') config_dir = config.get_default_config_directory()
with open(filename, mode="rb") as commands: with open(config_dir + '/' + filename, mode="rb") as commands:
cmds = tomllib.load(commands) cmds = tomllib.load(commands)
if section == 'all': if section == 'all':
cmd_list = '' cmd_list = ''
@ -316,7 +217,7 @@ class Html:
async def extract_image_from_html(url): async def extract_image_from_html(url):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: url: {}'.format(function_name, url)) logger.debug('{}: url: {}'.format(function_name, url))
result = await fetch.http(settings_network, url) result = await fetch.http(url)
if not result['error']: if not result['error']:
data = result['content'] data = result['content']
tree = html.fromstring(data) tree = html.fromstring(data)
@ -325,7 +226,6 @@ class Html:
'//img[not(' '//img[not('
'contains(@src, "avatar") or ' 'contains(@src, "avatar") or '
'contains(@src, "cc-by-sa") or ' 'contains(@src, "cc-by-sa") or '
'contains(@src, "data:image/") or '
'contains(@src, "emoji") or ' 'contains(@src, "emoji") or '
'contains(@src, "icon") or ' 'contains(@src, "icon") or '
'contains(@src, "logo") or ' 'contains(@src, "logo") or '
@ -442,19 +342,6 @@ class Task:
.format(task, jid_bare)) .format(task, jid_bare))
class Toml:
def open_file(filename: str) -> dict:
with open(filename, mode="rb") as fn:
data = tomllib.load(fn)
return data
def save_file(filename: str, data: dict) -> None:
with open(filename, 'w') as fn:
data_as_string = tomli_w.dumps(data)
fn.write(data_as_string)
""" """
FIXME FIXME
@ -491,23 +378,21 @@ class Url:
return hostname return hostname
async def replace_hostname(configuration_directory, proxies, settings_network, url, url_type): async def replace_hostname(url, url_type):
""" """
Replace hostname. Replace hostname.
Parameters Parameters
---------- ----------
proxies : list
A list of hostnames.
url : str url : str
A URL. URL.
url_type : str url_type : str
A "feed" or a "link". "feed" or "link".
Returns Returns
------- -------
url : str url : str
A processed URL. URL.
""" """
url_new = None url_new = None
parted_url = urlsplit(url) parted_url = urlsplit(url)
@ -517,6 +402,7 @@ class Url:
pathname = parted_url.path pathname = parted_url.path
queries = parted_url.query queries = parted_url.query
fragment = parted_url.fragment fragment = parted_url.fragment
proxies = config.open_config_file('proxies.toml')['proxies']
for proxy_name in proxies: for proxy_name in proxies:
proxy = proxies[proxy_name] proxy = proxies[proxy_name]
if hostname in proxy['hostname'] and url_type in proxy['type']: if hostname in proxy['hostname'] and url_type in proxy['type']:
@ -536,21 +422,26 @@ class Url:
print(proxy_url) print(proxy_url)
print(url_new) print(url_new)
print('>>>') print('>>>')
response = await fetch.http(settings_network, url_new) response = await fetch.http(url_new)
if (response and if (response and
response['status_code'] == 200 and response['status_code'] == 200 and
# response.reason == 'OK' and # response.reason == 'OK' and
url_new.startswith(proxy_url)): break url_new.startswith(proxy_url)):
break
else: else:
proxies_obsolete_file = os.path.join(configuration_directory, 'proxies_obsolete.toml') config_dir = config.get_default_config_directory()
proxies_file = os.path.join(configuration_directory, 'proxies.toml') proxies_obsolete_file = config_dir + '/proxies_obsolete.toml'
breakpoint() proxies_file = config_dir + '/proxies.toml'
proxies_obsolete = Toml.open_file(proxies_obsolete_file) if not os.path.isfile(proxies_obsolete_file):
proxies_obsolete['proxies'][proxy_name][proxy_type].append(proxy_url) config.create_skeleton(proxies_file)
Toml.save_file(proxies_obsolete_file, proxies_obsolete) config.backup_obsolete(proxies_obsolete_file,
# TODO self.proxies might need to be changed, so self probably should be passed. proxy_name, proxy_type,
proxies['proxies'][proxy_name][proxy_type].remove(proxy_url) proxy_url)
Toml.save_file(proxies_file, proxies) try:
config.update_proxies(proxies_file, proxy_name,
proxy_type, proxy_url)
except ValueError as e:
logger.error([str(e), proxy_url])
url_new = None url_new = None
else: else:
logger.warning('No proxy URLs for {}. ' logger.warning('No proxy URLs for {}. '
@ -561,21 +452,19 @@ class Url:
return url_new return url_new
def remove_tracking_parameters(trackers, url): def remove_tracking_parameters(url):
""" """
Remove queries with tracking parameters. Remove queries with tracking parameters.
Parameters Parameters
---------- ----------
trackers : list
A list of queries.
url : str url : str
A URL. URL.
Returns Returns
------- -------
url : str url : str
A processed URL. URL.
""" """
if url.startswith('data:') and ';base64,' in url: if url.startswith('data:') and ';base64,' in url:
return url return url
@ -585,6 +474,7 @@ class Url:
pathname = parted_url.path pathname = parted_url.path
queries = parse_qs(parted_url.query) queries = parse_qs(parted_url.query)
fragment = parted_url.fragment fragment = parted_url.fragment
trackers = config.open_config_file('queries.toml')['trackers']
for tracker in trackers: for tracker in trackers:
if tracker in queries: del queries[tracker] if tracker in queries: del queries[tracker]
queries_new = urlencode(queries, doseq=True) queries_new = urlencode(queries, doseq=True)
@ -823,12 +713,12 @@ class Utilities:
return url_digest return url_digest
def pick_a_feed(dir_config, lang=None): def pick_a_feed(lang=None):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: lang: {}' logger.debug('{}: lang: {}'
.format(function_name, lang)) .format(function_name, lang))
filename_feeds = os.path.join(dir_config, 'feeds.toml') config_dir = config.get_default_config_directory()
with open(filename_feeds, mode="rb") as feeds: with open(config_dir + '/' + 'feeds.toml', mode="rb") as feeds:
urls = tomllib.load(feeds) urls = tomllib.load(feeds)
import random import random
url = random.choice(urls['feeds']) url = random.choice(urls['feeds'])

4
slixfeed/version.py
View file

@ -1,2 +1,2 @@
__version__ = '0.1.107' __version__ = '0.1.85'
__version_info__ = (0, 1, 107) __version_info__ = (0, 1, 85)

2253
slixfeed/xmpp/adhoc.py Normal file
View file

File diff suppressed because it is too large Load diff

488
slixfeed/xmpp/chat.py
View file

@ -24,32 +24,22 @@ TODO
""" """
import asyncio import asyncio
import os
from pathlib import Path
from random import randrange # pending_tasks: Use a list and read the first index (i.e. index 0). from random import randrange # pending_tasks: Use a list and read the first index (i.e. index 0).
import slixfeed.config as config
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.fetch as fetch
from slixfeed.fetch import Http
from slixfeed.log import Logger from slixfeed.log import Logger
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
from slixfeed.syndication import FeedTask from slixfeed.syndication import FeedTask
from slixfeed.utilities import Database, Documentation, Html, MD, Task, Url from slixfeed.utilities import Documentation, Html, MD, Task, Url
from slixfeed.xmpp.commands import XmppCommands from slixfeed.xmpp.commands import XmppCommands
from slixfeed.xmpp.message import XmppMessage from slixfeed.xmpp.message import XmppMessage
from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.status import XmppStatusTask from slixfeed.xmpp.status import XmppStatusTask
from slixfeed.xmpp.upload import XmppUpload from slixfeed.xmpp.upload import XmppUpload
from slixfeed.xmpp.utilities import XmppUtilities from slixfeed.xmpp.utilities import XmppUtilities
from slixmpp import JID
from slixmpp.stanza import Message
import sys import sys
import time import time
from typing import Optional
try:
from slixfeed.xmpp.encryption import XmppOmemo
except Exception as e:
print('Encryption of type OMEMO is not enabled. Reason: ' + str(e))
logger = Logger(__name__) logger = Logger(__name__)
@ -65,8 +55,7 @@ logger = Logger(__name__)
class XmppChat: class XmppChat:
async def process_message( async def process_message(self, message):
self, message: Message, allow_untrusted: bool = False) -> None:
""" """
Process incoming message stanzas. Be aware that this also Process incoming message stanzas. Be aware that this also
includes MUC messages and error messages. It is usually includes MUC messages and error messages. It is usually
@ -80,70 +69,106 @@ class XmppChat:
for stanza objects and the Message stanza to see for stanza objects and the Message stanza to see
how it may be used. how it may be used.
""" """
message_from = message['from'] if message['type'] in ('chat', 'groupchat', 'normal'):
message_type = message['type'] jid_bare = message['from'].bare
if message_type in ('chat', 'groupchat', 'normal'): command = ' '.join(message['body'].split())
jid_bare = message_from.bare
message_body = message['body']
command = ' '.join(message_body.split())
command_time_start = time.time() command_time_start = time.time()
if self.omemo_present and self['xep_0384'].is_encrypted(message): # if (message['type'] == 'groupchat' and
command, omemo_decrypted = await XmppOmemo.decrypt( # message['muc']['nick'] == self.alias):
self, message) # return
else:
omemo_decrypted = None
# FIXME Code repetition. See below. # FIXME Code repetition. See below.
if message_type == 'groupchat': # TODO Check alias by nickname associated with conference
alias = message['muc']['nick'] if message['type'] == 'groupchat':
self_alias = XmppUtilities.get_self_alias(self, jid_bare) if (message['muc']['nick'] == self.alias):
return
if (alias == self_alias or jid_full = str(message['from'])
not XmppUtilities.is_moderator(self, jid_bare, alias) or if not XmppUtilities.is_moderator(self, jid_bare, jid_full):
(not message_body.startswith(self_alias + ' ') and
not message_body.startswith(self_alias + ',') and
not message_body.startswith(self_alias + ':'))):
return return
# Adding one to the length because of if message['type'] == 'groupchat':
# assumption that a comma or a dot is added # nick = message['from'][message['from'].index('/')+1:]
self_alias_length = len(self_alias) + 1 # nick = str(message['from'])
command = command[self_alias_length:].lstrip() # nick = nick[nick.index('/')+1:]
if (message['muc']['nick'] == self.alias or
not message['body'].startswith('!')):
return
# token = await initdb(
# jid_bare,
# sqlite.get_setting_value,
# 'token'
# )
# if token == 'accepted':
# operator = await initdb(
# jid_bare,
# sqlite.get_setting_value,
# 'masters'
# )
# if operator:
# if nick not in operator:
# return
# approved = False
jid_full = str(message['from'])
if not XmppUtilities.is_moderator(self, jid_bare, jid_full):
return
# if role == 'moderator':
# approved = True
# TODO Implement a list of temporary operators
# Once an operator is appointed, the control would last
# untile the participant has been disconnected from MUC
# An operator is a function to appoint non moderators.
# Changing nickname is fine and consist of no problem.
# if not approved:
# operator = await initdb(
# jid_bare,
# sqlite.get_setting_value,
# 'masters'
# )
# if operator:
# if nick in operator:
# approved = True
# if not approved:
# return
if isinstance(command, Message): command = command['body'] # # Begin processing new JID
# # Deprecated in favour of event 'presence_available'
# db_dir = config.get_default_data_directory()
# os.chdir(db_dir)
# if jid + '.db' not in os.listdir():
# await task_jid(jid)
# await compose.message(self, jid_bare, message)
if message['type'] == 'groupchat':
command = command[1:]
command_lowercase = command.lower() command_lowercase = command.lower()
# This is a work-around to empty messages that are caused by function
# self.register_handler(CoroutineCallback( of module client.py.
# The code was taken from the cho bot xample of slixmpp-omemo.
#if not command_lowercase: return
logger.debug([message_from.full, ':', command]) logger.debug([str(message['from']), ':', command])
# Support private message via groupchat # Support private message via groupchat
# See https://codeberg.org/poezio/slixmpp/issues/3506 # See https://codeberg.org/poezio/slixmpp/issues/3506
if message_type == 'chat' and message.get_plugin('muc', check=True): if message['type'] == 'chat' and message.get_plugin('muc', check=True):
# jid_bare = message_from.bare # jid_bare = message['from'].bare
jid_full = message_from.full jid_full = str(message['from'])
if (jid_bare == jid_full[:jid_full.index('/')]): if (jid_bare == jid_full[:jid_full.index('/')]):
# TODO Count and alert of MUC-PM attempts # TODO Count and alert of MUC-PM attempts
return return
response = None response = None
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
match command_lowercase: match command_lowercase:
case 'help': case 'help':
command_list = XmppCommands.print_help()
command_list = XmppCommands.print_help(self.dir_config)
response = ('Available command keys:\n' response = ('Available command keys:\n'
f'```\n{command_list}\n```\n' '```\n{}\n```\n'
'Usage: `help <key>`') 'Usage: `help <key>`'
.format(command_list))
case 'help all': case 'help all':
command_list = Documentation.manual( command_list = Documentation.manual('commands.toml', section='all')
self.dir_config, section='all')
response = ('Complete list of commands:\n' response = ('Complete list of commands:\n'
f'```\n{command_list}\n```' '```\n{}\n```'
.format()) .format(command_list))
case _ if command_lowercase.startswith('help'): case _ if command_lowercase.startswith('help'):
command = command[5:].lower() command = command[5:].lower()
command = command.split(' ') command = command.split(' ')
@ -151,64 +176,72 @@ class XmppChat:
command_root = command[0] command_root = command[0]
command_name = command[1] command_name = command[1]
command_list = Documentation.manual( command_list = Documentation.manual(
self.dir_config, section=command_root, 'commands.toml', section=command_root, command=command_name)
command=command_name)
if command_list: if command_list:
command_list = ''.join(command_list) command_list = ''.join(command_list)
response = (command_list) response = (command_list)
else: else:
response = f'KeyError for {command_root} {command_name}' response = ('KeyError for {} {}'
.format(command_root, command_name))
elif len(command) == 1: elif len(command) == 1:
command = command[0] command = command[0]
command_list = Documentation.manual( command_list = Documentation.manual('commands.toml', command)
self.dir_config, command)
if command_list: if command_list:
command_list = ' '.join(command_list) command_list = ' '.join(command_list)
response = (f'Available command `{command}` keys:\n' response = ('Available command `{}` keys:\n'
f'```\n{command_list}\n```\n' '```\n{}\n```\n'
f'Usage: `help {command} <command>`') 'Usage: `help {} <command>`'
.format(command, command_list, command))
else: else:
response = f'KeyError for {command}' response = 'KeyError for {}'.format(command)
else: else:
response = ('Invalid. Enter command key ' response = ('Invalid. Enter command key '
'or command key & name') 'or command key & name')
case 'info': case 'info':
entries = XmppCommands.print_info_list(self) entries = XmppCommands.print_info_list()
response = ('Available command options:\n' response = ('Available command options:\n'
f'```\n{entries}\n```\n' '```\n{}\n```\n'
'Usage: `info <option>`') 'Usage: `info <option>`'
.format(entries))
case _ if command_lowercase.startswith('info'): case _ if command_lowercase.startswith('info'):
entry = command[5:].lower() entry = command[5:].lower()
response = XmppCommands.print_info_specific(self, entry) response = XmppCommands.print_info_specific(entry)
case _ if command_lowercase in ['greetings', 'hallo', 'hello', case _ if command_lowercase in ['greetings', 'hallo', 'hello',
'hey', 'hi', 'hola', 'holla', 'hey', 'hi', 'hola', 'holla',
'hollo']: 'hollo']:
response = (f'Greeting. My name is {self.alias}.\n' response = ('Greeting! My name is {}.\n'
'I am an Atom/RSS News Bot.\n' 'I am an RSS News Bot.\n'
'Send "help" for further instructions.\n') 'Send "help" for further instructions.\n'
.format(self.alias))
case _ if command_lowercase.startswith('add'): case _ if command_lowercase.startswith('add'):
command = command[4:] command = command[4:]
url = command.split(' ')[0] url = command.split(' ')[0]
title = ' '.join(command.split(' ')[1:]) title = ' '.join(command.split(' ')[1:])
response = await XmppCommands.feed_add( response = XmppCommands.feed_add(
url, db_file, jid_bare, title) url, db_file, jid_bare, title)
case _ if command_lowercase.startswith('allow +'): case _ if command_lowercase.startswith('allow +'):
val = command[7:] val = command[7:]
if val: if val:
await XmppCommands.set_filter_allow( await XmppCommands.set_filter_allow(
db_file, val, True) db_file, val, True)
response = f'Approved keywords\n```\n{val}\n```' response = ('Approved keywords\n'
'```\n{}\n```'
.format(val))
else: else:
response = ('No action has been taken.\n' response = ('No action has been taken.'
'\n'
'Missing keywords.') 'Missing keywords.')
case _ if command_lowercase.startswith('allow -'): case _ if command_lowercase.startswith('allow -'):
val = command[7:] val = command[7:]
if val: if val:
await XmppCommands.set_filter_allow( await XmppCommands.set_filter_allow(
db_file, val, False) db_file, val, False)
response = f'Approved keywords\n```\n{val}\n```' response = ('Approved keywords\n'
'```\n{}\n```'
.format(val))
else: else:
response = ('No action has been taken.\n' response = ('No action has been taken.'
'\n'
'Missing keywords.') 'Missing keywords.')
case _ if command_lowercase.startswith('archive'): case _ if command_lowercase.startswith('archive'):
val = command[8:] val = command[8:]
@ -254,18 +287,24 @@ class XmppChat:
if val: if val:
await XmppCommands.set_filter_allow( await XmppCommands.set_filter_allow(
db_file, val, True) db_file, val, True)
response = f'Rejected keywords\n```\n{val}\n```' response = ('Rejected keywords\n'
'```\n{}\n```'
.format(val))
else: else:
response = ('No action has been taken.\n' response = ('No action has been taken.'
'\n'
'Missing keywords.') 'Missing keywords.')
case _ if command_lowercase.startswith('deny -'): case _ if command_lowercase.startswith('deny -'):
val = command[6:] val = command[6:]
if val: if val:
await XmppCommands.set_filter_allow( await XmppCommands.set_filter_allow(
db_file, val, False) db_file, val, False)
response = f'Rejected keywords\n```\n{val}\n```' response = ('Rejected keywords\n'
'```\n{}\n```'
.format(val))
else: else:
response = ('No action has been taken.\n' response = ('No action has been taken.'
'\n'
'Missing keywords.') 'Missing keywords.')
case _ if command_lowercase.startswith('disable'): case _ if command_lowercase.startswith('disable'):
response = await XmppCommands.feed_disable( response = await XmppCommands.feed_disable(
@ -279,7 +318,8 @@ class XmppChat:
if ext in ('md', 'opml'): # html xbel if ext in ('md', 'opml'): # html xbel
status_type = 'dnd' status_type = 'dnd'
status_message = ('📤️ Procesing request to ' status_message = ('📤️ Procesing request to '
f'export feeds into {ext.upper()}...') 'export feeds into {}...'
.format(ext.upper()))
# pending_tasks_num = len(self.pending_tasks[jid_bare]) # pending_tasks_num = len(self.pending_tasks[jid_bare])
pending_tasks_num = randrange(10000, 99999) pending_tasks_num = randrange(10000, 99999)
self.pending_tasks[jid_bare][pending_tasks_num] = status_message self.pending_tasks[jid_bare][pending_tasks_num] = status_message
@ -287,25 +327,15 @@ class XmppChat:
# self.pending_tasks[jid_bare][self.pending_tasks_counter] = status_message # self.pending_tasks[jid_bare][self.pending_tasks_counter] = status_message
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
status_type=status_type) status_type=status_type)
pathname, response = XmppCommands.export_feeds( filename, response = XmppCommands.export_feeds(
self.dir_data, self.dir_cache, jid_bare, ext) jid_bare, ext)
encrypt_omemo = Config.get_setting_value(self, jid_bare, 'omemo') url = await XmppUpload.start(self, jid_bare, filename)
encrypted = True if encrypt_omemo else False
url = await XmppUpload.start(self, jid_bare, Path(pathname), encrypted=encrypted)
# response = ( # response = (
# f'Feeds exported successfully to {ex}.\n{url}' # 'Feeds exported successfully to {}.\n{}'
# ) # ).format(ex, url)
# XmppMessage.send_oob_reply_message(message, url, response) # XmppMessage.send_oob_reply_message(message, url, response)
if url:
chat_type = await XmppUtilities.get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
if self.omemo_present and encrypted:
url_encrypted, omemo_encrypted = await XmppOmemo.encrypt(
self, message_from, 'chat', url)
XmppMessage.send_omemo_oob(self, message_from, url_encrypted, chat_type)
else:
XmppMessage.send_oob(self, jid_bare, url, chat_type) XmppMessage.send_oob(self, jid_bare, url, chat_type)
else:
response = 'OPML file export has been failed.'
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
XmppStatusTask.restart_task(self, jid_bare) XmppStatusTask.restart_task(self, jid_bare)
@ -314,16 +344,16 @@ class XmppChat:
'Try: md or opml') 'Try: md or opml')
case _ if command_lowercase.startswith('feeds'): case _ if command_lowercase.startswith('feeds'):
query = command[6:] query = command[6:]
result, number = XmppCommands.list_feeds(self.dir_config, db_file, query) result, number = XmppCommands.list_feeds(db_file, query)
if number: if number:
if query: if query:
first_line = f'Subscriptions containing "{query}":\n\n```\n' first_line = 'Subscriptions containing "{}":\n\n```\n'.format(query)
else: else:
first_line = 'Subscriptions:\n\n```\n' first_line = 'Subscriptions:\n\n```\n'
response = (first_line + result + response = (first_line + result +
f'\n```\nTotal of {number} feeds') '\n```\nTotal of {} feeds'.format(number))
case 'goodbye': case 'goodbye':
if message_type == 'groupchat': if message['type'] == 'groupchat':
await XmppCommands.muc_leave(self, jid_bare) await XmppCommands.muc_leave(self, jid_bare)
else: else:
response = 'This command is valid in groupchat only.' response = 'This command is valid in groupchat only.'
@ -348,18 +378,18 @@ class XmppChat:
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
XmppStatusTask.restart_task(self, jid_bare) XmppStatusTask.restart_task(self, jid_bare)
case _ if command_lowercase.startswith('pubsub list'): case _ if command_lowercase.startswith('pubsub list'):
jid_full_pubsub = command[12:] jid = command[12:]
response = f'List of nodes for {jid_full_pubsub}:\n```\n' response = 'List of nodes for {}:\n```\n'.format(jid)
response = await XmppCommands.pubsub_list(self, jid_full_pubsub) response = await XmppCommands.pubsub_list(self, jid)
response += '```' response += '```'
case _ if command_lowercase.startswith('pubsub send'): case _ if command_lowercase.startswith('pubsub send'):
if XmppUtilities.is_operator(self, jid_bare): if XmppUtilities.is_operator(self, jid_bare):
info = command[12:] info = command[12:]
info = info.split(' ') info = info.split(' ')
jid_full_pubsub = info[0] jid = info[0]
# num = int(info[1]) # num = int(info[1])
if jid_full_pubsub: if jid:
response = XmppCommands.pubsub_send(self, info, jid_full_pubsub) response = XmppCommands.pubsub_send(self, info, jid)
else: else:
response = ('This action is restricted. ' response = ('This action is restricted. '
'Type: sending news to PubSub.') 'Type: sending news to PubSub.')
@ -372,7 +402,8 @@ class XmppChat:
command_lowercase.startswith('rss:/')): command_lowercase.startswith('rss:/')):
url = command url = command
status_type = 'dnd' status_type = 'dnd'
status_message = f'📫️ Processing request to fetch data from {url}' status_message = ('📫️ Processing request to fetch data from {}'
.format(url))
# pending_tasks_num = len(self.pending_tasks[jid_bare]) # pending_tasks_num = len(self.pending_tasks[jid_bare])
pending_tasks_num = randrange(10000, 99999) pending_tasks_num = randrange(10000, 99999)
self.pending_tasks[jid_bare][pending_tasks_num] = status_message self.pending_tasks[jid_bare][pending_tasks_num] = status_message
@ -387,9 +418,9 @@ class XmppChat:
XmppStatusTask.restart_task(self, jid_bare) XmppStatusTask.restart_task(self, jid_bare)
# except: # except:
# response = ( # response = (
# f'> {url}\nNews source is in the process ' # '> {}\nNews source is in the process '
# 'of being added to the subscription ' # 'of being added to the subscription '
# 'list.' # 'list.'.format(url)
# ) # )
case _ if command_lowercase.startswith('interval'): case _ if command_lowercase.startswith('interval'):
val = command[9:] val = command[9:]
@ -422,13 +453,6 @@ class XmppChat:
self, jid_bare, db_file) self, jid_bare, db_file)
case _ if command_lowercase.startswith('next'): case _ if command_lowercase.startswith('next'):
num = command[5:] num = command[5:]
if num:
try:
int(num)
except:
# NOTE Show this text as a status message
# response = 'Argument for command "next" must be an integer.'
num = None
await XmppChatAction.send_unread_items(self, jid_bare, num) await XmppChatAction.send_unread_items(self, jid_bare, num)
XmppStatusTask.restart_task(self, jid_bare) XmppStatusTask.restart_task(self, jid_bare)
case _ if command_lowercase.startswith('node delete'): case _ if command_lowercase.startswith('node delete'):
@ -450,12 +474,6 @@ class XmppChat:
case 'old': case 'old':
response = await XmppCommands.set_old_on( response = await XmppCommands.set_old_on(
self, jid_bare, db_file) self, jid_bare, db_file)
case 'omemo off':
response = await XmppCommands.set_omemo_off(
self, jid_bare, db_file)
case 'omemo on':
response = await XmppCommands.set_omemo_on(
self, jid_bare, db_file)
case 'options': case 'options':
response = 'Options:\n```' response = 'Options:\n```'
response += XmppCommands.print_options(self, jid_bare) response += XmppCommands.print_options(self, jid_bare)
@ -478,7 +496,7 @@ class XmppChat:
Task.stop(self, jid_bare, 'status') Task.stop(self, jid_bare, 'status')
status_type = 'dnd' status_type = 'dnd'
status_message = ('📫️ Processing request to fetch data ' status_message = ('📫️ Processing request to fetch data '
f'from {url}') 'from {}'.format(url))
pending_tasks_num = randrange(10000, 99999) pending_tasks_num = randrange(10000, 99999)
self.pending_tasks[jid_bare][pending_tasks_num] = status_message self.pending_tasks[jid_bare][pending_tasks_num] = status_message
response = await XmppCommands.feed_read( response = await XmppCommands.feed_read(
@ -494,7 +512,7 @@ class XmppChat:
if not num: num = 5 if not num: num = 5
count, result = XmppCommands.print_recent(self, db_file, num) count, result = XmppCommands.print_recent(self, db_file, num)
if count: if count:
response = f'Recent {num} fetched titles:\n\n```' response = 'Recent {} fetched titles:\n\n```'.format(num)
response += result + '```\n' response += result + '```\n'
else: else:
response = result response = result
@ -528,53 +546,15 @@ class XmppChat:
case _ if command_lowercase.startswith('search'): case _ if command_lowercase.startswith('search'):
query = command[7:] query = command[7:]
response = XmppCommands.search_items(db_file, query) response = XmppCommands.search_items(db_file, query)
case _ if command_lowercase.startswith('blacklist'):
if XmppUtilities.is_operator(self, jid_bare):
action_jid = command[9:].strip()
action_jid_split = action_jid.split(' ')
if len(action_jid_split) == 2:
action, jid = action_jid_split
if jid and action == 'add':
response = XmppCommands.add_jid_to_selector(self, jid, 'blacklist')
elif jid and action == 'delete':
response = XmppCommands.del_jid_from_selector(self, jid, 'blacklist')
else:
response = f'Unknown action {action}.'
elif len(action_jid_split) > 2:
response = 'USAGE: blacklist <action> <jid>'
else:
response = XmppCommands.print_selector(self.blacklist)
else:
response = ('This action is restricted. '
'Type: managing blacklist.')
case _ if command_lowercase.startswith('whitelist'):
if XmppUtilities.is_operator(self, jid_bare):
action_jid = command[9:].strip()
action_jid_split = action_jid.split(' ')
if len(action_jid_split) == 2:
action, jid = action_jid_split
if jid and action == 'add':
response = XmppCommands.add_jid_to_selector(self, jid, 'whitelist')
elif jid and action == 'delete':
response = XmppCommands.del_jid_from_selector(self, jid, 'whitelist')
else:
response = f'Unknown action {action}.'
elif len(action_jid_split) > 2:
response = 'USAGE: whitelist <action> <jid>'
else:
response = XmppCommands.print_selector(self.whitelist)
else:
response = ('This action is restricted. '
'Type: managing blacklist.')
case 'start': case 'start':
status_type = 'available' status_type = 'available'
status_message = '📫️ Welcome back.' status_message = '📫️ Welcome back!'
XmppPresence.send(self, jid_bare, status_message, XmppPresence.send(self, jid_bare, status_message,
status_type=status_type) status_type=status_type)
await asyncio.sleep(5) await asyncio.sleep(5)
callbacks = (FeedTask, XmppChatTask, XmppStatusTask) tasks = (FeedTask, XmppChatTask, XmppStatusTask)
response = await XmppCommands.scheduler_start( response = await XmppCommands.scheduler_start(
self, db_file, jid_bare, callbacks) self, db_file, jid_bare, tasks)
case 'stats': case 'stats':
response = XmppCommands.print_statistics(db_file) response = XmppCommands.print_statistics(db_file)
case 'stop': case 'stop':
@ -601,52 +581,57 @@ class XmppChat:
command_time_finish = time.time() command_time_finish = time.time()
command_time_total = command_time_finish - command_time_start command_time_total = command_time_finish - command_time_start
command_time_total = round(command_time_total, 3) command_time_total = round(command_time_total, 3)
if response: if response: XmppMessage.send_reply(self, message, response)
encrypt_omemo = Config.get_setting_value(self, jid_bare, 'omemo') if Config.get_setting_value(self.settings, jid_bare, 'finished'):
encrypted = True if encrypt_omemo else False response_finished = 'Finished. Total time: {}s'.format(command_time_total)
if self.omemo_present and encrypted and self['xep_0384'].is_encrypted(message):
response_encrypted, omemo_encrypted = await XmppOmemo.encrypt(
self, message_from, 'chat', response)
if omemo_decrypted and omemo_encrypted:
# message_from = message['from']
# message_type = message['type']
XmppMessage.send_omemo(self, message_from, message_type, response_encrypted)
# XmppMessage.send_omemo_reply(self, message, response_encrypted)
else:
XmppMessage.send_reply(self, message, response)
if Config.get_setting_value(self, jid_bare, 'finished'):
response_finished = f'Finished. Total time: {command_time_total}s'
XmppMessage.send_reply(self, message, response_finished) XmppMessage.send_reply(self, message, response_finished)
# if not response: response = 'EMPTY MESSAGE - ACTION ONLY'
# data_dir = config.get_default_data_directory()
# if not os.path.isdir(data_dir):
# os.mkdir(data_dir)
# if not os.path.isdir(data_dir + '/logs/'):
# os.mkdir(data_dir + '/logs/')
# MD.log_to_markdown(
# dt.current_time(), os.path.join(data_dir, 'logs', jid_bare),
# jid_bare, command)
# MD.log_to_markdown(
# dt.current_time(), os.path.join(data_dir, 'logs', jid_bare),
# jid_bare, response)
# print(
# 'Message : {}\n'
# 'JID : {}\n'
# '{}\n'
# .format(command, jid_bare, response)
# )
class XmppChatAction: class XmppChatAction:
async def send_unread_items(self, jid_bare, num: Optional[int] = None): async def send_unread_items(self, jid_bare, num=None):
""" """
Send news items as messages. Send news items as messages.
Parameters Parameters
---------- ----------
jid_bare : str jid : str
Jabber ID. Jabber ID.
num : str, optional num : str, optional
Number. The default is None. Number. The default is None.
""" """
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug(f'{function_name}: jid: {jid_bare} num: {num}') logger.debug('{}: jid: {} num: {}'.format(function_name, jid_bare, num))
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
encrypt_omemo = Config.get_setting_value(self, jid_bare, 'omemo') show_media = Config.get_setting_value(self.settings, jid_bare, 'media')
encrypted = True if encrypt_omemo else False
jid = JID(jid_bare)
show_media = Config.get_setting_value(self, jid_bare, 'media')
if not num: if not num:
num = Config.get_setting_value(self, jid_bare, 'quantum') num = Config.get_setting_value(self.settings, jid_bare, 'quantum')
else: else:
num = int(num) num = int(num)
results = sqlite.get_unread_entries(db_file, num) results = sqlite.get_unread_entries(db_file, num)
news_digest = '' news_digest = ''
media_url = None media = None
chat_type = await XmppUtilities.get_chat_type(self, jid_bare) chat_type = await XmppUtilities.get_chat_type(self, jid_bare)
for result in results: for result in results:
ix = result[0] ix = result[0]
@ -659,8 +644,7 @@ class XmppChatAction:
if enclosure: enclosure = enclosure[0] if enclosure: enclosure = enclosure[0]
title_f = sqlite.get_feed_title(db_file, feed_id) title_f = sqlite.get_feed_title(db_file, feed_id)
title_f = title_f[0] title_f = title_f[0]
news_digest += await XmppChatAction.list_unread_entries( news_digest += await XmppChatAction.list_unread_entries(self, result, title_f, jid_bare)
self, result, title_f, jid_bare)
# print(db_file) # print(db_file)
# print(result[0]) # print(result[0])
# breakpoint() # breakpoint()
@ -674,95 +658,19 @@ class XmppChatAction:
# elif enclosure: # elif enclosure:
if show_media: if show_media:
if enclosure: if enclosure:
media_url = enclosure media = enclosure
else: else:
media_url = await Html.extract_image_from_html(self.settings_network, url) media = await Html.extract_image_from_html(url)
try:
http_headers = await Http.fetch_headers(self.settings_network, media_url)
if ('Content-Length' in http_headers):
if int(http_headers['Content-Length']) < 100000:
media_url = None
else:
media_url = None
except Exception as e:
print(media_url)
logger.error(e)
media_url = None
if media_url and news_digest: if media and news_digest:
if self.omemo_present and encrypt_omemo:
news_digest_encrypted, omemo_encrypted = await XmppOmemo.encrypt(
self, jid, 'chat', news_digest)
if self.omemo_present and encrypt_omemo and omemo_encrypted:
XmppMessage.send_omemo(
self, jid, chat_type, news_digest_encrypted)
else:
# Send textual message # Send textual message
XmppMessage.send(self, jid_bare, news_digest, chat_type) XmppMessage.send(self, jid_bare, news_digest, chat_type)
news_digest = '' news_digest = ''
# Send media # Send media
if self.omemo_present and encrypt_omemo: XmppMessage.send_oob(self, jid_bare, media, chat_type)
# if not media_url.startswith('data:'): media = None
filename = media_url.split('/').pop().split('?')[0]
if not filename: breakpoint()
pathname = os.path.join(self.dir_cache, filename)
# http_response = await Http.response(media_url)
http_headers = await Http.fetch_headers(self.settings_network, media_url)
if ('Content-Length' in http_headers and
int(http_headers['Content-Length']) < 3000000):
status = await Http.fetch_media(self.settings_network, media_url, pathname)
if status:
filesize = os.path.getsize(pathname)
media_url_new = await XmppUpload.start(
self, jid_bare, Path(pathname), filesize,
encrypted=encrypted)
else:
media_url_new = media_url
else:
media_url_new = media_url
# else:
# import io, base64
# from PIL import Image
# file_content = media_url.split(',').pop()
# file_extension = media_url.split(';')[0].split(':').pop().split('/').pop()
# img = Image.open(io.BytesIO(base64.decodebytes(bytes(file_content, "utf-8"))))
# filename = 'image.' + file_extension
# pathname = os.path.join(cache_dir, filename)
# img.save(pathname)
# filesize = os.path.getsize(pathname)
# media_url_new = await XmppUpload.start(
# self, jid_bare, Path(pathname), filesize, encrypted=encrypted)
media_url_new_encrypted, omemo_encrypted = await XmppOmemo.encrypt(
self, jid, 'chat', media_url_new)
if media_url_new_encrypted and omemo_encrypted:
# NOTE Tested against Gajim.
# FIXME This only works with aesgcm URLs, and it does
# not work with http URLs.
# url = saxutils.escape(url)
# AttributeError: 'Encrypted' object has no attribute 'replace'
XmppMessage.send_omemo_oob(
self, jid, media_url_new_encrypted, chat_type)
else:
# NOTE Tested against Gajim.
# FIXME Jandle data: URIs.
if not media_url.startswith('data:'):
http_headers = await Http.fetch_headers(self.settings_network, media_url)
if ('Content-Length' in http_headers and
int(http_headers['Content-Length']) > 100000):
print(http_headers['Content-Length'])
XmppMessage.send_oob(self, jid_bare, media_url, chat_type)
else:
XmppMessage.send_oob(self, jid_bare, media_url, chat_type)
media_url = None
if news_digest: if news_digest:
if self.omemo_present and encrypt_omemo:
news_digest_encrypted, omemo_encrypted = await XmppOmemo.encrypt(
self, jid, 'chat', news_digest)
if self.omemo_present and encrypt_omemo and omemo_encrypted:
XmppMessage.send_omemo(
self, jid, chat_type, news_digest_encrypted)
else:
XmppMessage.send(self, jid_bare, news_digest, chat_type) XmppMessage.send(self, jid_bare, news_digest, chat_type)
# TODO Add while loop to assure delivery. # TODO Add while loop to assure delivery.
# print(await current_time(), ">>> ACT send_message",jid) # print(await current_time(), ">>> ACT send_message",jid)
@ -812,7 +720,8 @@ class XmppChatAction:
async def list_unread_entries(self, result, feed_title, jid): async def list_unread_entries(self, result, feed_title, jid):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug(f'{function_name}: feed_title: {feed_title} jid: {jid}') logger.debug('{}: feed_title: {} jid: {}'
.format(function_name, feed_title, jid))
# TODO Add filtering # TODO Add filtering
# TODO Do this when entry is added to list and mark it as read # TODO Do this when entry is added to list and mark it as read
# DONE! # DONE!
@ -828,7 +737,7 @@ class XmppChatAction:
# print("accepted:", result[1]) # print("accepted:", result[1])
# results.extend([result]) # results.extend([result])
# news_list = f'You have got {num} news items:\n' # news_list = "You've got {} news items:\n".format(num)
# NOTE Why doesn't this work without list? # NOTE Why doesn't this work without list?
# i.e. for result in results # i.e. for result in results
# for result in results.fetchall(): # for result in results.fetchall():
@ -847,7 +756,7 @@ class XmppChatAction:
summary = summary.replace(' ', ' ') summary = summary.replace(' ', ' ')
# summary = summary.replace(' ', ' ') # summary = summary.replace(' ', ' ')
summary = ' '.join(summary.split()) summary = ' '.join(summary.split())
length = Config.get_setting_value(self, jid, 'length') length = Config.get_setting_value(self.settings, jid, 'length')
length = int(length) length = int(length)
summary = summary[:length] + " […]" summary = summary[:length] + " […]"
# summary = summary.strip().split('\n') # summary = summary.strip().split('\n')
@ -856,11 +765,12 @@ class XmppChatAction:
else: else:
summary = '*** No summary ***' summary = '*** No summary ***'
link = result[2] link = result[2]
link = Url.remove_tracking_parameters(self.trackers, link) link = Url.remove_tracking_parameters(link)
link = await Url.replace_hostname(self.dir_config, self.proxies, self.settings_network, link, "link") or link link = await Url.replace_hostname(link, "link") or link
feed_id = result[4] feed_id = result[4]
# news_item = (f'\n{str(title)}\n{str(link)}\n{str(feed_title)} [{str(ix)}]\n') # news_item = ("\n{}\n{}\n{} [{}]\n").format(str(title), str(link),
formatting = Config.get_setting_value(self, jid, 'formatting') # str(feed_title), str(ix))
formatting = Config.get_setting_value(self.settings, jid, 'formatting')
news_item = formatting.format(feed_title=feed_title, news_item = formatting.format(feed_title=feed_title,
title=title, title=title,
summary=summary, summary=summary,
@ -875,12 +785,11 @@ class XmppChatTask:
async def task_message(self, jid_bare): async def task_message(self, jid_bare):
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings: if jid_bare not in self.settings:
Config.add_settings_jid(self, jid_bare, db_file) Config.add_settings_jid(self.settings, jid_bare, db_file)
while True: while True:
update_interval = Config.get_setting_value( update_interval = Config.get_setting_value(self.settings, jid_bare, 'interval')
self, jid_bare, 'interval')
update_interval = 60 * int(update_interval) update_interval = 60 * int(update_interval)
last_update_time = sqlite.get_last_update_time(db_file) last_update_time = sqlite.get_last_update_time(db_file)
if last_update_time: if last_update_time:
@ -911,12 +820,13 @@ class XmppChatTask:
return return
if jid_bare not in self.task_manager: if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {} self.task_manager[jid_bare] = {}
logger.info(f'Creating new task manager for JID {jid_bare}') logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info(f'Stopping task "interval" for JID {jid_bare}') logger.info('Stopping task "interval" for JID {}'.format(jid_bare))
try: try:
self.task_manager[jid_bare]['interval'].cancel() self.task_manager[jid_bare]['interval'].cancel()
except: except:
logger.info('No task "interval" for JID {jid_bare} (XmppChatTask.task_message)') logger.info('No task "interval" for JID {} (XmppChatTask.task_message)'
logger.info('Starting tasks "interval" for JID {jid_bare}') .format(jid_bare))
logger.info('Starting tasks "interval" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['interval'] = asyncio.create_task( self.task_manager[jid_bare]['interval'] = asyncio.create_task(
XmppChatTask.task_message(self, jid_bare)) XmppChatTask.task_message(self, jid_bare))

675
slixfeed/xmpp/client.py
View file

File diff suppressed because it is too large Load diff

454
slixfeed/xmpp/commands.py
View file

@ -2,14 +2,14 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from feedparser import parse from feedparser import parse
import os
from random import randrange from random import randrange
import slixfeed.config as config
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.fetch as fetch import slixfeed.fetch as fetch
from slixfeed.log import Logger from slixfeed.log import Logger
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
from slixfeed.syndication import Feed, FeedDiscovery, Opml from slixfeed.syndication import Feed, FeedDiscovery, Opml
from slixfeed.utilities import Database, DateAndTime, Documentation, String, Url, Utilities from slixfeed.utilities import DateAndTime, Documentation, String, Url, Utilities
from slixfeed.version import __version__ from slixfeed.version import __version__
from slixfeed.xmpp.bookmark import XmppBookmark from slixfeed.xmpp.bookmark import XmppBookmark
from slixfeed.xmpp.muc import XmppMuc from slixfeed.xmpp.muc import XmppMuc
@ -18,7 +18,6 @@ from slixfeed.xmpp.presence import XmppPresence
from slixfeed.xmpp.status import XmppStatusTask from slixfeed.xmpp.status import XmppStatusTask
from slixfeed.xmpp.utilities import XmppUtilities from slixfeed.xmpp.utilities import XmppUtilities
import sys import sys
import tomli_w
try: try:
import tomllib import tomllib
@ -38,60 +37,61 @@ logger = Logger(__name__)
class XmppCommands: class XmppCommands:
def print_help(dir_config): def print_help():
result = Documentation.manual(dir_config) result = Documentation.manual('commands.toml')
message = '\n'.join(result) message = '\n'.join(result)
return message return message
def print_help_list(dir_config): def print_help_list():
command_list = Documentation.manual(dir_config, section='all') command_list = Documentation.manual('commands.toml', section='all')
message = ('Complete list of commands:\n' message = ('Complete list of commands:\n'
f'```\n{command_list}\n```') '```\n{}\n```'.format(command_list))
return message return message
def print_help_specific(dir_config, command_root, command_name): def print_help_specific(command_root, command_name):
command_list = Documentation.manual(dir_config, command_list = Documentation.manual('commands.toml',
section=command_root, section=command_root,
command=command_name) command=command_name)
if command_list: if command_list:
command_list = ''.join(command_list) command_list = ''.join(command_list)
message = (command_list) message = (command_list)
else: else:
message = f'KeyError for {command_root} {command_name}' message = 'KeyError for {} {}'.format(command_root, command_name)
return message return message
def print_help_key(dir_config, command): def print_help_key(command):
command_list = Documentation.manual(dir_config, command) command_list = Documentation.manual('commands.toml', command)
if command_list: if command_list:
command_list = ' '.join(command_list) command_list = ' '.join(command_list)
message = (f'Available command `{command}` keys:\n' message = ('Available command `{}` keys:\n'
f'```\n{command_list}\n```\n' '```\n{}\n```\n'
f'Usage: `help {command} <command>`') 'Usage: `help {} <command>`'
.format(command, command_list, command))
else: else:
message = f'KeyError for {command}' message = 'KeyError for {}'.format(command)
return message return message
def print_info_list(self): def print_info_list():
file_info = os.path.join(self.dir_config, 'information.toml') config_dir = config.get_default_config_directory()
with open(file_info, mode="rb") as information: with open(config_dir + '/' + 'information.toml', mode="rb") as information:
result = tomllib.load(information) result = tomllib.load(information)
message = '\n'.join(result) message = '\n'.join(result)
return message return message
def print_info_specific(self, entry): def print_info_specific(entry):
file_info = os.path.join(self.dir_config, 'information.toml') config_dir = config.get_default_config_directory()
with open(file_info, mode="rb") as information: with open(config_dir + '/' + 'information.toml', mode="rb") as information:
entries = tomllib.load(information) entries = tomllib.load(information)
if entry in entries: if entry in entries:
# command_list = '\n'.join(command_list) # command_list = '\n'.join(command_list)
message = (entries[entry]['info']) message = (entries[entry]['info'])
else: else:
message = f'KeyError for {entry}' message = 'KeyError for {}'.format(entry)
return message return message
@ -132,7 +132,7 @@ class XmppCommands:
identifier) identifier)
feed_id = sqlite.get_feed_id(db_file, url) feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0] feed_id = feed_id[0]
result = await fetch.http(self.settings_network, url) result = await fetch.http(url)
if not result['error']: if not result['error']:
document = result['content'] document = result['content']
feed = parse(document) feed = parse(document)
@ -174,8 +174,10 @@ class XmppCommands:
# the look into function "check_updates" of module "task". # the look into function "check_updates" of module "task".
# await action.scan(self, jid_bare, db_file, url) # await action.scan(self, jid_bare, db_file, url)
# if jid_bare not in self.settings: # if jid_bare not in self.settings:
# Config.add_settings_jid(self, jid_bare, db_file) # Config.add_settings_jid(self.settings, jid_bare,
# old = Config.get_setting_value(self, jid_bare, 'old') # db_file)
# old = Config.get_setting_value(self.settings, jid_bare,
# 'old')
# if old: # if old:
# # task.clean_tasks_xmpp_chat(self, jid_bare, ['status']) # # task.clean_tasks_xmpp_chat(self, jid_bare, ['status'])
# # await send_status(jid) # # await send_status(jid)
@ -185,16 +187,18 @@ class XmppCommands:
# feed_id = feed_id[0] # feed_id = feed_id[0]
# await sqlite.mark_feed_as_read(db_file, feed_id) # await sqlite.mark_feed_as_read(db_file, feed_id)
message = (f'> {url}\n' message = ('> {}\n'
'News source has been ' 'News source has been '
'added to subscription list.') 'added to subscription list.'
.format(url))
else: else:
ix = exist[0] ix = exist[0]
name = exist[1] name = exist[1]
message = (f'> {url}\n' message = ('> {}\n'
f'News source "{name}" is already ' 'News source "{}" is already '
'listed in the subscription list at ' 'listed in the subscription list at '
f'index {ix}') 'index {}'
.format(url, name, ix))
else: else:
message = ('No action has been taken. Missing URL.') message = ('No action has been taken. Missing URL.')
return message return message
@ -220,9 +224,9 @@ class XmppCommands:
keywords = sqlite.get_filter_value(db_file, 'allow') keywords = sqlite.get_filter_value(db_file, 'allow')
if keywords: keywords = str(keywords[0]) if keywords: keywords = str(keywords[0])
if axis: if axis:
val = config.add_to_list(val, keywords) val = await config.add_to_list(val, keywords)
else: else:
val = config.remove_from_list(val, keywords) val = await config.remove_from_list(val, keywords)
if sqlite.is_filter_key(db_file, 'allow'): if sqlite.is_filter_key(db_file, 'allow'):
await sqlite.update_filter_value(db_file, ['allow', val]) await sqlite.update_filter_value(db_file, ['allow', val])
else: else:
@ -230,7 +234,8 @@ class XmppCommands:
def get_archive(self, jid_bare): def get_archive(self, jid_bare):
result = Config.get_setting_value(self, jid_bare, 'archive') result = Config.get_setting_value(
self.settings, jid_bare, 'archive')
message = str(result) message = str(result)
return message return message
@ -241,11 +246,12 @@ class XmppCommands:
if val_new > 500: if val_new > 500:
message = 'Value may not be greater than 500.' message = 'Value may not be greater than 500.'
else: else:
val_old = Config.get_setting_value(self, jid_bare, 'archive') val_old = Config.get_setting_value(
self.settings, jid_bare, 'archive')
await Config.set_setting_value( await Config.set_setting_value(
self, jid_bare, db_file, 'archive', val_new) self.settings, jid_bare, db_file, 'archive', val_new)
message = ('Maximum archived items has been set to {val_new} ' message = ('Maximum archived items has been set to {} (was: {}).'
'(was: {val_old}).') .format(val_new, val_old))
except: except:
message = 'No action has been taken. Enter a numeric value only.' message = 'No action has been taken. Enter a numeric value only.'
return message return message
@ -253,25 +259,28 @@ class XmppCommands:
async def bookmark_add(self, muc_jid): async def bookmark_add(self, muc_jid):
await XmppBookmark.add(self, jid=muc_jid) await XmppBookmark.add(self, jid=muc_jid)
message = f'Groupchat {muc_jid} has been added to bookmarks.' message = ('Groupchat {} has been added to bookmarks.'
.format(muc_jid))
return message return message
async def bookmark_del(self, muc_jid): async def bookmark_del(self, muc_jid):
await XmppBookmark.remove(self, muc_jid) await XmppBookmark.remove(self, muc_jid)
message = f'Groupchat {muc_jid} has been removed from bookmarks.' message = ('Groupchat {} has been removed from bookmarks.'
.format(muc_jid))
return message return message
async def restore_default(self, jid_bare, key=None): async def restore_default(self, jid_bare, key=None):
if key: if key:
self.settings[jid_bare][key] = None self.settings[jid_bare][key] = None
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
await sqlite.delete_setting(db_file, key) await sqlite.delete_setting(db_file, key)
message = f'Setting {key} has been restored to default value.' message = ('Setting {} has been restored to default value.'
.format(key))
else: else:
del self.settings[jid_bare] del self.settings[jid_bare]
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
await sqlite.delete_settings(db_file) await sqlite.delete_settings(db_file)
message = 'Default settings have been restored.' message = 'Default settings have been restored.'
return message return message
@ -279,7 +288,7 @@ class XmppCommands:
async def clear_filter(db_file, key): async def clear_filter(db_file, key):
await sqlite.delete_filter(db_file, key) await sqlite.delete_filter(db_file, key)
message = f'Filter {key} has been purged.' message = 'Filter {} has been purged.'.format(key)
return message return message
@ -287,11 +296,11 @@ class XmppCommands:
conferences = await XmppBookmark.get_bookmarks(self) conferences = await XmppBookmark.get_bookmarks(self)
message = '\nList of groupchats:\n\n```\n' message = '\nList of groupchats:\n\n```\n'
for conference in conferences: for conference in conferences:
conference_name = conference['name'] message += ('Name: {}\n'
conference_jid = conference['jid'] 'Room: {}\n'
message += (f'Name: {conference_name}\n' '\n'
f'Room: {conference_jid}\n\n') .format(conference['name'], conference['jid']))
message += f'```\nTotal of {len(conferences)} groupchats.\n' message += ('```\nTotal of {} groupchats.\n'.format(len(conferences)))
return message return message
@ -315,19 +324,19 @@ class XmppCommands:
keywords = sqlite.get_filter_value(db_file, 'deny') keywords = sqlite.get_filter_value(db_file, 'deny')
if keywords: keywords = str(keywords[0]) if keywords: keywords = str(keywords[0])
if axis: if axis:
val = config.add_to_list(val, keywords) val = await config.add_to_list(val, keywords)
else: else:
val = config.remove_from_list(val, keywords) val = await config.remove_from_list(val, keywords)
if sqlite.is_filter_key(db_file, 'deny'): if sqlite.is_filter_key(db_file, 'deny'):
await sqlite.update_filter_value(db_file, ['deny', val]) await sqlite.update_filter_value(db_file, ['deny', val])
else: else:
await sqlite.set_filter_value(db_file, ['deny', val]) await sqlite.set_filter_value(db_file, ['deny', val])
def export_feeds(dir_data, dir_cache, jid_bare, ext): def export_feeds(jid_bare, ext):
pathname = Feed.export_feeds(dir_data, dir_cache, jid_bare, ext) filename = Feed.export_feeds(jid_bare, ext)
message = f'Feeds successfuly exported to {ext}.' message = 'Feeds successfuly exported to {}.'.format(ext)
return pathname, message return filename, message
def fetch_gemini(): def fetch_gemini():
@ -337,10 +346,11 @@ class XmppCommands:
async def import_opml(self, db_file, jid_bare, command): async def import_opml(self, db_file, jid_bare, command):
url = command url = command
result = await fetch.http(self.settings_network, url) result = await fetch.http(url)
count = await Opml.import_from_file(db_file, result) count = await Opml.import_from_file(db_file, result)
if count: if count:
message = f'Successfully imported {count} feeds.' message = ('Successfully imported {} feeds.'
.format(count))
else: else:
message = 'OPML file was not imported.' message = 'OPML file was not imported.'
return message return message
@ -352,7 +362,7 @@ class XmppCommands:
for item in iq['disco_items']: for item in iq['disco_items']:
item_id = item['node'] item_id = item['node']
item_name = item['name'] item_name = item['name']
message += f'Name: {item_name}\nNode: {item_id}\n\n' message += 'Name: {}\nNode: {}\n\n'.format(item_name, item_id)
return message return message
@ -381,7 +391,7 @@ class XmppCommands:
jid = info[0] jid = info[0]
if '/' not in jid: if '/' not in jid:
url = info[1] url = info[1]
db_file = Database.instantiate(self.dir_data, jid) db_file = config.get_pathname_to_database(jid)
if len(info) > 2: if len(info) > 2:
identifier = info[2] identifier = info[2]
else: else:
@ -395,7 +405,8 @@ class XmppCommands:
break break
# task.clean_tasks_xmpp_chat(self, jid_bare, ['status']) # task.clean_tasks_xmpp_chat(self, jid_bare, ['status'])
status_type = 'dnd' status_type = 'dnd'
status_message = '📫️ Processing request to fetch data from {url}' status_message = ('📫️ Processing request to fetch data from {}'
.format(url))
# pending_tasks_num = len(self.pending_tasks[jid_bare]) # pending_tasks_num = len(self.pending_tasks[jid_bare])
pending_tasks_num = randrange(10000, 99999) pending_tasks_num = randrange(10000, 99999)
self.pending_tasks[jid_bare][pending_tasks_num] = status_message self.pending_tasks[jid_bare][pending_tasks_num] = status_message
@ -407,41 +418,40 @@ class XmppCommands:
url.startswith('itpc:/') or url.startswith('itpc:/') or
url.startswith('rss:/')): url.startswith('rss:/')):
url = Url.feed_to_http(url) url = Url.feed_to_http(url)
url = (await Url.replace_hostname(self.dir_config, self.proxies, self.settings_network, url, 'feed')) or url url = (await Url.replace_hostname(url, 'feed')) or url
result = await Feed.add_feed(self, jid_bare, db_file, url, result = await Feed.add_feed(self, jid_bare, db_file, url,
identifier) identifier)
if isinstance(result, list): if isinstance(result, list):
results = result results = result
message = f'Syndication feeds found for {url}\n\n```\n' message = "Syndication feeds found for {}\n\n```\n".format(url)
for result in results: for result in results:
result_name = result['name'] message += ("Title : {}\n"
result_link = result['link'] "Link : {}\n"
message += ('Title : {result_name}\n' "\n"
'Link : {result_link}\n\n') .format(result['name'], result['link']))
message += f'```\nTotal of {len(results)} feeds.' message += '```\nTotal of {} feeds.'.format(len(results))
elif result['exist']: elif result['exist']:
result_link = result['link'] message = ('> {}\nNews source "{}" is already '
result_name = result['name']
result_index = result['index']
message = (f'> {result_link}\nNews source "{result_name}" is already '
'listed in the subscription list at ' 'listed in the subscription list at '
f'index {result_index}') 'index {}'
.format(result['link'],
result['name'],
result['index']))
elif result['identifier']: elif result['identifier']:
result_link = result['link'] message = ('> {}\nIdentifier "{}" is already '
result_identifier = result['identifier'] 'allocated to index {}'
result_index = result['index'] .format(result['link'],
message = (f'> {result_link}\nIdentifier "{result_identifier}" is already ' result['identifier'],
f'allocated to index {result_index}') result['index']))
elif result['error']: elif result['error']:
result_message = result['message'] message = ('> {}\nNo subscriptions were found. '
result_code = result['code'] 'Reason: {} (status code: {})'
message = (f'> {url}\nNo subscriptions were found. ' .format(url, result['message'],
f'Reason: {result_message} (status code: {result_code})') result['code']))
else: else:
result_link = result['link'] message = ('> {}\nNews source "{}" has been '
result_name = result['name'] 'added to subscription list.'
message = (f'> {result_link}\nNews source "{result_name}" has been ' .format(result['link'], result['name']))
'added to subscription list.')
# task.clean_tasks_xmpp_chat(self, jid_bare, ['status']) # task.clean_tasks_xmpp_chat(self, jid_bare, ['status'])
del self.pending_tasks[jid_bare][pending_tasks_num] del self.pending_tasks[jid_bare][pending_tasks_num]
# del self.pending_tasks[jid_bare][self.pending_tasks_counter] # del self.pending_tasks[jid_bare][self.pending_tasks_counter]
@ -449,9 +459,9 @@ class XmppCommands:
XmppStatusTask.restart_task(self, jid_bare) XmppStatusTask.restart_task(self, jid_bare)
# except: # except:
# response = ( # response = (
# f'> {url}\nNews source is in the process ' # '> {}\nNews source is in the process '
# 'of being added to the subscription ' # 'of being added to the subscription '
# 'list.' # 'list.'.format(url)
# ) # )
else: else:
message = ('No action has been taken.' message = ('No action has been taken.'
@ -470,8 +480,7 @@ class XmppCommands:
async def fetch_http(self, url, db_file, jid_bare): async def fetch_http(self, url, db_file, jid_bare):
if url.startswith('feed:/') or url.startswith('rss:/'): if url.startswith('feed:/') or url.startswith('rss:/'):
url = Url.feed_to_http(url) url = Url.feed_to_http(url)
url = (await Url.replace_hostname( url = (await Url.replace_hostname(url, 'feed')) or url
self.dir_config, self.proxies, self.settings_network, url, 'feed') or url)
counter = 0 counter = 0
while True: while True:
identifier = String.generate_identifier(url, counter) identifier = String.generate_identifier(url, counter)
@ -483,13 +492,15 @@ class XmppCommands:
result = await Feed.add_feed(self, jid_bare, db_file, url, identifier) result = await Feed.add_feed(self, jid_bare, db_file, url, identifier)
if isinstance(result, list): if isinstance(result, list):
results = result results = result
message = f"Syndication feeds found for {url}\n\n```\n" message = ("Syndication feeds found for {}\n\n```\n"
.format(url))
for result in results: for result in results:
message += ("Title : {}\n" message += ("Title : {}\n"
"Link : {}\n" "Link : {}\n"
"\n" "\n"
.format(result['name'], result['link'])) .format(result['name'], result['link']))
message += f"```\nTotal of {len(results)} feeds." message += ('```\nTotal of {} feeds.'
.format(len(results)))
elif result['exist']: elif result['exist']:
message = ('> {}\nNews source "{}" is already ' message = ('> {}\nNews source "{}" is already '
'listed in the subscription list at ' 'listed in the subscription list at '
@ -507,14 +518,14 @@ class XmppCommands:
.format(result['link'], result['name'])) .format(result['link'], result['name']))
# except: # except:
# response = ( # response = (
# f'> {url}\nNews source is in the process ' # '> {}\nNews source is in the process '
# 'of being added to the subscription ' # 'of being added to the subscription '
# 'list.' # 'list.'.format(url)
# ) # )
return message return message
def list_feeds(dir_config, db_file, query=None): def list_feeds(db_file, query=None):
if query: if query:
feeds = sqlite.search_feeds(db_file, query) feeds = sqlite.search_feeds(db_file, query)
else: else:
@ -523,12 +534,14 @@ class XmppCommands:
message = '' message = ''
if number: if number:
for id, title, url in feeds: for id, title, url in feeds:
message += (f'\nName : {str(title)} [{str(id)}]' message += ('\nName : {} [{}]'
f'\nURL : {str(url)}\n') '\nURL : {}'
'\n'
.format(str(title), str(id), str(url)))
elif query: elif query:
message = f"No feeds were found for: {query}" message = "No feeds were found for: {}".format(query)
else: else:
url = Utilities.pick_a_feed(dir_config) url = Utilities.pick_a_feed()
message = ('List of subscriptions is empty. ' message = ('List of subscriptions is empty. '
'To add a feed, send a URL.\n' 'To add a feed, send a URL.\n'
'Featured news: *{}*\n{}' 'Featured news: *{}*\n{}'
@ -537,7 +550,8 @@ class XmppCommands:
def get_interval(self, jid_bare): def get_interval(self, jid_bare):
result = Config.get_setting_value(self, jid_bare, 'interval') result = Config.get_setting_value(
self.settings, jid_bare, 'interval')
message = str(result) message = str(result)
return message return message
@ -545,11 +559,12 @@ class XmppCommands:
async def set_interval(self, db_file, jid_bare, val): async def set_interval(self, db_file, jid_bare, val):
try: try:
val_new = int(val) val_new = int(val)
val_old = Config.get_setting_value(self, jid_bare, 'interval') val_old = Config.get_setting_value(
self.settings, jid_bare, 'interval')
await Config.set_setting_value( await Config.set_setting_value(
self, jid_bare, db_file, 'interval', val_new) self.settings, jid_bare, db_file, 'interval', val_new)
message = (f'Updates will be sent every {val_new} minutes ' message = ('Updates will be sent every {} minutes '
f'(was: {val_old}).') '(was: {}).'.format(val_new, val_old))
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
message = ('No action has been taken. Enter a numeric value only.') message = ('No action has been taken. Enter a numeric value only.')
@ -569,19 +584,20 @@ class XmppCommands:
result = await XmppMuc.join(self, muc_jid) result = await XmppMuc.join(self, muc_jid)
# await XmppBookmark.add(self, jid=muc_jid) # await XmppBookmark.add(self, jid=muc_jid)
if result == 'ban': if result == 'ban':
message = f'{self.alias} is banned from {muc_jid}' message = '{} is banned from {}'.format(self.alias, muc_jid)
else: else:
await XmppBookmark.add(self, muc_jid) await XmppBookmark.add(self, muc_jid)
message = f'Joined groupchat {muc_jid}' message = 'Joined groupchat {}'.format(muc_jid)
else: else:
message = f'> {muc_jid}\nGroupchat JID appears to be invalid.' message = '> {}\nGroupchat JID appears to be invalid.'.format(muc_jid)
else: else:
message = '> {}\nGroupchat JID is missing.' message = '> {}\nGroupchat JID is missing.'
return message return message
def get_length(self, jid_bare): def get_length(self, jid_bare):
result = Config.get_setting_value(self, jid_bare, 'length') result = Config.get_setting_value(
self.settings, jid_bare, 'length')
result = str(result) result = str(result)
return result return result
@ -589,16 +605,18 @@ class XmppCommands:
async def set_length(self, db_file, jid_bare, val): async def set_length(self, db_file, jid_bare, val):
try: try:
val_new = int(val) val_new = int(val)
val_old = Config.get_setting_value(self, jid_bare, 'length') val_old = Config.get_setting_value(
self.settings, jid_bare, 'length')
await Config.set_setting_value( await Config.set_setting_value(
self, jid_bare, db_file, 'length', val_new) self.settings, jid_bare, db_file, 'length', val_new)
if not val_new: # i.e. val_new == 0 if not val_new: # i.e. val_new == 0
# TODO Add action to disable limit # TODO Add action to disable limit
message = ('Summary length limit is disabled ' message = ('Summary length limit is disabled '
f'(was: {val_old}).') '(was: {}).'.format(val_old))
else: else:
message = ('Summary maximum length is set to ' message = ('Summary maximum length is set to '
f'{val_new} characters (was: {val_old}).') '{} characters (was: {}).'
.format(val_new, val_old))
except: except:
message = ('No action has been taken.' message = ('No action has been taken.'
'\n' '\n'
@ -607,41 +625,29 @@ class XmppCommands:
async def set_media_off(self, jid_bare, db_file): async def set_media_off(self, jid_bare, db_file):
await Config.set_setting_value(self, jid_bare, db_file, 'media', 0) await Config.set_setting_value(self.settings, jid_bare, db_file, 'media', 0)
message = 'Media is disabled.' message = 'Media is disabled.'
return message return message
async def set_media_on(self, jid_bare, db_file): async def set_media_on(self, jid_bare, db_file):
await Config.set_setting_value(self, jid_bare, db_file, 'media', 1) await Config.set_setting_value(self.settings, jid_bare, db_file, 'media', 1)
message = 'Media is enabled.' message = 'Media is enabled.'
return message return message
async def set_old_off(self, jid_bare, db_file): async def set_old_off(self, jid_bare, db_file):
await Config.set_setting_value(self, jid_bare, db_file, 'old', 0) await Config.set_setting_value(self.settings, jid_bare, db_file, 'old', 0)
message = 'Only new items of newly added feeds be delivered.' message = 'Only new items of newly added feeds be delivered.'
return message return message
async def set_old_on(self, jid_bare, db_file): async def set_old_on(self, jid_bare, db_file):
await Config.set_setting_value(self, jid_bare, db_file, 'old', 1) await Config.set_setting_value(self.settings, jid_bare, db_file, 'old', 1)
message = 'All items of newly added feeds be delivered.' message = 'All items of newly added feeds be delivered.'
return message return message
async def set_omemo_off(self, jid_bare, db_file):
await Config.set_setting_value(self, jid_bare, db_file, 'omemo', 0)
message = 'OMEMO is disabled.'
return message
async def set_omemo_on(self, jid_bare, db_file):
await Config.set_setting_value(self, jid_bare, db_file, 'omemo', 1)
message = 'OMEMO is enabled.'
return message
def node_delete(self, info): def node_delete(self, info):
info = info.split(' ') info = info.split(' ')
if len(info) > 2: if len(info) > 2:
@ -649,7 +655,7 @@ class XmppCommands:
nid = info[1] nid = info[1]
if jid: if jid:
XmppPubsub.delete_node(self, jid, nid) XmppPubsub.delete_node(self, jid, nid)
message = f'Deleted node: {nid}' message = 'Deleted node: {}'.format(nid)
else: else:
message = 'PubSub JID is missing. Enter PubSub JID.' message = 'PubSub JID is missing. Enter PubSub JID.'
else: else:
@ -667,7 +673,7 @@ class XmppCommands:
nid = info[1] nid = info[1]
if jid: if jid:
XmppPubsub.purge_node(self, jid, nid) XmppPubsub.purge_node(self, jid, nid)
message = f'Purged node: {nid}' message = 'Purged node: {}'.format(nid)
else: else:
message = 'PubSub JID is missing. Enter PubSub JID.' message = 'PubSub JID is missing. Enter PubSub JID.'
else: else:
@ -681,8 +687,8 @@ class XmppCommands:
def print_options(self, jid_bare): def print_options(self, jid_bare):
message = '' message = ''
for key in self.settings[jid_bare]: for key in self.settings[jid_bare]:
val = Config.get_setting_value(self, jid_bare, key) val = Config.get_setting_value(self.settings, jid_bare, key)
# val = Config.get_setting_value(self, jid_bare, key) # val = Config.get_setting_value(self.settings, jid_bare, key)
steps = 11 - len(key) steps = 11 - len(key)
pulse = '' pulse = ''
for step in range(steps): for step in range(steps):
@ -692,7 +698,8 @@ class XmppCommands:
def get_quantum(self, jid_bare): def get_quantum(self, jid_bare):
result = Config.get_setting_value(self, jid_bare, 'quantum') result = Config.get_setting_value(
self.settings, jid_bare, 'quantum')
message = str(result) message = str(result)
return message return message
@ -700,14 +707,16 @@ class XmppCommands:
async def set_quantum(self, db_file, jid_bare, val): async def set_quantum(self, db_file, jid_bare, val):
try: try:
val_new = int(val) val_new = int(val)
val_old = Config.get_setting_value(self, jid_bare, 'quantum') val_old = Config.get_setting_value(
self.settings, jid_bare, 'quantum')
# response = ( # response = (
# f'Every update will contain {response} news items.' # 'Every update will contain {} news items.'
# ) # ).format(response)
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
await Config.set_setting_value( await Config.set_setting_value(self.settings, jid_bare,
self, jid_bare, db_file, 'quantum', val_new) db_file, 'quantum', val_new)
message = f'Next update will contain {val_new} news items (was: {val_old}).' message = ('Next update will contain {} news items (was: {}).'
.format(val_new, val_old))
except: except:
message = 'No action has been taken. Enter a numeric value only.' message = 'No action has been taken. Enter a numeric value only.'
return message return message
@ -724,12 +733,12 @@ class XmppCommands:
async def feed_read(self, jid_bare, data, url): async def feed_read(self, jid_bare, data, url):
if url.startswith('feed:/') or url.startswith('rss:/'): if url.startswith('feed:/') or url.startswith('rss:/'):
url = Url.feed_to_http(url) url = Url.feed_to_http(url)
url = (await Url.replace_hostname(self.dir_config, self.proxies, self.settings_network, url, 'feed')) or url url = (await Url.replace_hostname(url, 'feed')) or url
match len(data): match len(data):
case 1: case 1:
if url.startswith('http'): if url.startswith('http'):
while True: while True:
result = await fetch.http(self.settings_network, url) result = await fetch.http(url)
status = result['status_code'] status = result['status_code']
if result and not result['error']: if result and not result['error']:
document = result['content'] document = result['content']
@ -738,24 +747,28 @@ class XmppCommands:
message = Feed.view_feed(url, feed) message = Feed.view_feed(url, feed)
break break
else: else:
result = await FeedDiscovery.probe_page(self.settings_network, self.pathnames, url, document) result = await FeedDiscovery.probe_page(url, document)
if isinstance(result, list): if isinstance(result, list):
results = result results = result
message = f"Syndication feeds found for {url}\n\n```\n" message = ("Syndication feeds found for {}\n\n```\n"
.format(url))
for result in results: for result in results:
result_name = result['name'] message += ("Title : {}\n"
result_link = result['link'] "Link : {}\n"
message += ("Title : {result_name}\n" "\n"
"Link : {result_link}\n\n") .format(result['name'], result['link']))
message += f'```\nTotal of {results} feeds.' message += ('```\nTotal of {} feeds.'
.format(len(results)))
break break
elif not result: elif not result:
message = f'> {url}\nNo subscriptions were found.' message = ('> {}\nNo subscriptions were found.'
.format(url))
break break
else: else:
url = result['link'] url = result['link']
else: else:
message = f'> {url}\nFailed to load URL. Reason: {status}' message = ('> {}\nFailed to load URL. Reason: {}'
.format(url, status))
break break
else: else:
message = ('No action has been taken. Missing URL.') message = ('No action has been taken. Missing URL.')
@ -763,7 +776,7 @@ class XmppCommands:
num = data[1] num = data[1]
if url.startswith('http'): if url.startswith('http'):
while True: while True:
result = await fetch.http(self.settings_network, url) result = await fetch.http(url)
if result and not result['error']: if result and not result['error']:
document = result['content'] document = result['content']
status = result['status_code'] status = result['status_code']
@ -772,25 +785,28 @@ class XmppCommands:
message = Feed.view_entry(url, feed, num) message = Feed.view_entry(url, feed, num)
break break
else: else:
result = await FeedDiscovery.probe_page( result = await FeedDiscovery.probe_page(url, document)
self.settings_network, self.pathnames, url, document)
if isinstance(result, list): if isinstance(result, list):
results = result results = result
message = f"Syndication feeds found for {url}\n\n```\n" message = ("Syndication feeds found for {}\n\n```\n"
.format(url))
for result in results: for result in results:
result_name = result['name'] message += ("Title : {}\n"
result_link = result['link'] "Link : {}\n"
message += (f"Title : {result_name}\n" "\n"
f"Link : {result_link}\\n") .format(result['name'], result['link']))
message += f'```\nTotal of {len(results)} feeds.' message += ('```\nTotal of {} feeds.'
.format(len(results)))
break break
elif not result: elif not result:
message = f'> {url}\nNo subscriptions were found.' message = ('> {}\nNo subscriptions were found.'
.format(url))
break break
else: else:
url = result['link'] url = result['link']
else: else:
message = f'> {url}\nFailed to load URL. Reason: {status}' message = ('> {}\nFailed to load URL. Reason: {}'
.format(url, status))
break break
else: else:
message = ('No action has been taken.' message = ('No action has been taken.'
@ -817,7 +833,7 @@ class XmppCommands:
message = '' message = ''
for i in result: for i in result:
title, url, date = i title, url, date = i
message += f'\n{title}\n{url}\n' message += ('\n{}\n{}\n'.format(title, url))
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
count = False count = False
@ -857,13 +873,13 @@ class XmppCommands:
if len(sub_removed): if len(sub_removed):
message += '\nThe following subscriptions have been removed:\n\n' message += '\nThe following subscriptions have been removed:\n\n'
for url in sub_removed: for url in sub_removed:
message += f'{url}\n' message += '{}\n'.format(url)
if len(url_invalid): if len(url_invalid):
urls = ', '.join(url_invalid) urls = ', '.join(url_invalid)
message += f'\nThe following URLs do not exist:\n\n{urls}\n' message += '\nThe following URLs do not exist:\n\n{}\n'.format(urls)
if len(ixs_invalid): if len(ixs_invalid):
ixs = ', '.join(ixs_invalid) ixs = ', '.join(ixs_invalid)
message += f'\nThe following indexes do not exist:\n\n{ixs}\n' message += '\nThe following indexes do not exist:\n\n{}\n'.format(ixs)
message += '\n```' message += '\n```'
else: else:
message = ('No action has been taken.' message = ('No action has been taken.'
@ -904,13 +920,13 @@ class XmppCommands:
if len(sub_marked): if len(sub_marked):
message += '\nThe following subscriptions have been marked as read:\n\n' message += '\nThe following subscriptions have been marked as read:\n\n'
for url in sub_marked: for url in sub_marked:
message += f'{url}\n' message += '{}\n'.format(url)
if len(url_invalid): if len(url_invalid):
urls = ', '.join(url_invalid) urls = ', '.join(url_invalid)
message += f'\nThe following URLs do not exist:\n\n{urls}\n' message += '\nThe following URLs do not exist:\n\n{}\n'.format(urls)
if len(ixs_invalid): if len(ixs_invalid):
ixs = ', '.join(ixs_invalid) ixs = ', '.join(ixs_invalid)
message += f'\nThe following indexes do not exist:\n\n{ixs}\n' message += '\nThe following indexes do not exist:\n\n{}\n'.format(ixs)
message += '\n```' message += '\n```'
else: else:
await sqlite.mark_all_as_read(db_file) await sqlite.mark_all_as_read(db_file)
@ -922,33 +938,37 @@ class XmppCommands:
if query: if query:
if len(query) > 3: if len(query) > 3:
results = sqlite.search_entries(db_file, query) results = sqlite.search_entries(db_file, query)
message = f"Search results for '{query}':\n\n```" message = ("Search results for '{}':\n\n```"
.format(query))
for result in results: for result in results:
message += f'\n{str(result[0])}\n{str(result[1])}\n' message += ("\n{}\n{}\n"
.format(str(result[0]), str(result[1])))
if len(results): if len(results):
message += f'```\nTotal of {len(results)} results' message += "```\nTotal of {} results".format(len(results))
else: else:
message = f'No results were found for: {query}' message = "No results were found for: {}".format(query)
else: else:
message = 'Enter at least 4 characters to search' message = 'Enter at least 4 characters to search'
else: else:
message = ('No action has been taken.\n' message = ('No action has been taken.'
'\n'
'Missing search query.') 'Missing search query.')
return message return message
# Tasks are classes which are passed to this function # Tasks are classes which are passed to this function
# On an occasion in which they would have returned, variable "tasks" might be called "callback" # On an occasion in which they would have returned, variable "tasks" might be called "callback"
async def scheduler_start(self, db_file, jid_bare, callbacks): async def scheduler_start(self, db_file, jid_bare, tasks):
await Config.set_setting_value(self, jid_bare, db_file, 'enabled', 1) await Config.set_setting_value(self.settings, jid_bare, db_file, 'enabled', 1)
for callback in callbacks: for task in tasks:
callback.restart_task(self, jid_bare) task.restart_task(self, jid_bare)
message = 'Updates are enabled.' message = 'Updates are enabled.'
return message return message
async def scheduler_stop(self, db_file, jid_bare): async def scheduler_stop(self, db_file, jid_bare):
await Config.set_setting_value(self, jid_bare, db_file, 'enabled', 0) await Config.set_setting_value(
self.settings, jid_bare, db_file, 'enabled', 0)
for task in ('interval', 'status'): for task in ('interval', 'status'):
if (jid_bare in self.task_manager and if (jid_bare in self.task_manager and
task in self.task_manager[jid_bare]): task in self.task_manager[jid_bare]):
@ -987,9 +1007,12 @@ class XmppCommands:
"\n" "\n"
"```" "```"
"\n" "\n"
f"News items : {entries_unread}/{entries}\n" "News items : {}/{}\n"
f"News sources : {feeds_active}/{feeds_all}\n" "News sources : {}/{}\n"
"```") "```").format(entries_unread,
entries,
feeds_active,
feeds_all)
return message return message
@ -1002,9 +1025,11 @@ class XmppCommands:
name = name[0] name = name[0]
addr = sqlite.get_feed_url(db_file, feed_id) addr = sqlite.get_feed_url(db_file, feed_id)
addr = addr[0] addr = addr[0]
message = f'Updates are now disabled for subscription:\n{addr}\n{name}' message = ('Updates are now disabled for subscription:\n{}\n{}'
.format(addr, name))
except: except:
message = f'No action has been taken. No news source with index {feed_id}.' message = ('No action has been taken. No news source with index {}.'
.format(feed_id))
XmppStatusTask.restart_task(self, jid_bare) XmppStatusTask.restart_task(self, jid_bare)
return message return message
@ -1015,11 +1040,13 @@ class XmppCommands:
await sqlite.set_enabled_status(db_file, feed_id, 1) await sqlite.set_enabled_status(db_file, feed_id, 1)
name = sqlite.get_feed_title(db_file, feed_id)[0] name = sqlite.get_feed_title(db_file, feed_id)[0]
addr = sqlite.get_feed_url(db_file, feed_id)[0] addr = sqlite.get_feed_url(db_file, feed_id)[0]
message = (f'> {addr}\n' message = ('> {}\n'
f'Updates are now enabled for news source "{name}"') 'Updates are now enabled for news source "{}"'
.format(addr, name))
except: except:
message = ('No action has been taken.\n' message = ('No action has been taken.'
f'No news source with index {feed_id}.') '\n'
'No news source with index {}.'.format(feed_id))
return message return message
@ -1041,11 +1068,13 @@ class XmppCommands:
else: else:
await sqlite.set_feed_title(db_file, feed_id, await sqlite.set_feed_title(db_file, feed_id,
name) name)
message = (f'> {name_old}\n' message = ('> {}'
f'Subscription #{feed_id} has been ' '\n'
f'renamed to "{name}".') 'Subscription #{} has been '
'renamed to "{}".'.format(
name_old,feed_id, name))
else: else:
message = f'Subscription with Id {feed_id} does not exist.' message = 'Subscription with Id {} does not exist.'.format(feed_id)
except: except:
message = ('No action has been taken.' message = ('No action has been taken.'
'\n' '\n'
@ -1058,50 +1087,11 @@ class XmppCommands:
return message return message
def add_jid_to_selector(self, jid, list_type):
filename = os.path.join(self.dir_config, 'selector.toml')
match list_type:
case 'blacklist':
list_type_list = self.blacklist
case 'whitelist':
list_type_list = self.whitelist
if jid in list_type_list:
message = f'Jabber ID {jid} is already included in {list_type}.\nNo action has been committed.'
else:
list_type_list.append(jid)
Config.update_toml_file(filename, self.selector)
message = f'Jabber ID {jid} has been added to {list_type}.'
return message
def del_jid_from_selector(self, jid, list_type):
filename = os.path.join(self.dir_config, 'selector.toml')
match list_type:
case 'blacklist':
list_type_list = self.blacklist
case 'whitelist':
list_type_list = self.whitelist
if jid in list_type_list:
list_type_list.remove(jid)
Config.update_toml_file(filename, self.selector)
message = f'Jabber ID "{jid}" has been removed from {list_type}.'
else:
message = f'Jabber ID "{jid}" was not found in {list_type}.\nNo action has been committed.'
return message
def print_selector(list_type):
jids = ' '.join(list_type) if list_type else '(empty).'
message = f'Jabber IDs: {jids}'
return message
def print_support_jid(): def print_support_jid():
muc_jid = 'slixfeed@chat.woodpeckersnest.space' muc_jid = 'slixfeed@chat.woodpeckersnest.space'
message = f'Join xmpp:{muc_jid}?join' message = 'Join xmpp:{}?join'.format(muc_jid)
return message return message
async def invite_jid_to_muc(self, jid_bare): async def invite_jid_to_muc(self, jid_bare):
muc_jid = 'slixfeed@chat.woodpeckersnest.space' muc_jid = 'slixfeed@chat.woodpeckersnest.space'
if await XmppUtilities.get_chat_type(self, jid_bare) == 'chat': if await XmppUtilities.get_chat_type(self, jid_bare) == 'chat':

3769
slixfeed/xmpp/component.py Normal file
View file

File diff suppressed because it is too large Load diff

347
slixfeed/xmpp/encryption.py
View file

@ -1,347 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
TODO
1) Deprecate "add" (see above) and make it interactive.
Slixfeed: Do you still want to add this URL to subscription list?
See: case _ if command_lowercase.startswith("add"):
2) If subscription is inadequate (see XmppPresence.request), send a message that says so.
elif not self.client_roster[jid]["to"]:
breakpoint()
message.reply("Share online status to activate bot.").send()
return
3) Set timeout for moderator interaction.
If moderator interaction has been made, and moderator approves the bot, then
the bot will add the given groupchat to bookmarks; otherwise, the bot will
send a message that it was not approved and therefore leaves the groupchat.
"""
import json
from omemo.storage import Just, Maybe, Nothing, Storage
from omemo.types import DeviceInformation, JSONType
import os
from slixfeed.config import Data
from slixfeed.log import Logger
from slixmpp import JID
from slixmpp.exceptions import IqTimeout, IqError
#from slixmpp.plugins import register_plugin
from slixmpp.stanza import Message
from slixmpp_omemo import TrustLevel, XEP_0384
from typing import Any, Dict, FrozenSet, Literal, Optional, Union
logger = Logger(__name__)
# for task in main_task:
# task.cancel()
# Deprecated in favour of event "presence_available"
# if not main_task:
# await select_file()
class XmppOmemo:
async def decrypt(self, stanza: Message):
omemo_decrypted = None
mto = stanza["from"]
mtype = stanza["type"]
namespace = self['xep_0384'].is_encrypted(stanza)
if namespace is None:
omemo_decrypted = False
response = f"Unencrypted message or unsupported message encryption: {stanza['body']}"
else:
print(f'Message in namespace {namespace} received: {stanza}')
try:
response, device_information = await self['xep_0384'].decrypt_message(stanza)
print(f'Information about sender: {device_information}')
omemo_decrypted = True
except Exception as e: # pylint: disable=broad-exception-caught
response = f'Error {type(e).__name__}: {e}'
return response, omemo_decrypted
async def encrypt(
self,
mto: JID,
mtype: Literal['chat', 'normal'],
mbody: str
) -> None:
if isinstance(mbody, str):
reply = self.make_message(mto=mto, mtype=mtype)
reply['body'] = mbody
reply.set_to(mto)
reply.set_from(self.boundjid)
# It might be a good idea to strip everything except for the body from the stanza,
# since some things might break when echoed.
message, encryption_errors = await self['xep_0384'].encrypt_message(reply, mto)
if len(encryption_errors) > 0:
print(f'There were non-critical errors during encryption: {encryption_errors}')
#log.info(f'There were non-critical errors during encryption: {encryption_errors}')
# for namespace, message in messages.items():
# message['eme']['namespace'] = namespace
# message['eme']['name'] = self['xep_0380'].mechanisms[namespace]
return message, True
async def _decrypt(self, message: Message, allow_untrusted: bool = False):
jid = message['from']
try:
print('XmppOmemo.decrypt')
message_omemo_encrypted = message['omemo_encrypted']
message_body = await self['xep_0384'].decrypt_message(
message_omemo_encrypted, jid, allow_untrusted)
# decrypt_message returns Optional[str]. It is possible to get
# body-less OMEMO message (see KeyTransportMessages), currently
# used for example to send heartbeats to other devices.
if message_body is not None:
response = message_body.decode('utf8')
omemo_decrypted = True
else:
omemo_decrypted = response = None
retry = None
except (MissingOwnKey,) as exn:
print('XmppOmemo.decrypt. except: MissingOwnKey')
# The message is missing our own key, it was not encrypted for
# us, and we can't decrypt it.
response = ('Error: Your message has not been encrypted for '
'Slixfeed (MissingOwnKey).')
omemo_decrypted = False
retry = False
logger.error(exn)
except (NoAvailableSession,) as exn:
print('XmppOmemo.decrypt. except: NoAvailableSession')
# We received a message from that contained a session that we
# don't know about (deleted session storage, etc.). We can't
# decrypt the message, and it's going to be lost.
# Here, as we need to initiate a new encrypted session, it is
# best if we send an encrypted message directly. XXX: Is it
# where we talk about self-healing messages?
response = ('Error: Your message has not been encrypted for '
'Slixfeed (NoAvailableSession).')
omemo_decrypted = False
retry = False
logger.error(exn)
except (UndecidedException, UntrustedException) as exn:
print('XmppOmemo.decrypt. except: UndecidedException')
print('XmppOmemo.decrypt. except: UntrustedException')
# We received a message from an untrusted device. We can
# choose to decrypt the message nonetheless, with the
# `allow_untrusted` flag on the `decrypt_message` call, which
# we will do here. This is only possible for decryption,
# encryption will require us to decide if we trust the device
# or not. Clients _should_ indicate that the message was not
# trusted, or in undecided state, if they decide to decrypt it
# anyway.
response = (f'Error: Device "{exn.device}" is not present in the '
'trusted devices of Slixfeed.')
omemo_decrypted = False
retry = True
logger.error(exn)
# We resend, setting the `allow_untrusted` parameter to True.
# await XmppChat.process_message(self, message, allow_untrusted=True)
except (EncryptionPrepareException,) as exn:
print('XmppOmemo.decrypt. except: EncryptionPrepareException')
# Slixmpp tried its best, but there were errors it couldn't
# resolve. At this point you should have seen other exceptions
# and given a chance to resolve them already.
response = ('Error: Your message has not been encrypted for '
'Slixfeed (EncryptionPrepareException).')
omemo_decrypted = False
retry = False
logger.error(exn)
except (Exception,) as exn:
print('XmppOmemo.decrypt. except: Exception')
response = ('Error: Your message has not been encrypted for '
'Slixfeed (Unknown).')
omemo_decrypted = False
retry = False
logger.error(exn)
raise
return response, omemo_decrypted, retry
async def _encrypt(self, jid: JID, message_body):
print(jid)
print(message_body)
expect_problems = {} # type: Optional[Dict[JID, List[int]]]
while True:
try:
print('XmppOmemo.encrypt')
# `encrypt_message` excepts the plaintext to be sent, a list of
# bare JIDs to encrypt to, and optionally a dict of problems to
# expect per bare JID.
#
# Note that this function returns an `<encrypted/>` object,
# and not a full Message stanza. This combined with the
# `recipients` parameter that requires for a list of JIDs,
# allows you to encrypt for 1:1 as well as groupchats (MUC).
#
# `expect_problems`: See EncryptionPrepareException handling.
recipients = [jid]
message_body = await self['xep_0384'].encrypt_message(
message_body, recipients, expect_problems)
omemo_encrypted = True
break
except UndecidedException as exn:
print('XmppOmemo.encrypt. except: UndecidedException')
# The library prevents us from sending a message to an
# untrusted/undecided barejid, so we need to make a decision here.
# This is where you prompt your user to ask what to do. In
# this bot we will automatically trust undecided recipients.
await self['xep_0384'].trust(exn.bare_jid, exn.device, exn.ik)
omemo_encrypted = False
# TODO: catch NoEligibleDevicesException
except EncryptionPrepareException as exn:
print('XmppOmemo.encrypt. except: EncryptionPrepareException')
# This exception is being raised when the library has tried
# all it could and doesn't know what to do anymore. It
# contains a list of exceptions that the user must resolve, or
# explicitely ignore via `expect_problems`.
# TODO: We might need to bail out here if errors are the same?
for error in exn.errors:
if isinstance(error, MissingBundleException):
# We choose to ignore MissingBundleException. It seems
# to be somewhat accepted that it's better not to
# encrypt for a device if it has problems and encrypt
# for the rest, rather than error out. The "faulty"
# device won't be able to decrypt and should display a
# generic message. The receiving end-user at this
# point can bring up the issue if it happens.
message_body = (f'Could not find keys for device '
'"{error.device}"'
f' of recipient "{error.bare_jid}". '
'Skipping.')
omemo_encrypted = False
jid = JID(error.bare_jid)
device_list = expect_problems.setdefault(jid, [])
device_list.append(error.device)
except (IqError, IqTimeout) as exn:
print('XmppOmemo.encrypt. except: IqError, IqTimeout')
message_body = ('An error occured while fetching information '
'on a recipient.\n%r' % exn)
omemo_encrypted = False
except Exception as exn:
print('XmppOmemo.encrypt. except: Exception')
message_body = ('An error occured while attempting to encrypt'
'.\n%r' % exn)
omemo_encrypted = False
raise
return message_body, omemo_encrypted
class StorageImpl(Storage):
"""
Example storage implementation that stores all data in a single JSON file.
"""
dir_data = Data.get_directory()
omemo_dir = os.path.join(dir_data, 'omemo')
JSON_FILE = os.path.join(omemo_dir, 'omemo.json')
# TODO Pass JID
#JSON_FILE = os.path.join(omemo_dir, f'{jid_bare}.json')
def __init__(self) -> None:
super().__init__()
self.__data: Dict[str, JSONType] = {}
try:
with open(self.JSON_FILE, encoding="utf8") as f:
self.__data = json.load(f)
except Exception: # pylint: disable=broad-exception-caught
pass
async def _load(self, key: str) -> Maybe[JSONType]:
if key in self.__data:
return Just(self.__data[key])
return Nothing()
async def _store(self, key: str, value: JSONType) -> None:
self.__data[key] = value
with open(self.JSON_FILE, "w", encoding="utf8") as f:
json.dump(self.__data, f)
async def _delete(self, key: str) -> None:
self.__data.pop(key, None)
with open(self.JSON_FILE, "w", encoding="utf8") as f:
json.dump(self.__data, f)
class XEP_0384Impl(XEP_0384): # pylint: disable=invalid-name
"""
Example implementation of the OMEMO plugin for Slixmpp.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None: # pylint: disable=redefined-outer-name
super().__init__(*args, **kwargs)
# Just the type definition here
self.__storage: Storage
def plugin_init(self) -> None:
self.__storage = StorageImpl()
super().plugin_init()
@property
def storage(self) -> Storage:
return self.__storage
@property
def _btbv_enabled(self) -> bool:
return True
async def _devices_blindly_trusted(
self,
blindly_trusted: FrozenSet[DeviceInformation],
identifier: Optional[str]
) -> None:
print(f"[{identifier}] Devices trusted blindly: {blindly_trusted}")
#log.info(f"[{identifier}] Devices trusted blindly: {blindly_trusted}")
async def _prompt_manual_trust(
self,
manually_trusted: FrozenSet[DeviceInformation],
identifier: Optional[str]
) -> None:
# Since BTBV is enabled and we don't do any manual trust adjustments in the example, this method
# should never be called. All devices should be automatically trusted blindly by BTBV.
# To show how a full implementation could look like, the following code will prompt for a trust
# decision using `input`:
session_mananger = await self.get_session_manager()
for device in manually_trusted:
while True:
answer = input(f"[{identifier}] Trust the following device? (yes/no) {device}")
if answer in { "yes", "no" }:
await session_mananger.set_trust(
device.bare_jid,
device.identity_key,
TrustLevel.TRUSTED.value if answer == "yes" else TrustLevel.DISTRUSTED.value
)
break
print("Please answer yes or no.")
#register_plugin(XEP_0384Impl)

16
slixfeed/xmpp/groupchat.py
View file

@ -34,24 +34,14 @@ class XmppGroupchat:
'bookmark {}'.format(bookmark['name'])) 'bookmark {}'.format(bookmark['name']))
alias = bookmark["nick"] alias = bookmark["nick"]
muc_jid = bookmark["jid"] muc_jid = bookmark["jid"]
# Message.printer('Joining to MUC {} ...'.format(muc_jid)) Message.printer('Joining to MUC {} ...'.format(muc_jid))
print('Joining to MUC {} ...'.format(muc_jid))
result = await XmppMuc.join(self, muc_jid, alias) result = await XmppMuc.join(self, muc_jid, alias)
match result: if result == 'ban':
case 'ban':
await XmppBookmark.remove(self, muc_jid) await XmppBookmark.remove(self, muc_jid)
logger.warning('{} is banned from {}'.format(self.alias, muc_jid)) logger.warning('{} is banned from {}'.format(self.alias, muc_jid))
logger.warning('Groupchat {} has been removed from bookmarks' logger.warning('Groupchat {} has been removed from bookmarks'
.format(muc_jid)) .format(muc_jid))
case 'error': else:
logger.warning('An error has occured while attempting '
'to join to groupchat {}'
.format(muc_jid))
case 'timeout':
logger.warning('Timeout has reached while attempting '
'to join to groupchat {}'
.format(muc_jid))
case _:
logger.info('Autojoin groupchat\n' logger.info('Autojoin groupchat\n'
'Name : {}\n' 'Name : {}\n'
'JID : {}\n' 'JID : {}\n'

12
slixfeed/xmpp/ipc.py
View file

@ -11,8 +11,8 @@ socket (i.e. clients[fd]) from the respective client.
import asyncio import asyncio
import os import os
import slixfeed.config as config
from slixfeed.syndication import FeedTask from slixfeed.syndication import FeedTask
from slixfeed.utilities import Database
from slixfeed.xmpp.chat import XmppChatTask from slixfeed.xmpp.chat import XmppChatTask
from slixfeed.xmpp.commands import XmppCommands from slixfeed.xmpp.commands import XmppCommands
from slixfeed.xmpp.chat import XmppChatAction from slixfeed.xmpp.chat import XmppChatAction
@ -85,7 +85,7 @@ class XmppIpcServer:
if '~' in data: if '~' in data:
data_list = data.split('~') data_list = data.split('~')
jid_bare = data_list[0] jid_bare = data_list[0]
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
command = data_list[1] command = data_list[1]
else: else:
command = data command = data
@ -177,7 +177,7 @@ class XmppIpcServer:
ext = command[7:] ext = command[7:]
if ext in ('md', 'opml'): if ext in ('md', 'opml'):
filename, result = XmppCommands.export_feeds( filename, result = XmppCommands.export_feeds(
self.dir_data, self.dir_cache, jid_bare, ext) self, jid_bare, ext)
response = result + ' : ' + filename response = result + ' : ' + filename
else: else:
response = 'Unsupported filetype. Try: md or opml' response = 'Unsupported filetype. Try: md or opml'
@ -204,10 +204,10 @@ class XmppIpcServer:
response = await XmppCommands.import_opml( response = await XmppCommands.import_opml(
self, db_file, jid_bare, command) self, db_file, jid_bare, command)
case 'info': case 'info':
response = XmppCommands.print_info_list(self) response = XmppCommands.print_info_list()
case _ if command.startswith('info'): case _ if command.startswith('info'):
entry = command[5:].lower() entry = command[5:].lower()
response = XmppCommands.print_info_specific(self, entry) response = XmppCommands.print_info_specific(entry)
case 'pubsub list': case 'pubsub list':
response = await XmppCommands.pubsub_list( response = await XmppCommands.pubsub_list(
self, jid_bare) self, jid_bare)
@ -231,7 +231,7 @@ class XmppIpcServer:
command.startswith('itpc:/') or command.startswith('itpc:/') or
command.startswith('rss:/')): command.startswith('rss:/')):
response = await XmppCommands.fetch_http( response = await XmppCommands.fetch_http(
self.settings_network, command, db_file, jid_bare) self, command, db_file, jid_bare)
case _ if command.startswith('interval'): case _ if command.startswith('interval'):
val = command[9:] val = command[9:]
if val: if val:

5
slixfeed/xmpp/iq.py
View file

@ -10,13 +10,10 @@ class XmppIQ:
async def send(self, iq): async def send(self, iq):
try: try:
result = await iq.send(timeout=15) await iq.send(timeout=15)
except IqTimeout as e: except IqTimeout as e:
logger.error('Error Timeout') logger.error('Error Timeout')
logger.error(str(e)) logger.error(str(e))
result = e
except IqError as e: except IqError as e:
logger.error('Error XmppIQ') logger.error('Error XmppIQ')
logger.error(str(e)) logger.error(str(e))
result = e
return result

41
slixfeed/xmpp/message.py
View file

@ -2,7 +2,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from slixfeed.log import Logger from slixfeed.log import Logger
from slixmpp import JID
import xml.sax.saxutils as saxutils import xml.sax.saxutils as saxutils
logger = Logger(__name__) logger = Logger(__name__)
@ -40,46 +39,6 @@ class XmppMessage:
mnick=self.alias) mnick=self.alias)
def send_omemo(self, jid: JID, chat_type, response_encrypted):
# jid_from = str(self.boundjid) if self.is_component else None
# message = self.make_message(mto=jid, mfrom=jid_from, mtype=chat_type)
# eme_ns = 'eu.siacs.conversations.axolotl'
# message['eme']['namespace'] = eme_ns
# message['eme']['name'] = self['xep_0380'].mechanisms[eme_ns]
# message['eme'] = {'name': self['xep_0380'].mechanisms[eme_ns]}
# message['eme'] = {'namespace': eme_ns}
# message.append(response_encrypted)
for namespace, message in response_encrypted.items():
message['eme']['namespace'] = namespace
message['eme']['name'] = self['xep_0380'].mechanisms[namespace]
message.send()
def send_omemo_oob(self, jid: JID, url_encrypted, chat_type, aesgcm=False):
jid_from = str(self.boundjid) if self.is_component else None
# if not aesgcm: url_encrypted = saxutils.escape(url_encrypted)
message = self.make_message(mto=jid, mfrom=jid_from, mtype=chat_type)
eme_ns = 'eu.siacs.conversations.axolotl'
# message['eme']['namespace'] = eme_ns
# message['eme']['name'] = self['xep_0380'].mechanisms[eme_ns]
message['eme'] = {'namespace': eme_ns}
# message['eme'] = {'name': self['xep_0380'].mechanisms[eme_ns]}
message['oob']['url'] = url_encrypted
message.append(url_encrypted)
message.send()
# FIXME Solve this function
def send_omemo_reply(self, message, response_encrypted):
eme_ns = 'eu.siacs.conversations.axolotl'
# message['eme']['namespace'] = eme_ns
# message['eme']['name'] = self['xep_0380'].mechanisms[eme_ns]
message['eme'] = {'namespace': eme_ns}
# message['eme'] = {'name': self['xep_0380'].mechanisms[eme_ns]}
message.append(response_encrypted)
message.reply(message['body']).send()
# NOTE We might want to add more characters # NOTE We might want to add more characters
# def escape_to_xml(raw_string): # def escape_to_xml(raw_string):
# escape_map = { # escape_map = {

11
slixfeed/xmpp/muc.py
View file

@ -16,7 +16,6 @@ FIXME
1) Save name of groupchat instead of jid as name 1) Save name of groupchat instead of jid as name
""" """
from asyncio import TimeoutError
from slixmpp.exceptions import IqError, IqTimeout, PresenceError from slixmpp.exceptions import IqError, IqTimeout, PresenceError
from slixfeed.log import Logger from slixfeed.log import Logger
@ -47,7 +46,7 @@ class XmppMuc:
# ) # )
logger.info('Joining groupchat\nJID : {}\n'.format(jid)) logger.info('Joining groupchat\nJID : {}\n'.format(jid))
jid_from = str(self.boundjid) if self.is_component else None jid_from = str(self.boundjid) if self.is_component else None
if not alias: alias = self.alias if alias == None: self.alias
try: try:
await self.plugin['xep_0045'].join_muc_wait(jid, await self.plugin['xep_0045'].join_muc_wait(jid,
alias, alias,
@ -69,11 +68,6 @@ class XmppMuc:
logger.error(str(e)) logger.error(str(e))
logger.error(jid) logger.error(jid)
result = 'timeout' result = 'timeout'
except TimeoutError as e:
logger.error('Timeout AsyncIO')
logger.error(str(e))
logger.error(jid)
result = 'timeout'
except PresenceError as e: except PresenceError as e:
logger.error('Error Presence') logger.error('Error Presence')
logger.error(str(e)) logger.error(str(e))
@ -81,9 +75,6 @@ class XmppMuc:
e.presence['error']['code'] == '403'): e.presence['error']['code'] == '403'):
logger.warning('{} is banned from {}'.format(self.alias, jid)) logger.warning('{} is banned from {}'.format(self.alias, jid))
result = 'ban' result = 'ban'
elif e.condition == 'conflict':
logger.warning(e.presence['error']['text'])
result = 'conflict'
else: else:
result = 'error' result = 'error'
return result return result

10
slixfeed/xmpp/profile.py
View file

@ -27,6 +27,7 @@ TODO
import glob import glob
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.config as config
from slixfeed.log import Logger from slixfeed.log import Logger
from slixmpp.exceptions import IqTimeout, IqError from slixmpp.exceptions import IqTimeout, IqError
import os import os
@ -56,7 +57,9 @@ async def update(self):
async def set_avatar(self): async def set_avatar(self):
config_dir = self.dir_config config_dir = config.get_default_config_directory()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
filename = glob.glob(config_dir + '/image.*') filename = glob.glob(config_dir + '/image.*')
if not filename and os.path.isdir('/usr/share/slixfeed/'): if not filename and os.path.isdir('/usr/share/slixfeed/'):
# filename = '/usr/share/slixfeed/image.svg' # filename = '/usr/share/slixfeed/image.svg'
@ -108,7 +111,8 @@ def set_identity(self, category):
async def set_vcard(self): async def set_vcard(self):
vcard = self.plugin['xep_0054'].make_vcard() vcard = self.plugin['xep_0054'].make_vcard()
profile = self.data_accounts_xmpp['profile'] profile = config.get_values('accounts.toml', 'xmpp')['profile']
for key in profile: vcard[key] = profile[key] for key in profile:
vcard[key] = profile[key]
await self.plugin['xep_0054'].publish_vcard(vcard) await self.plugin['xep_0054'].publish_vcard(vcard)

158
slixfeed/xmpp/publish.py
View file

@ -9,14 +9,14 @@ Functions create_node and create_entry are derived from project atomtopubsub.
import asyncio import asyncio
import hashlib import hashlib
import os
import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub
from slixmpp.xmlstream import ET from slixmpp.xmlstream import ET
import slixfeed.config as config
from slixfeed.config import Config from slixfeed.config import Config
from slixfeed.log import Logger from slixfeed.log import Logger
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
from slixfeed.syndication import Feed from slixfeed.syndication import Feed
from slixfeed.utilities import Database, String, Url, Utilities from slixfeed.utilities import String, Url, Utilities
from slixfeed.xmpp.iq import XmppIQ from slixfeed.xmpp.iq import XmppIQ
import sys import sys
@ -44,10 +44,10 @@ class XmppPubsub:
return results return results
async def get_node_properties(self, jid_bare, node): async def get_node_properties(self, jid, node):
config = await self.plugin['xep_0060'].get_node_config(jid_bare, node) config = await self.plugin['xep_0060'].get_node_config(jid, node)
subscriptions = await self.plugin['xep_0060'].get_node_subscriptions(jid_bare, node) subscriptions = await self.plugin['xep_0060'].get_node_subscriptions(jid, node)
affiliations = await self.plugin['xep_0060'].get_node_affiliations(jid_bare, node) affiliations = await self.plugin['xep_0060'].get_node_affiliations(jid, node)
properties = {'config': config, properties = {'config': config,
'subscriptions': subscriptions, 'subscriptions': subscriptions,
'affiliations': affiliations} 'affiliations': affiliations}
@ -55,48 +55,49 @@ class XmppPubsub:
return properties return properties
async def get_node_configuration(self, jid, node_id):
async def get_node_configuration(self, jid_bare, node_id): node = await self.plugin['xep_0060'].get_node_config(jid, node_id)
node = await self.plugin['xep_0060'].get_node_config(jid_bare, node_id) if not node:
print('NODE CONFIG', node_id, str(node))
return node return node
async def get_nodes(self, jid_bare): async def get_nodes(self, jid):
nodes = await self.plugin['xep_0060'].get_nodes(jid_bare) nodes = await self.plugin['xep_0060'].get_nodes(jid)
# 'self' would lead to slixmpp.jid.InvalidJID: idna validation failed: # 'self' would lead to slixmpp.jid.InvalidJID: idna validation failed:
return nodes return nodes
async def get_item(self, jid_bare, node, item_id): async def get_item(self, jid, node, item_id):
item = await self.plugin['xep_0060'].get_item(jid_bare, node, item_id) item = await self.plugin['xep_0060'].get_item(jid, node, item_id)
return item return item
async def get_items(self, jid_bare, node): async def get_items(self, jid, node):
items = await self.plugin['xep_0060'].get_items(jid_bare, node) items = await self.plugin['xep_0060'].get_items(jid, node)
return items return items
def delete_node(self, jid_bare, node): def delete_node(self, jid, node):
jid_from = self.boundjid.bare if self.is_component else None jid_from = str(self.boundjid) if self.is_component else None
self.plugin['xep_0060'].delete_node(jid_bare, node, ifrom=jid_from) self.plugin['xep_0060'].delete_node(jid, node, ifrom=jid_from)
def purge_node(self, jid_bare, node): def purge_node(self, jid, node):
jid_from = self.boundjid.bare if self.is_component else None jid_from = str(self.boundjid) if self.is_component else None
self.plugin['xep_0060'].purge(jid_bare, node, ifrom=jid_from) self.plugin['xep_0060'].purge(jid, node, ifrom=jid_from)
# iq = self.Iq(stype='set', # iq = self.Iq(stype='set',
# sto=jid_bare, # sto=jid,
# sfrom=jid_from) # sfrom=jid_from)
# iq['pubsub']['purge']['node'] = node # iq['pubsub']['purge']['node'] = node
# return iq # return iq
# TODO Make use of var "xep" with match/case (XEP-0060, XEP-0277, XEP-0472) # TODO Make use of var "xep" with match/case (XEP-0060, XEP-0277, XEP-0472)
def create_node(self, jid_bare, node, xep=None ,title=None, subtitle=None): def create_node(self, jid, node, xep ,title=None, subtitle=None):
jid_from = self.boundjid.bare if self.is_component else None jid_from = str(self.boundjid) if self.is_component else None
iq = self.Iq(stype='set', iq = self.Iq(stype='set',
sto=jid_bare, sto=jid,
sfrom=jid_from) sfrom=jid_from)
iq['pubsub']['create']['node'] = node iq['pubsub']['create']['node'] = node
form = iq['pubsub']['configure']['form'] form = iq['pubsub']['configure']['form']
@ -130,8 +131,8 @@ class XmppPubsub:
# TODO Consider to create a separate function called "create_atom_entry" # TODO Consider to create a separate function called "create_atom_entry"
# or "create_rfc4287_entry" for anything related to variable "node_entry". # or "create_rfc4287_entry" for anything related to variable "node_entry".
def create_entry(self, jid_bare, node_id, item_id, node_item): def create_entry(self, jid, node_id, item_id, node_item):
iq = self.Iq(stype="set", sto=jid_bare) iq = self.Iq(stype="set", sto=jid)
iq['pubsub']['publish']['node'] = node_id iq['pubsub']['publish']['node'] = node_id
item = pubsub.Item() item = pubsub.Item()
@ -152,8 +153,8 @@ class XmppPubsub:
return iq return iq
def _create_entry(self, jid_bare, node, entry, version): def _create_entry(self, jid, node, entry, version):
iq = self.Iq(stype="set", sto=jid_bare) iq = self.Iq(stype="set", sto=jid)
iq['pubsub']['publish']['node'] = node iq['pubsub']['publish']['node'] = node
item = pubsub.Item() item = pubsub.Item()
@ -259,10 +260,21 @@ class XmppPubsubAction:
async def send_selected_entry(self, jid_bare, node_id, entry_id): async def send_selected_entry(self, jid_bare, node_id, entry_id):
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare)) logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare))
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
report = {}
if jid_bare == self.boundjid.bare:
node_id = 'urn:xmpp:microblog:0'
node_subtitle = None
node_title = None
else:
feed_id = sqlite.get_feed_id_by_entry_index(db_file, entry_id) feed_id = sqlite.get_feed_id_by_entry_index(db_file, entry_id)
feed_id = feed_id[0] feed_id = feed_id[0]
node_id, node_title, node_subtitle = sqlite.get_feed_properties(db_file, feed_id) node_id, node_title, node_subtitle = sqlite.get_feed_properties(db_file, feed_id)
print('THIS IS A TEST')
print(node_id)
print(node_title)
print(node_subtitle)
print('THIS IS A TEST')
xep = None xep = None
iq_create_node = XmppPubsub.create_node( iq_create_node = XmppPubsub.create_node(
self, jid_bare, node_id, xep, node_title, node_subtitle) self, jid_bare, node_id, xep, node_title, node_subtitle)
@ -272,14 +284,14 @@ class XmppPubsubAction:
print(node_id) print(node_id)
entry_dict = Feed.pack_entry_into_dict(db_file, entry) entry_dict = Feed.pack_entry_into_dict(db_file, entry)
node_item = Feed.create_rfc4287_entry(entry_dict) node_item = Feed.create_rfc4287_entry(entry_dict)
item_id = Utilities.hash_url_to_md5(entry_dict['link']) entry_url = entry_dict['link']
item_id = Utilities.hash_url_to_md5(entry_url)
iq_create_entry = XmppPubsub.create_entry( iq_create_entry = XmppPubsub.create_entry(
self, jid_bare, node_id, item_id, node_item) self, jid_bare, node_id, item_id, node_item)
await XmppIQ.send(self, iq_create_entry) await XmppIQ.send(self, iq_create_entry)
await sqlite.mark_as_read(db_file, entry_id) await sqlite.mark_as_read(db_file, entry_id)
report = entry_url
# NOTE This value is returned for the sake of testing return report
return entry_dict['link']
async def send_unread_items(self, jid_bare): async def send_unread_items(self, jid_bare):
@ -287,7 +299,7 @@ class XmppPubsubAction:
Parameters Parameters
---------- ----------
jid_bare : str jid_bare : TYPE
Bare Jabber ID. Bare Jabber ID.
Returns Returns
@ -298,7 +310,7 @@ class XmppPubsubAction:
""" """
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare)) logger.debug('{}: jid_bare: {}'.format(function_name, jid_bare))
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
report = {} report = {}
subscriptions = sqlite.get_active_feeds_url(db_file) subscriptions = sqlite.get_active_feeds_url(db_file)
for url in subscriptions: for url in subscriptions:
@ -309,8 +321,14 @@ class XmppPubsubAction:
feed_id = sqlite.get_feed_id(db_file, url) feed_id = sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0] feed_id = feed_id[0]
# Publish to node 'urn:xmpp:microblog:0' for own JID
# Publish to node based on feed identifier for PubSub service. # Publish to node based on feed identifier for PubSub service.
if jid_bare == self.boundjid.bare:
node_id = 'urn:xmpp:microblog:0'
node_subtitle = None
node_title = None
else:
# node_id = feed_properties[2] # node_id = feed_properties[2]
# node_title = feed_properties[3] # node_title = feed_properties[3]
# node_subtitle = feed_properties[5] # node_subtitle = feed_properties[5]
@ -332,24 +350,11 @@ class XmppPubsubAction:
node_subtitle = sqlite.get_feed_subtitle(db_file, feed_id) node_subtitle = sqlite.get_feed_subtitle(db_file, feed_id)
node_subtitle = node_subtitle[0] node_subtitle = node_subtitle[0]
xep = None xep = None
#node_exist = await XmppPubsub.get_node_configuration(self, jid_bare, node_id) node_exist = await XmppPubsub.get_node_configuration(self, jid_bare, node_id)
nodes = await XmppPubsub.get_nodes(self, jid_bare)
node_items = nodes['disco_items']['items']
node_exist = False
for node_item in node_items:
if node_item[1] == node_id:
node_exist = True
break
print(['node_exist', node_exist])
if not node_exist: if not node_exist:
iq_create_node = XmppPubsub.create_node( iq_create_node = XmppPubsub.create_node(
self, jid_bare, node_id, xep, node_title, node_subtitle) self, jid_bare, node_id, xep, node_title, node_subtitle)
result = await XmppIQ.send(self, iq_create_node) await XmppIQ.send(self, iq_create_node)
result_condition = result.iq['error']['condition']
if result_condition in ('forbidden', 'service-unavailable'):
reason = result.iq['error']['text']
print('Creation of node {} for JID {} has failed'.format(node_id, jid_bare, reason))
return
entries = sqlite.get_unread_entries_of_feed(db_file, feed_id) entries = sqlite.get_unread_entries_of_feed(db_file, feed_id)
report[url] = len(entries) report[url] = len(entries)
for entry in entries: for entry in entries:
@ -357,63 +362,24 @@ class XmppPubsubAction:
node_entry = Feed.create_rfc4287_entry(feed_entry) node_entry = Feed.create_rfc4287_entry(feed_entry)
entry_url = feed_entry['link'] entry_url = feed_entry['link']
item_id = Utilities.hash_url_to_md5(entry_url) item_id = Utilities.hash_url_to_md5(entry_url)
print(['PubSub node item was sent to', jid_bare, node_id]) print('PubSub node item was sent to', jid_bare, node_id)
print([entry_url, item_id]) print(entry_url)
print(item_id)
iq_create_entry = XmppPubsub.create_entry( iq_create_entry = XmppPubsub.create_entry(
self, jid_bare, node_id, item_id, node_entry) self, jid_bare, node_id, item_id, node_entry)
result = await XmppIQ.send(self, iq_create_entry) await XmppIQ.send(self, iq_create_entry)
ix = entry[0] ix = entry[0]
await sqlite.mark_as_read(db_file, ix) await sqlite.mark_as_read(db_file, ix)
print(report)
return report return report
class XmppPubsubTask: class XmppPubsubTask:
async def loop_task(self, jid_bare):
db_file = Database.instantiate(self.dir_data, jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self, jid_bare, db_file)
while True:
print('Looping task "publish" for JID {}'.format(jid_bare))
if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info('Stopping task "publish" for JID {}'.format(jid_bare))
try:
self.task_manager[jid_bare]['publish'].cancel()
except:
logger.info('No task "publish" for JID {} (XmppPubsubAction.send_unread_items)'
.format(jid_bare))
logger.info('Starting tasks "publish" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['publish'] = asyncio.create_task(
XmppPubsubAction.send_unread_items(self, jid_bare))
await asyncio.sleep(60 * 180)
def restart_task(self, jid_bare):
db_file = Database.instantiate(self.dir_data, jid_bare)
if jid_bare not in self.settings:
Config.add_settings_jid(self, jid_bare, db_file)
if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info('Stopping task "publish" for JID {}'.format(jid_bare))
try:
self.task_manager[jid_bare]['publish'].cancel()
except:
logger.info('No task "publish" for JID {} (XmppPubsubAction.send_unread_items)'
.format(jid_bare))
logger.info('Starting tasks "publish" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['publish'] = asyncio.create_task(
XmppPubsubAction.send_unread_items(self, jid_bare))
async def task_publish(self, jid_bare): async def task_publish(self, jid_bare):
db_file = Database.instantiate(self.dir_data, jid_bare) db_file = config.get_pathname_to_database(jid_bare)
if jid_bare not in self.settings: if jid_bare not in self.settings:
Config.add_settings_jid(self, jid_bare, db_file) Config.add_settings_jid(self.settings, jid_bare, db_file)
while True: while True:
await XmppPubsubAction.send_unread_items(self, jid_bare) await XmppPubsubAction.send_unread_items(self, jid_bare)
await asyncio.sleep(60 * 180) await asyncio.sleep(60 * 180)

24
slixfeed/xmpp/status.py
View file

@ -2,8 +2,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import asyncio import asyncio
import os
from slixfeed.config import Config from slixfeed.config import Config
import slixfeed.config as config
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
from slixfeed.log import Logger from slixfeed.log import Logger
from slixfeed.xmpp.presence import XmppPresence from slixfeed.xmpp.presence import XmppPresence
@ -25,11 +25,11 @@ class XmppStatus:
Jabber ID. Jabber ID.
""" """
function_name = sys._getframe().f_code.co_name function_name = sys._getframe().f_code.co_name
logger.debug(f'{function_name}: jid: {jid_bare}') logger.debug('{}: jid: {}'.format(function_name, jid_bare))
status_text = '📜️ Slixfeed RSS News Bot' status_text = '📜️ Slixfeed RSS News Bot'
enabled = Config.get_setting_value(self, jid_bare, 'enabled') db_file = config.get_pathname_to_database(jid_bare)
enabled = Config.get_setting_value(self.settings, jid_bare, 'enabled')
if enabled: if enabled:
db_file = Database.instantiate(self.dir_data, jid_bare)
jid_task = self.pending_tasks[jid_bare] if jid_bare in self.pending_tasks else None jid_task = self.pending_tasks[jid_bare] if jid_bare in self.pending_tasks else None
if jid_task and len(jid_task): if jid_task and len(jid_task):
# print('status dnd for ' + jid_bare) # print('status dnd for ' + jid_bare)
@ -47,10 +47,10 @@ class XmppStatus:
if unread: if unread:
# print('status unread for ' + jid_bare) # print('status unread for ' + jid_bare)
status_mode = 'chat' status_mode = 'chat'
status_text = f'📬️ There are {str(unread)} news items' status_text = '📬️ There are {} news items'.format(str(unread))
else: else:
# print('status no news for ' + jid_bare) # print('status no news for ' + jid_bare)
status_mode = 'away' status_mode = 'available'
status_text = '📭️ No news' status_text = '📭️ No news'
else: else:
# print('status disabled for ' + jid_bare) # print('status disabled for ' + jid_bare)
@ -73,13 +73,14 @@ class XmppStatusTask:
return return
if jid_bare not in self.task_manager: if jid_bare not in self.task_manager:
self.task_manager[jid_bare] = {} self.task_manager[jid_bare] = {}
logger.info('Creating new task manager for JID {jid_bare}') logger.info('Creating new task manager for JID {}'.format(jid_bare))
logger.info('Stopping task "status" for JID {jid_bare}') logger.info('Stopping task "status" for JID {}'.format(jid_bare))
try: try:
self.task_manager[jid_bare]['status'].cancel() self.task_manager[jid_bare]['status'].cancel()
except: except:
logger.info(f'No task "status" for JID {jid_bare} (XmppStatusTask.start_task)') logger.info('No task "status" for JID {} (XmppStatusTask.start_task)'
logger.info(f'Starting tasks "status" for JID {jid_bare}') .format(jid_bare))
logger.info('Starting tasks "status" for JID {}'.format(jid_bare))
self.task_manager[jid_bare]['status'] = asyncio.create_task( self.task_manager[jid_bare]['status'] = asyncio.create_task(
XmppStatusTask.task_status(self, jid_bare)) XmppStatusTask.task_status(self, jid_bare))
@ -89,4 +90,5 @@ class XmppStatusTask:
'status' in self.task_manager[jid_bare]): 'status' in self.task_manager[jid_bare]):
self.task_manager[jid_bare]['status'].cancel() self.task_manager[jid_bare]['status'].cancel()
else: else:
logger.debug(f'No task "status" for JID {jid_bare}') logger.debug('No task "status" for JID {}'
.format(jid_bare))

36
slixfeed/xmpp/upload.py
View file

@ -6,51 +6,47 @@ Based on http_upload.py example from project slixmpp
https://codeberg.org/poezio/slixmpp/src/branch/master/examples/http_upload.py https://codeberg.org/poezio/slixmpp/src/branch/master/examples/http_upload.py
""" """
from pathlib import Path
from slixfeed.log import Logger from slixfeed.log import Logger
from slixmpp import JID
from slixmpp.exceptions import IqTimeout, IqError from slixmpp.exceptions import IqTimeout, IqError
from slixmpp.plugins.xep_0363.http_upload import HTTPError from slixmpp.plugins.xep_0363.http_upload import HTTPError
import sys
from typing import Optional
logger = Logger(__name__) logger = Logger(__name__)
# import sys # import sys
class XmppUpload: class XmppUpload:
async def start(self, jid, filename: Path, size: Optional[int] = None, async def start(self, jid, filename, domain=None):
encrypted: bool = False, domain: Optional[JID] = None):
logger.info(['Uploading file %s...', filename]) logger.info(['Uploading file %s...', filename])
try: try:
upload_file = self['xep_0363'].upload_file upload_file = self['xep_0363'].upload_file
if encrypted and not self['xep_0454']: # if self.encrypted and not self['xep_0454']:
print( # print(
'The xep_0454 module isn\'t available. ' # 'The xep_0454 module isn\'t available. '
'Ensure you have \'cryptography\' ' # 'Ensure you have \'cryptography\' '
'from extras_require installed.', # 'from extras_require installed.',
file=sys.stderr, # file=sys.stderr,
) # )
url = None # return
elif encrypted: # elif self.encrypted:
upload_file = self['xep_0454'].upload_file # upload_file = self['xep_0454'].upload_file
try: try:
url = await upload_file(filename, size, domain, timeout=10,) url = await upload_file(
filename, domain, timeout=10,
)
logger.info('Upload successful!') logger.info('Upload successful!')
logger.info(['Sending file to %s', jid]) logger.info(['Sending file to %s', jid])
except HTTPError: except HTTPError:
url = None url = ('Error: It appears that this server does not support '
'HTTP File Upload.')
logger.error('It appears that this server does not support ' logger.error('It appears that this server does not support '
'HTTP File Upload.') 'HTTP File Upload.')
# raise HTTPError( # raise HTTPError(
# "This server doesn't appear to support HTTP File Upload" # "This server doesn't appear to support HTTP File Upload"
# ) # )
except IqError as e: except IqError as e:
url = None
logger.error('Could not send message') logger.error('Could not send message')
logger.error(e) logger.error(e)
except IqTimeout as e: except IqTimeout as e:
url = None
# raise TimeoutError('Could not send message in time') # raise TimeoutError('Could not send message in time')
logger.error('Could not send message in time') logger.error('Could not send message in time')
logger.error(e) logger.error(e)

51
slixfeed/xmpp/utilities.py
View file

@ -9,13 +9,9 @@ logger = Logger(__name__)
# class XmppChat # class XmppChat
# class XmppUtility: # class XmppUtility:
class XmppUtilities: class XmppUtilities:
def get_self_alias(self, room):
"""Get self alias of a given group chat"""
jid_full = self.plugin['xep_0045'].get_our_jid_in_room(room)
alias = jid_full.split('/')[1]
return alias
async def get_chat_type(self, jid): async def get_chat_type(self, jid):
""" """
@ -64,18 +60,21 @@ class XmppUtilities:
return result return result
def is_access(self, jid, chat_type):
def is_access(self, jid_bare, jid_full, chat_type):
"""Determine access privilege""" """Determine access privilege"""
room = jid_bare = jid.bare operator = XmppUtilities.is_operator(self, jid_bare)
alias = jid.resource if operator:
if chat_type == 'groupchat': if chat_type == 'groupchat':
access = True if XmppUtilities.is_moderator(self, room, alias) else False if XmppUtilities.is_moderator(self, jid_bare, jid_full):
if access: print('Access granted to groupchat moderator ' + alias)
else:
print('Access granted to chat jid ' + jid_bare)
access = True access = True
else:
access = True
else:
access = False
return access return access
def is_operator(self, jid_bare): def is_operator(self, jid_bare):
"""Check if given JID is an operator""" """Check if given JID is an operator"""
result = False result = False
@ -86,28 +85,24 @@ class XmppUtilities:
break break
return result return result
def is_admin(self, room, alias):
"""Check if given JID is an administrator"""
affiliation = self.plugin['xep_0045'].get_jid_property(room, alias, 'affiliation')
result = True if affiliation == 'admin' else False
return result
def is_owner(self, room, alias): def is_moderator(self, jid_bare, jid_full):
"""Check if given JID is an owner"""
affiliation = self.plugin['xep_0045'].get_jid_property(room, alias, 'affiliation')
result = True if affiliation == 'owner' else False
return result
def is_moderator(self, room, alias):
"""Check if given JID is a moderator""" """Check if given JID is a moderator"""
role = self.plugin['xep_0045'].get_jid_property(room, alias, 'role') alias = jid_full[jid_full.index('/')+1:]
result = True if role == 'moderator' else False role = self.plugin['xep_0045'].get_jid_property(jid_bare, alias, 'role')
if role == 'moderator':
result = True
else:
result = False
return result return result
# NOTE Would this properly work when Alias and Local differ?
def is_member(self, jid_bare, jid_full): def is_member(self, jid_bare, jid_full):
"""Check if given JID is a member""" """Check if given JID is a member"""
alias = jid_full[jid_full.index('/')+1:] alias = jid_full[jid_full.index('/')+1:]
affiliation = self.plugin['xep_0045'].get_jid_property(jid_bare, alias, 'affiliation') affiliation = self.plugin['xep_0045'].get_jid_property(jid_bare, alias, 'affiliation')
result = True if affiliation == 'member' else False if affiliation == 'member':
result = True
else:
result = False
return result return result