sch/Slixfeed
3
1
Fork 1
Code Issues 1 Pull requests Actions Packages Projects Releases Wiki Activity

Add user agent setting. Add command reset (mark as read). Fix error with command recent. Fix error with command stats. Thanks roughnecks for reporting these issues.

Browse source
This commit is contained in:
Schimon Jehudah 2023-12-18 15:29:32 +00:00
parent d4b0d08480
commit 19bc626eb1
7 changed files with 181 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
slixfeed/__main__.py
View file

@ -59,6 +59,8 @@ TODO
Akkoma, HubZilla, Mastodon, Misskey, Pixelfed, Pleroma, Soapbox.
16) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger
17) See project offpunk/offblocklist.py
"""

10
slixfeed/confighandler.py
View file

@ -18,7 +18,7 @@ import os
from random import randrange
import yaml
async def get_value_default(key):
async def get_value_default(key, section):
"""
Get settings default value.
@ -39,12 +39,12 @@ async def get_value_default(key):
config_dir = '/usr/share/slixfeed/'
config_file = os.path.join(config_dir, r"settings.ini")
config.read(config_file)
if config.has_section("Settings"):
result = config["Settings"][key]
if config.has_section(section):
result = config[section][key]
return result
async def get_list(key):
async def get_list(key, file):
"""
Get settings default value.
@ -61,7 +61,7 @@ async def get_list(key):
config_dir = filehandler.get_default_confdir()
if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/'
config_file = os.path.join(config_dir, r"lists.yaml")
config_file = os.path.join(config_dir, file)
with open(config_file) as defaults:
default = yaml.safe_load(defaults)
result = default[key]

9
slixfeed/datahandler.py
View file

@ -18,13 +18,13 @@ from aiohttp import ClientError, ClientSession, ClientTimeout
from asyncio import TimeoutError
from asyncio.exceptions import IncompleteReadError
from bs4 import BeautifulSoup
from confighandler import get_list, get_value_default
from email.utils import parseaddr
from feedparser import parse
from http.client import IncompleteRead
from lxml import html
from datetimehandler import now, rfc2822_to_iso8601
from urlhandler import complete_url, join_url, trim_url
from confighandler import get_list
from listhandler import is_listed
import sqlitehandler as sqlite
from urllib import error
@ -527,8 +527,13 @@ async def download_feed(url):
msg: list or str
Document or error message.
"""
try:
user_agent = await get_value_default("user-agent", "Network")
except:
user_agent = "Slixfeed/0.1"
timeout = ClientTimeout(total=10)
async with ClientSession() as session:
headers = {user_agent}
async with ClientSession(headers=headers) as session:
# async with ClientSession(trust_env=True) as session:
try:
async with session.get(url, timeout=timeout) as response:

32
slixfeed/sqlitehandler.py
View file

@ -621,7 +621,7 @@ async def statistics(db_file):
# """.format(unread_entries, entries, feeds)
with create_connection(db_file) as conn:
cur = conn.cursor()
keys = []
vals = []
for key in [
"archive",
"interval",
@ -633,7 +633,12 @@ async def statistics(db_file):
"FROM settings "
"WHERE key = ?"
)
keys.extend([cur.execute(sql, (key,)).fetchone()[0]])
try:
val = cur.execute(sql, (key,)).fetchone()[0]
except:
print("Error for key:", key)
val = "none"
vals.extend([val])
msg = (
"```"
"\nSTATISTICS\n"
@ -648,10 +653,10 @@ async def statistics(db_file):
).format(
unread_entries, entries + archive,
active_feeds, feeds,
keys[0],
keys[1],
keys[2],
keys[3]
vals[0],
vals[1],
vals[2],
vals[3]
)
return msg
@ -1184,7 +1189,7 @@ async def list_feeds(db_file):
async def last_entries(db_file, num):
"""
Query entries
Query entries.
Parameters
----------
@ -1198,11 +1203,6 @@ async def last_entries(db_file, num):
titles_list : str
List of recent N entries as message.
"""
num = int(num)
if num > 50:
num = 50
elif num < 1:
num = 1
cur = get_cursor(db_file)
# sql = (
# "SELECT title, link "
@ -1211,11 +1211,11 @@ async def last_entries(db_file, num):
# "LIMIT :num"
# )
sql = (
"SELECT title, link "
"SELECT title, link, timestamp "
"FROM entries "
"WHERE read = 0 "
"UNION ALL "
"SELECT title, link "
"SELECT title, link, timestamp "
"FROM archive "
"WHERE read = 0 "
"ORDER BY timestamp DESC "
@ -1506,7 +1506,7 @@ async def set_settings_value_default(cur, key):
)
cur.execute(sql, (key,))
if not cur.fetchone():
val = await config.get_value_default(key)
val = await config.get_value_default(key, "Settings")
sql = (
"INSERT "
"INTO settings(key,value) "
@ -1623,7 +1623,7 @@ async def set_filters_value_default(cur, key):
)
cur.execute(sql, (key,))
if not cur.fetchone():
val = await config.get_list(key)
val = await config.get_list(key, "lists.yaml")
val = ",".join(val)
sql = (
"INSERT "

4
slixfeed/taskhandler.py
View file

@ -44,6 +44,7 @@ import logging
import os
import slixmpp
import confighandler as config
from datahandler import download_updates
from datetimehandler import current_time
from filehandler import initdb, get_default_dbdir
@ -394,7 +395,8 @@ async def check_updates(jid):
while True:
# print(await current_time(), "> CHCK UPDATE",jid)
await initdb(jid, download_updates)
await asyncio.sleep(60 * 90)
val = await config.get_value_default("check", "Settings")
await asyncio.sleep(60 * float(val))
# Schedule to call this function again in 90 minutes
# loop.call_at(
# loop.time() + 60 * 90,

7
slixfeed/urlhandler.py
View file

@ -5,7 +5,12 @@
TODO
1) ActivityPub URL revealer activitypub_to_http
1) ActivityPub URL revealer activitypub_to_http.
2) Remove tracking queries.
3) Redirect to Invidious, Librarian, Nitter, ProxiTok etc.
because they provide RSS.
"""

226
slixfeed/xmpphandler.py
View file

@ -835,17 +835,20 @@ class Slixfeed(slixmpp.ClientXMPP):
key = message[:7]
val = message[8:]
if val:
if int(val) > 500:
action = "Value may not be greater than 500."
else:
await initdb(
jid,
sqlite.set_settings_value,
[key, val]
)
action = (
"Maximum archived items has been set to {}."
).format(val)
try:
if int(val) > 500:
action = "Value may not be greater than 500."
else:
await initdb(
jid,
sqlite.set_settings_value,
[key, val]
)
action = (
"Maximum archived items has been set to {}."
).format(val)
except:
action = "Enter a numeric value only."
else:
action = "Missing value."
case _ if message_lowercase.startswith("deny +"):
@ -909,7 +912,7 @@ class Slixfeed(slixmpp.ClientXMPP):
["status"]
)
task = (
"📫️ Processing request to fetch data from {}"
"📫️ Processing request to fetch data from {} ..."
).format(url)
process_task_message(self, jid, task)
action = await initdb(
@ -1024,20 +1027,24 @@ class Slixfeed(slixmpp.ClientXMPP):
key = message[:6]
val = message[7:]
if val:
await initdb(
jid,
sqlite.set_settings_value,
[key, val]
)
if val == 0:
action = (
"Summary length limit is disabled."
try:
val = int(val)
await initdb(
jid,
sqlite.set_settings_value,
[key, val]
)
else:
action = (
"Summary maximum length "
"is set to {} characters."
).format(val)
if val == 0:
action = (
"Summary length limit is disabled."
)
else:
action = (
"Summary maximum length "
"is set to {} characters."
).format(val)
except:
action = "Enter a numeric value only."
else:
action = "Missing value."
# case _ if message_lowercase.startswith("mastership"):
@ -1112,17 +1119,21 @@ class Slixfeed(slixmpp.ClientXMPP):
key = message[:7]
val = message[8:]
if val:
# action = (
# "Every update will contain {} news items."
# ).format(action)
await initdb(
jid,
sqlite.set_settings_value,
[key, val]
)
action = (
"Next update will contain {} news items."
).format(val)
try:
val = int(val)
# action = (
# "Every update will contain {} news items."
# ).format(action)
await initdb(
jid,
sqlite.set_settings_value,
[key, val]
)
action = (
"Next update will contain {} news items."
).format(val)
except:
action = "Enter a numeric value only."
else:
action = "Missing value."
case "random":
@ -1132,7 +1143,7 @@ class Slixfeed(slixmpp.ClientXMPP):
data = data.split()
url = data[0]
task = (
"📫️ Processing request to fetch data from {}"
"📫️ Processing request to fetch data from {} ..."
).format(url)
process_task_message(self, jid, task)
await tasker.clean_tasks_xmpp(
@ -1167,13 +1178,21 @@ class Slixfeed(slixmpp.ClientXMPP):
case _ if message_lowercase.startswith("recent"):
num = message[7:]
if num:
action = await initdb(
jid,
sqlite.last_entries,
num
)
try:
num = int(num)
if num < 1 or num > 50:
action = "Value must be ranged from 1 to 50."
else:
action = await initdb(
jid,
sqlite.last_entries,
num
)
except:
action = "Enter a numeric value only."
else:
action = "Missing value."
# NOTE Should people be asked for numeric value?
case _ if message_lowercase.startswith("remove"):
ix = message[7:]
if ix:
@ -1200,6 +1219,37 @@ class Slixfeed(slixmpp.ClientXMPP):
)
else:
action = "Missing feed ID."
case _ if message_lowercase.startswith("reset"):
source = message[6:]
await tasker.clean_tasks_xmpp(
jid,
["status"]
)
task = (
"📫️ Marking entries as read..."
)
process_task_message(self, jid, task)
if source:
await initdb(
jid,
sqlite.mark_source_as_read,
source
)
action = (
"All entries of {} have been "
"marked as read.".format(source)
)
else:
await initdb(
jid,
sqlite.mark_all_as_read
)
action = "All entries have been marked as read."
await tasker.start_tasks_xmpp(
self,
jid,
["status"]
)
case _ if message_lowercase.startswith("search"):
query = message[7:]
if query:
@ -1430,35 +1480,41 @@ def print_help():
" For more information, visit https://xmpp.org/software/\n"
"\n"
"BASIC USAGE\n"
" URL\n"
" Add URL to subscription list.\n"
" add URL TITLE\n"
" Add URL to subscription list (without validity check).\n"
" join MUC\n"
" <url>\n"
" Add <url> to subscription list.\n"
" add <url> TITLE\n"
" Add <url> to subscription list (without validity check).\n"
" join <muc>\n"
" Join specified groupchat.\n"
" read URL\n"
" Display most recent 20 titles of given URL.\n"
" read URL N\n"
" Display specified entry number from given URL.\n"
" read <url>\n"
" Display most recent 20 titles of given <url>.\n"
" read <url> <n>\n"
" Display specified entry number from given <url>.\n"
"\n"
"MESSAGE OPTIONS\n"
" interval N\n"
" Set interval update to every N minutes.\n"
" length\n"
" Set maximum length of news item description. (0 for no limit)\n"
"CUSTOM ACTIONS\n"
" new\n"
" Send only new items of newly added feeds.\n"
" next N\n"
" Send N next updates.\n"
" old\n"
" Send all items of newly added feeds.\n"
" quantum N\n"
" Set N amount of updates per interval.\n"
" next N\n"
" Send N next updates.\n"
" reset\n"
" Mark all entries as read and remove all archived entries\n"
" reset <url>\n"
" Mark entries of <url> as read and remove all archived entries of <url>.\n"
" start\n"
" Enable bot and send updates.\n"
" stop\n"
" Disable bot and stop updates.\n"
"\n"
"MESSAGE OPTIONS\n"
" interval <num>\n"
" Set interval update to every <num> minutes.\n"
" length\n"
" Set maximum length of news item description. (0 for no limit)\n"
" quantum <num>\n"
" Set <num> amount of updates per interval.\n"
"\n"
"GROUPCHAT OPTIONS\n"
" ! (command initiation)\n"
" Use exclamation mark to initiate an actionable command.\n"
@ -1486,20 +1542,20 @@ def print_help():
# " Reset deny list.\n"
"\n"
"EDIT OPTIONS\n"
" remove ID\n"
" Remove feed from subscription list.\n"
" status ID\n"
" Toggle update status of feed.\n"
" remove <id>\n"
" Remove feed of <id> from subscription list.\n"
" status <id>\n"
" Toggle update status of feed of <id>.\n"
"\n"
"SEARCH OPTIONS\n"
" feeds\n"
" List all subscriptions.\n"
" feeds TEXT\n"
" Search subscriptions by given keywords.\n"
" search TEXT\n"
" Search news items by given keywords.\n"
" recent N\n"
" List recent N news items (up to 50 items).\n"
" feeds <text>\n"
" Search subscriptions by given <text>.\n"
" search <text>\n"
" Search news items by given <text>.\n"
" recent <num>\n"
" List recent <num> news items (up to 50 items).\n"
"\n"
# "STATISTICS OPTIONS\n"
# " analyses\n"
@ -1551,29 +1607,31 @@ def print_cmd():
"```"
"\n"
"! : Use exclamation mark to initiate an actionable command (groupchats only).\n"
"<MUC> : Join specified groupchat.\n"
"<URL> : Add URL to subscription list.\n"
"add <URL> <TITLE> : Add URL to subscription list (without validity check).\n"
"<muc> : Join specified groupchat.\n"
"<url> : Add <url> to subscription list.\n"
"add <url> <title> : Add <url> to subscription list (without validity check).\n"
"allow + : Add keywords to allow (comma separates).\n"
"allow - : Delete keywords from allow list (comma separates).\n"
"deny + : Keywords to block (comma separates).\n"
"deny - : Delete keywords from deny list (comma separates).\n"
"feeds : List all subscriptions.\n"
"feeds <TEXT> : Search subscriptions by given keywords.\n"
"interval N : Set interval update to every N minutes.\n"
"join <MUC> : Join specified groupchat.\n"
"feeds <text> : Search subscriptions by given <text>.\n"
"interval <n> : Set interval update to every <n> minutes.\n"
"join <muc> : Join specified groupchat.\n"
"length : Set maximum length of news item description. (0 for no limit)\n"
"new : Send only new items of newly added feeds.\n"
"next N : Send N next updates.\n"
"next <n> : Send <n> next updates.\n"
"old : Send all items of newly added feeds.\n"
"quantum N : Set N amount of updates per interval.\n"
"read <URL> : Display most recent 20 titles of given URL.\n"
"read URL N : Display specified entry number from given URL.\n"
"recent N : List recent N news items (up to 50 items).\n"
"remove <ID> : Remove feed from subscription list.\n"
"search <TEXT> : Search news items by given keywords.\n"
"quantum <n> : Set <n> amount of updates per interval.\n"
"read <url> : Display most recent 20 titles of given <url>.\n"
"read <url> <n> : Display specified entry number from given <url>.\n"
"recent <n> : List recent <n> news items (up to 50 items).\n"
"reset : Mark all entries as read.\n"
"reset <url> : Mark entries of <url> as read.\n"
"remove <id> : Remove feed from subscription list.\n"
"search <text> : Search news items by given <text>.\n"
"start : Enable bot and send updates.\n"
"status <ID> : Toggle update status of feed.\n"
"status <id> : Toggle update status of feed.\n"
"stop : Disable bot and stop updates.\n"
"```"
)