thecoffemaker/Slixfeed
1
0
Fork 0
forked from sch/Slixfeed
Code Pull requests Activity

Add user agent setting. Add command reset (mark as read). Fix error with command recent. Fix error with command stats. Thanks roughnecks for reporting these issues.

Browse source
This commit is contained in:
Schimon Jehudah 2023-12-18 15:29:32 +00:00
parent d4b0d08480
commit 19bc626eb1
7 changed files with 181 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
slixfeed/__main__.py
View file

@ -60,6 +60,8 @@ TODO
16) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger 16) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger
17) See project offpunk/offblocklist.py
""" """
# vars and their meanings: # vars and their meanings:

10
slixfeed/confighandler.py
View file

@ -18,7 +18,7 @@ import os
from random import randrange from random import randrange
import yaml import yaml
async def get_value_default(key): async def get_value_default(key, section):
""" """
Get settings default value. Get settings default value.
@ -39,12 +39,12 @@ async def get_value_default(key):
config_dir = '/usr/share/slixfeed/' config_dir = '/usr/share/slixfeed/'
config_file = os.path.join(config_dir, r"settings.ini") config_file = os.path.join(config_dir, r"settings.ini")
config.read(config_file) config.read(config_file)
if config.has_section("Settings"): if config.has_section(section):
result = config["Settings"][key] result = config[section][key]
return result return result
async def get_list(key): async def get_list(key, file):
""" """
Get settings default value. Get settings default value.
@ -61,7 +61,7 @@ async def get_list(key):
config_dir = filehandler.get_default_confdir() config_dir = filehandler.get_default_confdir()
if not os.path.isdir(config_dir): if not os.path.isdir(config_dir):
config_dir = '/usr/share/slixfeed/' config_dir = '/usr/share/slixfeed/'
config_file = os.path.join(config_dir, r"lists.yaml") config_file = os.path.join(config_dir, file)
with open(config_file) as defaults: with open(config_file) as defaults:
default = yaml.safe_load(defaults) default = yaml.safe_load(defaults)
result = default[key] result = default[key]

9
slixfeed/datahandler.py
View file

@ -18,13 +18,13 @@ from aiohttp import ClientError, ClientSession, ClientTimeout
from asyncio import TimeoutError from asyncio import TimeoutError
from asyncio.exceptions import IncompleteReadError from asyncio.exceptions import IncompleteReadError
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from confighandler import get_list, get_value_default
from email.utils import parseaddr from email.utils import parseaddr
from feedparser import parse from feedparser import parse
from http.client import IncompleteRead from http.client import IncompleteRead
from lxml import html from lxml import html
from datetimehandler import now, rfc2822_to_iso8601 from datetimehandler import now, rfc2822_to_iso8601
from urlhandler import complete_url, join_url, trim_url from urlhandler import complete_url, join_url, trim_url
from confighandler import get_list
from listhandler import is_listed from listhandler import is_listed
import sqlitehandler as sqlite import sqlitehandler as sqlite
from urllib import error from urllib import error
@ -527,8 +527,13 @@ async def download_feed(url):
msg: list or str msg: list or str
Document or error message. Document or error message.
""" """
try:
user_agent = await get_value_default("user-agent", "Network")
except:
user_agent = "Slixfeed/0.1"
timeout = ClientTimeout(total=10) timeout = ClientTimeout(total=10)
async with ClientSession() as session: headers = {user_agent}
async with ClientSession(headers=headers) as session:
# async with ClientSession(trust_env=True) as session: # async with ClientSession(trust_env=True) as session:
try: try:
async with session.get(url, timeout=timeout) as response: async with session.get(url, timeout=timeout) as response:

32
slixfeed/sqlitehandler.py
View file

@ -621,7 +621,7 @@ async def statistics(db_file):
# """.format(unread_entries, entries, feeds) # """.format(unread_entries, entries, feeds)
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
cur = conn.cursor() cur = conn.cursor()
keys = [] vals = []
for key in [ for key in [
"archive", "archive",
"interval", "interval",
@ -633,7 +633,12 @@ async def statistics(db_file):
"FROM settings " "FROM settings "
"WHERE key = ?" "WHERE key = ?"
) )
keys.extend([cur.execute(sql, (key,)).fetchone()[0]]) try:
val = cur.execute(sql, (key,)).fetchone()[0]
except:
print("Error for key:", key)
val = "none"
vals.extend([val])
msg = ( msg = (
"```" "```"
"\nSTATISTICS\n" "\nSTATISTICS\n"
@ -648,10 +653,10 @@ async def statistics(db_file):
).format( ).format(
unread_entries, entries + archive, unread_entries, entries + archive,
active_feeds, feeds, active_feeds, feeds,
keys[0], vals[0],
keys[1], vals[1],
keys[2], vals[2],
keys[3] vals[3]
) )
return msg return msg
@ -1184,7 +1189,7 @@ async def list_feeds(db_file):
async def last_entries(db_file, num): async def last_entries(db_file, num):
""" """
Query entries Query entries.
Parameters Parameters
---------- ----------
@ -1198,11 +1203,6 @@ async def last_entries(db_file, num):
titles_list : str titles_list : str
List of recent N entries as message. List of recent N entries as message.
""" """
num = int(num)
if num > 50:
num = 50
elif num < 1:
num = 1
cur = get_cursor(db_file) cur = get_cursor(db_file)
# sql = ( # sql = (
# "SELECT title, link " # "SELECT title, link "
@ -1211,11 +1211,11 @@ async def last_entries(db_file, num):
# "LIMIT :num" # "LIMIT :num"
# ) # )
sql = ( sql = (
"SELECT title, link " "SELECT title, link, timestamp "
"FROM entries " "FROM entries "
"WHERE read = 0 " "WHERE read = 0 "
"UNION ALL " "UNION ALL "
"SELECT title, link " "SELECT title, link, timestamp "
"FROM archive " "FROM archive "
"WHERE read = 0 " "WHERE read = 0 "
"ORDER BY timestamp DESC " "ORDER BY timestamp DESC "
@ -1506,7 +1506,7 @@ async def set_settings_value_default(cur, key):
) )
cur.execute(sql, (key,)) cur.execute(sql, (key,))
if not cur.fetchone(): if not cur.fetchone():
val = await config.get_value_default(key) val = await config.get_value_default(key, "Settings")
sql = ( sql = (
"INSERT " "INSERT "
"INTO settings(key,value) " "INTO settings(key,value) "
@ -1623,7 +1623,7 @@ async def set_filters_value_default(cur, key):
) )
cur.execute(sql, (key,)) cur.execute(sql, (key,))
if not cur.fetchone(): if not cur.fetchone():
val = await config.get_list(key) val = await config.get_list(key, "lists.yaml")
val = ",".join(val) val = ",".join(val)
sql = ( sql = (
"INSERT " "INSERT "

4
slixfeed/taskhandler.py
View file

@ -44,6 +44,7 @@ import logging
import os import os
import slixmpp import slixmpp
import confighandler as config
from datahandler import download_updates from datahandler import download_updates
from datetimehandler import current_time from datetimehandler import current_time
from filehandler import initdb, get_default_dbdir from filehandler import initdb, get_default_dbdir
@ -394,7 +395,8 @@ async def check_updates(jid):
while True: while True:
# print(await current_time(), "> CHCK UPDATE",jid) # print(await current_time(), "> CHCK UPDATE",jid)
await initdb(jid, download_updates) await initdb(jid, download_updates)
await asyncio.sleep(60 * 90) val = await config.get_value_default("check", "Settings")
await asyncio.sleep(60 * float(val))
# Schedule to call this function again in 90 minutes # Schedule to call this function again in 90 minutes
# loop.call_at( # loop.call_at(
# loop.time() + 60 * 90, # loop.time() + 60 * 90,

7
slixfeed/urlhandler.py
View file

@ -5,7 +5,12 @@
TODO TODO
1) ActivityPub URL revealer activitypub_to_http 1) ActivityPub URL revealer activitypub_to_http.
2) Remove tracking queries.
3) Redirect to Invidious, Librarian, Nitter, ProxiTok etc.
because they provide RSS.
""" """

226
slixfeed/xmpphandler.py
View file

@ -835,17 +835,20 @@ class Slixfeed(slixmpp.ClientXMPP):
key = message[:7] key = message[:7]
val = message[8:] val = message[8:]
if val: if val:
if int(val) > 500: try:
action = "Value may not be greater than 500." if int(val) > 500:
else: action = "Value may not be greater than 500."
await initdb( else:
jid, await initdb(
sqlite.set_settings_value, jid,
[key, val] sqlite.set_settings_value,
) [key, val]
action = ( )
"Maximum archived items has been set to {}." action = (
).format(val) "Maximum archived items has been set to {}."
).format(val)
except:
action = "Enter a numeric value only."
else: else:
action = "Missing value." action = "Missing value."
case _ if message_lowercase.startswith("deny +"): case _ if message_lowercase.startswith("deny +"):
@ -909,7 +912,7 @@ class Slixfeed(slixmpp.ClientXMPP):
["status"] ["status"]
) )
task = ( task = (
"📫️ Processing request to fetch data from {}" "📫️ Processing request to fetch data from {} ..."
).format(url) ).format(url)
process_task_message(self, jid, task) process_task_message(self, jid, task)
action = await initdb( action = await initdb(
@ -1024,20 +1027,24 @@ class Slixfeed(slixmpp.ClientXMPP):
key = message[:6] key = message[:6]
val = message[7:] val = message[7:]
if val: if val:
await initdb( try:
jid, val = int(val)
sqlite.set_settings_value, await initdb(
[key, val] jid,
) sqlite.set_settings_value,
if val == 0: [key, val]
action = (
"Summary length limit is disabled."
) )
else: if val == 0:
action = ( action = (
"Summary maximum length " "Summary length limit is disabled."
"is set to {} characters." )
).format(val) else:
action = (
"Summary maximum length "
"is set to {} characters."
).format(val)
except:
action = "Enter a numeric value only."
else: else:
action = "Missing value." action = "Missing value."
# case _ if message_lowercase.startswith("mastership"): # case _ if message_lowercase.startswith("mastership"):
@ -1112,17 +1119,21 @@ class Slixfeed(slixmpp.ClientXMPP):
key = message[:7] key = message[:7]
val = message[8:] val = message[8:]
if val: if val:
# action = ( try:
# "Every update will contain {} news items." val = int(val)
# ).format(action) # action = (
await initdb( # "Every update will contain {} news items."
jid, # ).format(action)
sqlite.set_settings_value, await initdb(
[key, val] jid,
) sqlite.set_settings_value,
action = ( [key, val]
"Next update will contain {} news items." )
).format(val) action = (
"Next update will contain {} news items."
).format(val)
except:
action = "Enter a numeric value only."
else: else:
action = "Missing value." action = "Missing value."
case "random": case "random":
@ -1132,7 +1143,7 @@ class Slixfeed(slixmpp.ClientXMPP):
data = data.split() data = data.split()
url = data[0] url = data[0]
task = ( task = (
"📫️ Processing request to fetch data from {}" "📫️ Processing request to fetch data from {} ..."
).format(url) ).format(url)
process_task_message(self, jid, task) process_task_message(self, jid, task)
await tasker.clean_tasks_xmpp( await tasker.clean_tasks_xmpp(
@ -1167,13 +1178,21 @@ class Slixfeed(slixmpp.ClientXMPP):
case _ if message_lowercase.startswith("recent"): case _ if message_lowercase.startswith("recent"):
num = message[7:] num = message[7:]
if num: if num:
action = await initdb( try:
jid, num = int(num)
sqlite.last_entries, if num < 1 or num > 50:
num action = "Value must be ranged from 1 to 50."
) else:
action = await initdb(
jid,
sqlite.last_entries,
num
)
except:
action = "Enter a numeric value only."
else: else:
action = "Missing value." action = "Missing value."
# NOTE Should people be asked for numeric value?
case _ if message_lowercase.startswith("remove"): case _ if message_lowercase.startswith("remove"):
ix = message[7:] ix = message[7:]
if ix: if ix:
@ -1200,6 +1219,37 @@ class Slixfeed(slixmpp.ClientXMPP):
) )
else: else:
action = "Missing feed ID." action = "Missing feed ID."
case _ if message_lowercase.startswith("reset"):
source = message[6:]
await tasker.clean_tasks_xmpp(
jid,
["status"]
)
task = (
"📫️ Marking entries as read..."
)
process_task_message(self, jid, task)
if source:
await initdb(
jid,
sqlite.mark_source_as_read,
source
)
action = (
"All entries of {} have been "
"marked as read.".format(source)
)
else:
await initdb(
jid,
sqlite.mark_all_as_read
)
action = "All entries have been marked as read."
await tasker.start_tasks_xmpp(
self,
jid,
["status"]
)
case _ if message_lowercase.startswith("search"): case _ if message_lowercase.startswith("search"):
query = message[7:] query = message[7:]
if query: if query:
@ -1430,35 +1480,41 @@ def print_help():
" For more information, visit https://xmpp.org/software/\n" " For more information, visit https://xmpp.org/software/\n"
"\n" "\n"
"BASIC USAGE\n" "BASIC USAGE\n"
" URL\n" " <url>\n"
" Add URL to subscription list.\n" " Add <url> to subscription list.\n"
" add URL TITLE\n" " add <url> TITLE\n"
" Add URL to subscription list (without validity check).\n" " Add <url> to subscription list (without validity check).\n"
" join MUC\n" " join <muc>\n"
" Join specified groupchat.\n" " Join specified groupchat.\n"
" read URL\n" " read <url>\n"
" Display most recent 20 titles of given URL.\n" " Display most recent 20 titles of given <url>.\n"
" read URL N\n" " read <url> <n>\n"
" Display specified entry number from given URL.\n" " Display specified entry number from given <url>.\n"
"\n" "\n"
"MESSAGE OPTIONS\n" "CUSTOM ACTIONS\n"
" interval N\n"
" Set interval update to every N minutes.\n"
" length\n"
" Set maximum length of news item description. (0 for no limit)\n"
" new\n" " new\n"
" Send only new items of newly added feeds.\n" " Send only new items of newly added feeds.\n"
" next N\n"
" Send N next updates.\n"
" old\n" " old\n"
" Send all items of newly added feeds.\n" " Send all items of newly added feeds.\n"
" quantum N\n" " next N\n"
" Set N amount of updates per interval.\n" " Send N next updates.\n"
" reset\n"
" Mark all entries as read and remove all archived entries\n"
" reset <url>\n"
" Mark entries of <url> as read and remove all archived entries of <url>.\n"
" start\n" " start\n"
" Enable bot and send updates.\n" " Enable bot and send updates.\n"
" stop\n" " stop\n"
" Disable bot and stop updates.\n" " Disable bot and stop updates.\n"
"\n" "\n"
"MESSAGE OPTIONS\n"
" interval <num>\n"
" Set interval update to every <num> minutes.\n"
" length\n"
" Set maximum length of news item description. (0 for no limit)\n"
" quantum <num>\n"
" Set <num> amount of updates per interval.\n"
"\n"
"GROUPCHAT OPTIONS\n" "GROUPCHAT OPTIONS\n"
" ! (command initiation)\n" " ! (command initiation)\n"
" Use exclamation mark to initiate an actionable command.\n" " Use exclamation mark to initiate an actionable command.\n"
@ -1486,20 +1542,20 @@ def print_help():
# " Reset deny list.\n" # " Reset deny list.\n"
"\n" "\n"
"EDIT OPTIONS\n" "EDIT OPTIONS\n"
" remove ID\n" " remove <id>\n"
" Remove feed from subscription list.\n" " Remove feed of <id> from subscription list.\n"
" status ID\n" " status <id>\n"
" Toggle update status of feed.\n" " Toggle update status of feed of <id>.\n"
"\n" "\n"
"SEARCH OPTIONS\n" "SEARCH OPTIONS\n"
" feeds\n" " feeds\n"
" List all subscriptions.\n" " List all subscriptions.\n"
" feeds TEXT\n" " feeds <text>\n"
" Search subscriptions by given keywords.\n" " Search subscriptions by given <text>.\n"
" search TEXT\n" " search <text>\n"
" Search news items by given keywords.\n" " Search news items by given <text>.\n"
" recent N\n" " recent <num>\n"
" List recent N news items (up to 50 items).\n" " List recent <num> news items (up to 50 items).\n"
"\n" "\n"
# "STATISTICS OPTIONS\n" # "STATISTICS OPTIONS\n"
# " analyses\n" # " analyses\n"
@ -1551,29 +1607,31 @@ def print_cmd():
"```" "```"
"\n" "\n"
"! : Use exclamation mark to initiate an actionable command (groupchats only).\n" "! : Use exclamation mark to initiate an actionable command (groupchats only).\n"
"<MUC> : Join specified groupchat.\n" "<muc> : Join specified groupchat.\n"
"<URL> : Add URL to subscription list.\n" "<url> : Add <url> to subscription list.\n"
"add <URL> <TITLE> : Add URL to subscription list (without validity check).\n" "add <url> <title> : Add <url> to subscription list (without validity check).\n"
"allow + : Add keywords to allow (comma separates).\n" "allow + : Add keywords to allow (comma separates).\n"
"allow - : Delete keywords from allow list (comma separates).\n" "allow - : Delete keywords from allow list (comma separates).\n"
"deny + : Keywords to block (comma separates).\n" "deny + : Keywords to block (comma separates).\n"
"deny - : Delete keywords from deny list (comma separates).\n" "deny - : Delete keywords from deny list (comma separates).\n"
"feeds : List all subscriptions.\n" "feeds : List all subscriptions.\n"
"feeds <TEXT> : Search subscriptions by given keywords.\n" "feeds <text> : Search subscriptions by given <text>.\n"
"interval N : Set interval update to every N minutes.\n" "interval <n> : Set interval update to every <n> minutes.\n"
"join <MUC> : Join specified groupchat.\n" "join <muc> : Join specified groupchat.\n"
"length : Set maximum length of news item description. (0 for no limit)\n" "length : Set maximum length of news item description. (0 for no limit)\n"
"new : Send only new items of newly added feeds.\n" "new : Send only new items of newly added feeds.\n"
"next N : Send N next updates.\n" "next <n> : Send <n> next updates.\n"
"old : Send all items of newly added feeds.\n" "old : Send all items of newly added feeds.\n"
"quantum N : Set N amount of updates per interval.\n" "quantum <n> : Set <n> amount of updates per interval.\n"
"read <URL> : Display most recent 20 titles of given URL.\n" "read <url> : Display most recent 20 titles of given <url>.\n"
"read URL N : Display specified entry number from given URL.\n" "read <url> <n> : Display specified entry number from given <url>.\n"
"recent N : List recent N news items (up to 50 items).\n" "recent <n> : List recent <n> news items (up to 50 items).\n"
"remove <ID> : Remove feed from subscription list.\n" "reset : Mark all entries as read.\n"
"search <TEXT> : Search news items by given keywords.\n" "reset <url> : Mark entries of <url> as read.\n"
"remove <id> : Remove feed from subscription list.\n"
"search <text> : Search news items by given <text>.\n"
"start : Enable bot and send updates.\n" "start : Enable bot and send updates.\n"
"status <ID> : Toggle update status of feed.\n" "status <id> : Toggle update status of feed.\n"
"stop : Disable bot and stop updates.\n" "stop : Disable bot and stop updates.\n"
"```" "```"
) )