diff --git a/slixfeed/__main__.py b/slixfeed/__main__.py index cc3a0ee..7f95840 100644 --- a/slixfeed/__main__.py +++ b/slixfeed/__main__.py @@ -59,6 +59,8 @@ TODO Akkoma, HubZilla, Mastodon, Misskey, Pixelfed, Pleroma, Soapbox. 16) Brand: News Broker, Newsman, Newsdealer, Laura Harbinger + +17) See project offpunk/offblocklist.py """ diff --git a/slixfeed/confighandler.py b/slixfeed/confighandler.py index 2236855..fa49aad 100644 --- a/slixfeed/confighandler.py +++ b/slixfeed/confighandler.py @@ -18,7 +18,7 @@ import os from random import randrange import yaml -async def get_value_default(key): +async def get_value_default(key, section): """ Get settings default value. @@ -39,12 +39,12 @@ async def get_value_default(key): config_dir = '/usr/share/slixfeed/' config_file = os.path.join(config_dir, r"settings.ini") config.read(config_file) - if config.has_section("Settings"): - result = config["Settings"][key] + if config.has_section(section): + result = config[section][key] return result -async def get_list(key): +async def get_list(key, file): """ Get settings default value. @@ -61,7 +61,7 @@ async def get_list(key): config_dir = filehandler.get_default_confdir() if not os.path.isdir(config_dir): config_dir = '/usr/share/slixfeed/' - config_file = os.path.join(config_dir, r"lists.yaml") + config_file = os.path.join(config_dir, file) with open(config_file) as defaults: default = yaml.safe_load(defaults) result = default[key] diff --git a/slixfeed/datahandler.py b/slixfeed/datahandler.py index da82a95..4a289a6 100644 --- a/slixfeed/datahandler.py +++ b/slixfeed/datahandler.py @@ -18,13 +18,13 @@ from aiohttp import ClientError, ClientSession, ClientTimeout from asyncio import TimeoutError from asyncio.exceptions import IncompleteReadError from bs4 import BeautifulSoup +from confighandler import get_list, get_value_default from email.utils import parseaddr from feedparser import parse from http.client import IncompleteRead from lxml import html from datetimehandler import now, rfc2822_to_iso8601 from urlhandler import complete_url, join_url, trim_url -from confighandler import get_list from listhandler import is_listed import sqlitehandler as sqlite from urllib import error @@ -527,8 +527,13 @@ async def download_feed(url): msg: list or str Document or error message. """ + try: + user_agent = await get_value_default("user-agent", "Network") + except: + user_agent = "Slixfeed/0.1" timeout = ClientTimeout(total=10) - async with ClientSession() as session: + headers = {user_agent} + async with ClientSession(headers=headers) as session: # async with ClientSession(trust_env=True) as session: try: async with session.get(url, timeout=timeout) as response: diff --git a/slixfeed/sqlitehandler.py b/slixfeed/sqlitehandler.py index beb700a..d3a3abc 100644 --- a/slixfeed/sqlitehandler.py +++ b/slixfeed/sqlitehandler.py @@ -621,7 +621,7 @@ async def statistics(db_file): # """.format(unread_entries, entries, feeds) with create_connection(db_file) as conn: cur = conn.cursor() - keys = [] + vals = [] for key in [ "archive", "interval", @@ -633,7 +633,12 @@ async def statistics(db_file): "FROM settings " "WHERE key = ?" ) - keys.extend([cur.execute(sql, (key,)).fetchone()[0]]) + try: + val = cur.execute(sql, (key,)).fetchone()[0] + except: + print("Error for key:", key) + val = "none" + vals.extend([val]) msg = ( "```" "\nSTATISTICS\n" @@ -648,10 +653,10 @@ async def statistics(db_file): ).format( unread_entries, entries + archive, active_feeds, feeds, - keys[0], - keys[1], - keys[2], - keys[3] + vals[0], + vals[1], + vals[2], + vals[3] ) return msg @@ -1184,7 +1189,7 @@ async def list_feeds(db_file): async def last_entries(db_file, num): """ - Query entries + Query entries. Parameters ---------- @@ -1198,11 +1203,6 @@ async def last_entries(db_file, num): titles_list : str List of recent N entries as message. """ - num = int(num) - if num > 50: - num = 50 - elif num < 1: - num = 1 cur = get_cursor(db_file) # sql = ( # "SELECT title, link " @@ -1211,11 +1211,11 @@ async def last_entries(db_file, num): # "LIMIT :num" # ) sql = ( - "SELECT title, link " + "SELECT title, link, timestamp " "FROM entries " "WHERE read = 0 " "UNION ALL " - "SELECT title, link " + "SELECT title, link, timestamp " "FROM archive " "WHERE read = 0 " "ORDER BY timestamp DESC " @@ -1506,7 +1506,7 @@ async def set_settings_value_default(cur, key): ) cur.execute(sql, (key,)) if not cur.fetchone(): - val = await config.get_value_default(key) + val = await config.get_value_default(key, "Settings") sql = ( "INSERT " "INTO settings(key,value) " @@ -1623,7 +1623,7 @@ async def set_filters_value_default(cur, key): ) cur.execute(sql, (key,)) if not cur.fetchone(): - val = await config.get_list(key) + val = await config.get_list(key, "lists.yaml") val = ",".join(val) sql = ( "INSERT " diff --git a/slixfeed/taskhandler.py b/slixfeed/taskhandler.py index b48dd82..9fe519c 100644 --- a/slixfeed/taskhandler.py +++ b/slixfeed/taskhandler.py @@ -44,6 +44,7 @@ import logging import os import slixmpp +import confighandler as config from datahandler import download_updates from datetimehandler import current_time from filehandler import initdb, get_default_dbdir @@ -394,7 +395,8 @@ async def check_updates(jid): while True: # print(await current_time(), "> CHCK UPDATE",jid) await initdb(jid, download_updates) - await asyncio.sleep(60 * 90) + val = await config.get_value_default("check", "Settings") + await asyncio.sleep(60 * float(val)) # Schedule to call this function again in 90 minutes # loop.call_at( # loop.time() + 60 * 90, diff --git a/slixfeed/urlhandler.py b/slixfeed/urlhandler.py index 29db1a0..c46e7c6 100644 --- a/slixfeed/urlhandler.py +++ b/slixfeed/urlhandler.py @@ -5,7 +5,12 @@ TODO -1) ActivityPub URL revealer activitypub_to_http +1) ActivityPub URL revealer activitypub_to_http. + +2) Remove tracking queries. + +3) Redirect to Invidious, Librarian, Nitter, ProxiTok etc. + because they provide RSS. """ diff --git a/slixfeed/xmpphandler.py b/slixfeed/xmpphandler.py index 3b4cd91..0052a73 100644 --- a/slixfeed/xmpphandler.py +++ b/slixfeed/xmpphandler.py @@ -835,17 +835,20 @@ class Slixfeed(slixmpp.ClientXMPP): key = message[:7] val = message[8:] if val: - if int(val) > 500: - action = "Value may not be greater than 500." - else: - await initdb( - jid, - sqlite.set_settings_value, - [key, val] - ) - action = ( - "Maximum archived items has been set to {}." - ).format(val) + try: + if int(val) > 500: + action = "Value may not be greater than 500." + else: + await initdb( + jid, + sqlite.set_settings_value, + [key, val] + ) + action = ( + "Maximum archived items has been set to {}." + ).format(val) + except: + action = "Enter a numeric value only." else: action = "Missing value." case _ if message_lowercase.startswith("deny +"): @@ -909,7 +912,7 @@ class Slixfeed(slixmpp.ClientXMPP): ["status"] ) task = ( - "📫️ Processing request to fetch data from {}" + "📫️ Processing request to fetch data from {} ..." ).format(url) process_task_message(self, jid, task) action = await initdb( @@ -1024,20 +1027,24 @@ class Slixfeed(slixmpp.ClientXMPP): key = message[:6] val = message[7:] if val: - await initdb( - jid, - sqlite.set_settings_value, - [key, val] - ) - if val == 0: - action = ( - "Summary length limit is disabled." + try: + val = int(val) + await initdb( + jid, + sqlite.set_settings_value, + [key, val] ) - else: - action = ( - "Summary maximum length " - "is set to {} characters." - ).format(val) + if val == 0: + action = ( + "Summary length limit is disabled." + ) + else: + action = ( + "Summary maximum length " + "is set to {} characters." + ).format(val) + except: + action = "Enter a numeric value only." else: action = "Missing value." # case _ if message_lowercase.startswith("mastership"): @@ -1112,17 +1119,21 @@ class Slixfeed(slixmpp.ClientXMPP): key = message[:7] val = message[8:] if val: - # action = ( - # "Every update will contain {} news items." - # ).format(action) - await initdb( - jid, - sqlite.set_settings_value, - [key, val] - ) - action = ( - "Next update will contain {} news items." - ).format(val) + try: + val = int(val) + # action = ( + # "Every update will contain {} news items." + # ).format(action) + await initdb( + jid, + sqlite.set_settings_value, + [key, val] + ) + action = ( + "Next update will contain {} news items." + ).format(val) + except: + action = "Enter a numeric value only." else: action = "Missing value." case "random": @@ -1132,7 +1143,7 @@ class Slixfeed(slixmpp.ClientXMPP): data = data.split() url = data[0] task = ( - "📫️ Processing request to fetch data from {}" + "📫️ Processing request to fetch data from {} ..." ).format(url) process_task_message(self, jid, task) await tasker.clean_tasks_xmpp( @@ -1167,13 +1178,21 @@ class Slixfeed(slixmpp.ClientXMPP): case _ if message_lowercase.startswith("recent"): num = message[7:] if num: - action = await initdb( - jid, - sqlite.last_entries, - num - ) + try: + num = int(num) + if num < 1 or num > 50: + action = "Value must be ranged from 1 to 50." + else: + action = await initdb( + jid, + sqlite.last_entries, + num + ) + except: + action = "Enter a numeric value only." else: action = "Missing value." + # NOTE Should people be asked for numeric value? case _ if message_lowercase.startswith("remove"): ix = message[7:] if ix: @@ -1200,6 +1219,37 @@ class Slixfeed(slixmpp.ClientXMPP): ) else: action = "Missing feed ID." + case _ if message_lowercase.startswith("reset"): + source = message[6:] + await tasker.clean_tasks_xmpp( + jid, + ["status"] + ) + task = ( + "📫️ Marking entries as read..." + ) + process_task_message(self, jid, task) + if source: + await initdb( + jid, + sqlite.mark_source_as_read, + source + ) + action = ( + "All entries of {} have been " + "marked as read.".format(source) + ) + else: + await initdb( + jid, + sqlite.mark_all_as_read + ) + action = "All entries have been marked as read." + await tasker.start_tasks_xmpp( + self, + jid, + ["status"] + ) case _ if message_lowercase.startswith("search"): query = message[7:] if query: @@ -1430,35 +1480,41 @@ def print_help(): " For more information, visit https://xmpp.org/software/\n" "\n" "BASIC USAGE\n" - " URL\n" - " Add URL to subscription list.\n" - " add URL TITLE\n" - " Add URL to subscription list (without validity check).\n" - " join MUC\n" + " \n" + " Add to subscription list.\n" + " add TITLE\n" + " Add to subscription list (without validity check).\n" + " join \n" " Join specified groupchat.\n" - " read URL\n" - " Display most recent 20 titles of given URL.\n" - " read URL N\n" - " Display specified entry number from given URL.\n" + " read \n" + " Display most recent 20 titles of given .\n" + " read \n" + " Display specified entry number from given .\n" "\n" - "MESSAGE OPTIONS\n" - " interval N\n" - " Set interval update to every N minutes.\n" - " length\n" - " Set maximum length of news item description. (0 for no limit)\n" + "CUSTOM ACTIONS\n" " new\n" " Send only new items of newly added feeds.\n" - " next N\n" - " Send N next updates.\n" " old\n" " Send all items of newly added feeds.\n" - " quantum N\n" - " Set N amount of updates per interval.\n" + " next N\n" + " Send N next updates.\n" + " reset\n" + " Mark all entries as read and remove all archived entries\n" + " reset \n" + " Mark entries of as read and remove all archived entries of .\n" " start\n" " Enable bot and send updates.\n" " stop\n" " Disable bot and stop updates.\n" "\n" + "MESSAGE OPTIONS\n" + " interval \n" + " Set interval update to every minutes.\n" + " length\n" + " Set maximum length of news item description. (0 for no limit)\n" + " quantum \n" + " Set amount of updates per interval.\n" + "\n" "GROUPCHAT OPTIONS\n" " ! (command initiation)\n" " Use exclamation mark to initiate an actionable command.\n" @@ -1486,20 +1542,20 @@ def print_help(): # " Reset deny list.\n" "\n" "EDIT OPTIONS\n" - " remove ID\n" - " Remove feed from subscription list.\n" - " status ID\n" - " Toggle update status of feed.\n" + " remove \n" + " Remove feed of from subscription list.\n" + " status \n" + " Toggle update status of feed of .\n" "\n" "SEARCH OPTIONS\n" " feeds\n" " List all subscriptions.\n" - " feeds TEXT\n" - " Search subscriptions by given keywords.\n" - " search TEXT\n" - " Search news items by given keywords.\n" - " recent N\n" - " List recent N news items (up to 50 items).\n" + " feeds \n" + " Search subscriptions by given .\n" + " search \n" + " Search news items by given .\n" + " recent \n" + " List recent news items (up to 50 items).\n" "\n" # "STATISTICS OPTIONS\n" # " analyses\n" @@ -1551,29 +1607,31 @@ def print_cmd(): "```" "\n" "! : Use exclamation mark to initiate an actionable command (groupchats only).\n" - " : Join specified groupchat.\n" - " : Add URL to subscription list.\n" - "add : Add URL to subscription list (without validity check).\n" + "<muc> : Join specified groupchat.\n" + "<url> : Add <url> to subscription list.\n" + "add <url> <title> : Add <url> to subscription list (without validity check).\n" "allow + : Add keywords to allow (comma separates).\n" "allow - : Delete keywords from allow list (comma separates).\n" "deny + : Keywords to block (comma separates).\n" "deny - : Delete keywords from deny list (comma separates).\n" "feeds : List all subscriptions.\n" - "feeds <TEXT> : Search subscriptions by given keywords.\n" - "interval N : Set interval update to every N minutes.\n" - "join <MUC> : Join specified groupchat.\n" + "feeds <text> : Search subscriptions by given <text>.\n" + "interval <n> : Set interval update to every <n> minutes.\n" + "join <muc> : Join specified groupchat.\n" "length : Set maximum length of news item description. (0 for no limit)\n" "new : Send only new items of newly added feeds.\n" - "next N : Send N next updates.\n" + "next <n> : Send <n> next updates.\n" "old : Send all items of newly added feeds.\n" - "quantum N : Set N amount of updates per interval.\n" - "read <URL> : Display most recent 20 titles of given URL.\n" - "read URL N : Display specified entry number from given URL.\n" - "recent N : List recent N news items (up to 50 items).\n" - "remove <ID> : Remove feed from subscription list.\n" - "search <TEXT> : Search news items by given keywords.\n" + "quantum <n> : Set <n> amount of updates per interval.\n" + "read <url> : Display most recent 20 titles of given <url>.\n" + "read <url> <n> : Display specified entry number from given <url>.\n" + "recent <n> : List recent <n> news items (up to 50 items).\n" + "reset : Mark all entries as read.\n" + "reset <url> : Mark entries of <url> as read.\n" + "remove <id> : Remove feed from subscription list.\n" + "search <text> : Search news items by given <text>.\n" "start : Enable bot and send updates.\n" - "status <ID> : Toggle update status of feed.\n" + "status <id> : Toggle update status of feed.\n" "stop : Disable bot and stop updates.\n" "```" )