thecoffemaker/Slixfeed
1
0
Fork 0
forked from sch/Slixfeed
Code Pull requests Activity

Add statistics

Browse source
This commit is contained in:
Schimon Jehudah 2023-10-04 12:37:31 +00:00
parent cf44241698
commit aa8c35d728
2 changed files with 351 additions and 316 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

323
slixfeed/__main__.py
View file

@ -43,8 +43,8 @@ import database
class Slixfeed(slixmpp.ClientXMPP): class Slixfeed(slixmpp.ClientXMPP):
""" """
Slixmpp bot that will send updates of feeds it Slixmpp news bot that will send updates
receives. from feeds it receives.
""" """
def __init__(self, jid, password): def __init__(self, jid, password):
slixmpp.ClientXMPP.__init__(self, jid, password) slixmpp.ClientXMPP.__init__(self, jid, password)
@ -66,7 +66,6 @@ class Slixfeed(slixmpp.ClientXMPP):
self.add_event_handler("disconnected", self.reconnect) self.add_event_handler("disconnected", self.reconnect)
async def start(self, event): async def start(self, event):
# print("start")
""" """
Process the session_start event. Process the session_start event.
@ -83,8 +82,6 @@ class Slixfeed(slixmpp.ClientXMPP):
await self.get_roster() await self.get_roster()
async def message(self, msg): async def message(self, msg):
# print("message")
# time.sleep(1)
""" """
Process incoming message stanzas. Be aware that this also Process incoming message stanzas. Be aware that this also
includes MUC messages and error messages. It is usually includes MUC messages and error messages. It is usually
@ -99,49 +96,34 @@ class Slixfeed(slixmpp.ClientXMPP):
if msg['type'] in ('chat', 'normal'): if msg['type'] in ('chat', 'normal'):
message = " ".join(msg['body'].split()) message = " ".join(msg['body'].split())
if message.lower().startswith('help'): if message.lower().startswith('help'):
print("COMMAND: help")
print("ACCOUNT: " + str(msg['from']))
action = print_help() action = print_help()
# NOTE: Might not need it # NOTE: Might not need it
elif message.lower().startswith('feed recent '): elif message.lower().startswith('recent '):
print("COMMAND: feed recent") action = await initdb(msg['from'].bare, database.last_entries, message[7:])
print("ACCOUNT: " + str(msg['from'])) elif message.lower().startswith('search '):
action = await initdb(msg['from'].bare, database.last_entries, message[12:]) action = await initdb( msg['from'].bare, database.search_entries, message[7:])
elif message.lower().startswith('feed search '): elif message.lower().startswith('list'):
print("COMMAND: feed search")
print("ACCOUNT: " + str(msg['from']))
action = await initdb( msg['from'].bare, database.search_entries, message[12:])
elif message.lower().startswith('feed list'):
print("COMMAND: feed list")
print("ACCOUNT: " + str(msg['from']))
action = await initdb(msg['from'].bare, database.list_subscriptions) action = await initdb(msg['from'].bare, database.list_subscriptions)
elif message.lower().startswith('feed add '): elif message.lower().startswith('add '):
print("COMMAND: feed add") action = await initdb(msg['from'].bare, add_feed, message[4:])
print("ACCOUNT: " + str(msg['from'])) elif message.lower().startswith('remove '):
action = await initdb(msg['from'].bare, add_feed, message[9:]) action = await initdb(msg['from'].bare, database.remove_feed, message[7:])
elif message.lower().startswith('feed remove '): elif message.lower().startswith('status '):
print("COMMAND: feed remove") action = await initdb(msg['from'].bare, database.toggle_status, message[7:])
print("ACCOUNT: " + str(msg['from'])) elif message.lower().startswith('unread'):
action = await initdb(msg['from'].bare, database.remove_feed, message[12:]) action = await initdb(msg['from'].bare, database.statistics)
elif message.lower().startswith('feed status '):
print("COMMAND: feed status")
print("ACCOUNT: " + str(msg['from']))
action = await initdb(msg['from'].bare, database.toggle_status, message[12:])
elif message.lower().startswith('enable'): elif message.lower().startswith('enable'):
print("COMMAND: enable")
print("ACCOUNT: " + str(msg['from']))
action = toggle_state(msg['from'].bare, True) action = toggle_state(msg['from'].bare, True)
elif message.lower().startswith('disable'): elif message.lower().startswith('disable'):
print("COMMAND: disable")
print("ACCOUNT: " + str(msg['from']))
action = toggle_state(msg['from'].bare, False) action = toggle_state(msg['from'].bare, False)
else: else:
action = 'Unknown command. Press "help" for list of commands' action = 'Unknown command. Press "help" for list of commands'
msg.reply(action).send() msg.reply(action).send()
print("COMMAND:", message)
print("ACCOUNT: " + str(msg['from']))
async def check_updates(self, event): async def check_updates(self, event):
# print("check_updates")
# time.sleep(1)
while True: while True:
print("Checking update") print("Checking update")
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
@ -156,13 +138,11 @@ class Slixfeed(slixmpp.ClientXMPP):
files = os.listdir(db_dir) files = os.listdir(db_dir)
for file in files: for file in files:
jid = file[:-3] jid = file[:-3]
print("download_updates",jid)
await initdb(jid, download_updates) await initdb(jid, download_updates)
# await asyncio.sleep(9)
await asyncio.sleep(90) await asyncio.sleep(90)
async def send_update(self, event): async def send_update(self, event):
# print("send_update")
# time.sleep(1)
while True: while True:
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
if not os.path.isdir(db_dir): if not os.path.isdir(db_dir):
@ -178,21 +158,39 @@ class Slixfeed(slixmpp.ClientXMPP):
for file in files: for file in files:
if not file.endswith('.db-jour.db'): if not file.endswith('.db-jour.db'):
jid = file[:-3] jid = file[:-3]
print("get_entry_unread",jid)
new = await initdb( new = await initdb(
jid, jid,
database.get_unread database.get_entry_unread
) )
if new: if new:
# NOTE Consider send_message msg = self.send_message(
msg = self.make_message(
mto=jid, mto=jid,
mbody=new, mbody=new,
mtype='chat' mtype='chat'
) )
msg.send() unread = await initdb(
jid,
database.get_number_of_entries_unread
)
if unread:
msg_status = ('📰 News items:', str(unread))
msg_status = ' '.join(msg_status)
else:
msg_status = '🗞 No News'
print(msg_status, 'for', jid)
# Send status message
self.send_presence(
pstatus=msg_status,
pto=jid,
#pfrom=None
)
# await asyncio.sleep(15) # await asyncio.sleep(15)
await asyncio.sleep(60 * 3) await asyncio.sleep(60 * 3)
@ -212,61 +210,53 @@ class Slixfeed(slixmpp.ClientXMPP):
for file in files: for file in files:
jid = file[:-3] jid = file[:-3]
unread = await initdb(
jid,
database.get_unread_entries_number
)
if unread:
msg_status = ('News', str(unread))
msg_status = ' '.join(msg_status)
else:
msg_status = 'No News'
print(msg_status, 'for', jid)
# NOTE Consider send_presence
sts = self.make_presence(
pstatus=msg_status,
pto=jid,
pfrom=jid,
pnick='Slixfeed'
)
sts.send()
await asyncio.sleep(60) await asyncio.sleep(60)
def print_help(): def print_help():
# print("print_help") """
# time.sleep(1) Print help manual.
"""
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n" msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
"\n" "\n"
"DESCRIPTION: \n" "DESCRIPTION: \n"
" Slixfeed is a news aggregator bot for online news feeds. \n" " Slixfeed is a news aggregator bot for online news feeds. \n"
" Supported filetypes: Atom, RDF and RSS. \n"
"\n" "\n"
"BASIC USAGE: \n" "BASIC USAGE: \n"
" enable \n" " enable \n"
" Send updates. \n" " Send updates. \n"
" disable \n" " disable \n"
" Stop sending updates. \n" " Stop sending updates. \n"
" batch N \n"
" Send N updates on ech interval. \n"
" interval N \n"
" Send an update each N minutes. \n"
" feed list \n" " feed list \n"
" List subscriptions. \n" " List subscriptions. \n"
"\n" "\n"
"EDIT OPTIONS: \n" "EDIT OPTIONS: \n"
" feed add URL \n" " add URL \n"
" Add URL to subscription list. \n" " Add URL to subscription list. \n"
" feed remove ID \n" " remove ID \n"
" Remove feed from subscription list. \n" " Remove feed from subscription list. \n"
" feed status ID \n" " status ID \n"
" Toggle update status of feed. \n" " Toggle update status of feed. \n"
"\n" "\n"
"SEARCH OPTIONS: \n" "SEARCH OPTIONS: \n"
" feed search TEXT \n" " search TEXT \n"
" Search news items by given keywords. \n" " Search news items by given keywords. \n"
" feed recent N \n" " recent N \n"
" List recent N news items (up to 50 items). \n" " List recent N news items (up to 50 items). \n"
"\n" "\n"
"STATISTICS OPTIONS: \n"
" analyses \n"
" Show report and statistics of feeds. \n"
" obsolete \n"
" List feeds that are not available. \n"
" unread \n"
" Print number of unread news items. \n"
"\n"
"BACKUP OPTIONS: \n" "BACKUP OPTIONS: \n"
" export opml \n" " export opml \n"
" Send an OPML file with your feeds. \n" " Send an OPML file with your feeds. \n"
@ -287,13 +277,10 @@ def print_help():
return msg return msg
# Function from buku # Function from jarun/buku
# https://github.com/jarun/buku
# Arun Prakash Jana (jarun) # Arun Prakash Jana (jarun)
# Dmitry Marakasov (AMDmi3) # Dmitry Marakasov (AMDmi3)
def get_default_dbdir(): def get_default_dbdir():
# print("get_default_dbdir")
# time.sleep(1)
"""Determine the directory path where dbfile will be stored. """Determine the directory path where dbfile will be stored.
If $XDG_DATA_HOME is defined, use it If $XDG_DATA_HOME is defined, use it
@ -301,10 +288,11 @@ def get_default_dbdir():
else if the platform is Windows, use %APPDATA% else if the platform is Windows, use %APPDATA%
else use the current directory. else use the current directory.
Returns :return: Path to database file.
-------
str Note
Path to database file. ----
This code was taken from the buku project.
""" """
# data_home = xdg.BaseDirectory.xdg_data_home # data_home = xdg.BaseDirectory.xdg_data_home
data_home = os.environ.get('XDG_DATA_HOME') data_home = os.environ.get('XDG_DATA_HOME')
@ -324,8 +312,13 @@ def get_default_dbdir():
# TODO Perhaps this needs to be executed # TODO Perhaps this needs to be executed
# just once per program execution # just once per program execution
async def initdb(jid, callback, message=None): async def initdb(jid, callback, message=None):
# print("initdb") """
# time.sleep(1) Callback function to instantiate action on database.
:param jid: JID (Jabber ID).
:param callback: Function name.
:param massage: Optional kwarg when a message is a part or required argument.
"""
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
if not os.path.isdir(db_dir): if not os.path.isdir(db_dir):
os.mkdir(db_dir) os.mkdir(db_dir)
@ -340,10 +333,11 @@ async def initdb(jid, callback, message=None):
# NOTE I don't think there should be "return" # NOTE I don't think there should be "return"
# because then we might stop scanning next URLs # because then we might stop scanning next URLs
async def download_updates(db_file): async def download_updates(db_file):
# print("download_updates") """
# print("db_file") Chack feeds for new entries.
# print(db_file)
# time.sleep(1) :param db_file: Database filename.
"""
urls = await database.get_subscriptions(db_file) urls = await database.get_subscriptions(db_file)
for url in urls: for url in urls:
@ -386,11 +380,9 @@ async def download_updates(db_file):
# TODO Place these couple of lines back down # TODO Place these couple of lines back down
# NOTE Need to correct the SQL statement to do so # NOTE Need to correct the SQL statement to do so
entries = feed.entries entries = feed.entries
length = len(entries) # length = len(entries)
# breakpoint()
# await database.remove_entry(db_file, source, length) # await database.remove_entry(db_file, source, length)
await database.remove_nonexistent_entries(db_file, feed, source) await database.remove_nonexistent_entries(db_file, feed, source)
# breakpoint()
new_entry = 0 new_entry = 0
for entry in entries: for entry in entries:
@ -407,23 +399,9 @@ async def download_updates(db_file):
# print('source:', source) # print('source:', source)
exist = await database.check_entry_exist(db_file, title, link) exist = await database.check_entry_exist(db_file, title, link)
# breakpoint()
# if exist:
# print("//////// OLD ////////")
# print(source)
# print('ex:',exist)
# if entry.has_key("id"):
# print('id:',entry.id)
if not exist: if not exist:
# breakpoint()
new_entry = new_entry + 1 new_entry = new_entry + 1
# print("******** NEW ********")
# print('T',title)
# if entry.has_key("date"):
# print('D',entry.date)
# print('L',link)
# print('ex',exist)
# TODO Enhance summary # TODO Enhance summary
if entry.has_key("summary"): if entry.has_key("summary"):
summary = entry.summary summary = entry.summary
@ -433,45 +411,50 @@ async def download_updates(db_file):
summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨" summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
else: else:
summary = '*** No summary ***' summary = '*** No summary ***'
#print('~~~~~~summary not in entry')
entry = (title, summary, link, source, 0); entry = (title, summary, link, source, 0);
await database.add_entry_and_set_date(db_file, source, entry) await database.add_entry_and_set_date(db_file, source, entry)
# print("### added", new_entry, "entries") # print("### added", new_entry, "entries")
async def download_feed(url): async def download_feed(url):
"""
Download content of given URL.
:param url: URL.
:return: Document or error message.
"""
# print("download_feed") # print("download_feed")
# time.sleep(1) # time.sleep(1)
timeout = aiohttp.ClientTimeout(total=10) timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
# async with aiohttp.ClientSession(trust_env=True) as session: # async with aiohttp.ClientSession(trust_env=True) as session:
try: try:
async with session.get(url, timeout=timeout) as response: async with session.get(url, timeout=timeout) as response:
status = response.status status = response.status
if response.status == 200: if response.status == 200:
try:
doc = await response.text() doc = await response.text()
# print (response.content_type) # print (response.content_type)
return [doc, status] return [doc, status]
except:
return [False, "The content of this document doesn't appear to be textual"]
else: else:
return [False, status] return [False, "HTTP Error: " + str(status)]
except aiohttp.ClientError as e: except aiohttp.ClientError as e:
print('Error', str(e)) print('Error', str(e))
return [False, "error"] return [False, "Error: " + str(e)]
except asyncio.TimeoutError as e: except asyncio.TimeoutError as e:
print('Timeout', str(e)) print('Timeout', str(e))
return [False, "timeout"] return [False, "Timeout"]
async def add_feed(db_file, url): async def add_feed(db_file, url):
# print("add_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Check whether feed exist, otherwise process it Check whether feed exist, otherwise process it.
:param db_file:
:param url: :param db_file: Database filename.
:return: string :param url: URL.
:return: Status message.
""" """
exist = await database.check_feed_exist(db_file, url) exist = await database.check_feed_exist(db_file, url)
@ -483,10 +466,10 @@ async def add_feed(db_file, url):
bozo = ("WARNING: Bozo detected. Failed to load <{}>.".format(url)) bozo = ("WARNING: Bozo detected. Failed to load <{}>.".format(url))
print(bozo) print(bozo)
try: try:
# tree = etree.fromstring(res[0]) # etree -> html # tree = etree.fromstring(res[0]) # etree is for xml
tree = html.fromstring(res[0]) tree = html.fromstring(res[0])
except: except:
return "Failed to parse {} as feed".format(url) return "Failed to parse URL <{}> as feed".format(url)
print("RSS Auto-Discovery Engaged") print("RSS Auto-Discovery Engaged")
xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]""" xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]"""
@ -518,28 +501,48 @@ async def add_feed(db_file, url):
return await add_feed(db_file, url) return await add_feed(db_file, url)
# Search for feeds by file extension and path # Search for feeds by file extension and path
paths = ["/atom", paths = [
"/app.php/feed", # phpbb
"/atom",
"/atom.php", "/atom.php",
"/atom.xml", "/atom.xml",
"/content-feeds/",
"/external.php?type=RSS2",
"/feed", # good practice
"/feed.atom",
# "/feed.json",
"/feed.php",
"/feed.rdf",
"/feed.rss",
"/feed.xml",
"/feed/atom/",
"/feeds/news_feed",
"/feeds/rss/news.xml.php",
"/forum_rss.php",
"/index.php/feed",
"/index.php?type=atom;action=.xml", #smf
"/index.php?type=rss;action=.xml", #smf
"/index.rss",
"/latest.rss",
"/news",
"/news.xml",
"/news.xml.php",
"/news/feed",
"/posts.rss", # discourse
"/rdf", "/rdf",
"/rdf.php", "/rdf.php",
"/rdf.xml", "/rdf.xml",
"/rss", "/rss",
# "/rss.json",
"/rss.php", "/rss.php",
"/rss.xml", "/rss.xml",
"/feed", "/timeline.rss",
"/feed.atom", "/xml/feed.rss",
"/feed.rdf", # "?format=atom",
"/feed.rss", # "?format=rdf",
"/feed.xml", # "?format=rss",
"/news", # "?format=xml"
"/news/feed", ]
"?format=rss",
"/feeds/news_feed",
"/content-feeds/",
"/app.php/feed", # phpBB
"/posts.rss" # Discourse
] # More paths "rss.json", "feed.json"
print("RSS Scan Mode Engaged") print("RSS Scan Mode Engaged")
feeds = {} feeds = {}
@ -551,16 +554,12 @@ async def add_feed(db_file, url):
for address in addresses: for address in addresses:
address = address.xpath('@href')[0] address = address.xpath('@href')[0]
if address.startswith('/'): if address.startswith('/'):
address = parted_url.netloc + address address = parted_url.scheme + '://' + parted_url.netloc + address
res = await download_feed(address) res = await download_feed(address)
# print(address)
if res[1] == 200: if res[1] == 200:
# print(address)
try: try:
feeds[address] = feedparser.parse(res[0])["feed"]["title"] feeds[address] = feedparser.parse(res[0])["feed"]["title"]
# print(feeds)
except: except:
# print('Not a feed')
continue continue
if len(feeds) > 1: if len(feeds) > 1:
msg = "RSS URL scan has found {} feeds:\n\n".format(len(feeds)) msg = "RSS URL scan has found {} feeds:\n\n".format(len(feeds))
@ -583,7 +582,18 @@ async def add_feed(db_file, url):
feeds = {} feeds = {}
parted_url = urlparse(url) parted_url = urlparse(url)
for path in paths: for path in paths:
# print(path) address = parted_url.scheme + '://' + parted_url.netloc + path
res = await download_feed(address)
if res[1] == 200:
# print(feedparser.parse(res[0])["feed"]["title"])
# feeds[address] = feedparser.parse(res[0])["feed"]["title"]
try:
title = feedparser.parse(res[0])["feed"]["title"]
except:
title = '*** No Title ***'
feeds[address] = title
# Check whether URL has path (i.e. not root)
if parted_url.path.split('/')[1]: if parted_url.path.split('/')[1]:
paths.extend([".atom", ".feed", ".rdf", ".rss"]) if '.rss' not in paths else -1 paths.extend([".atom", ".feed", ".rdf", ".rss"]) if '.rss' not in paths else -1
# if paths.index('.rss'): # if paths.index('.rss'):
@ -591,20 +601,13 @@ async def add_feed(db_file, url):
address = parted_url.scheme + '://' + parted_url.netloc + '/' + parted_url.path.split('/')[1] + path address = parted_url.scheme + '://' + parted_url.netloc + '/' + parted_url.path.split('/')[1] + path
res = await download_feed(address) res = await download_feed(address)
if res[1] == 200: if res[1] == 200:
# print('2res[1]') print('ATTENTION')
# print(res[1]) print(address)
# print(feedparser.parse(res[0])["feed"]["title"]) try:
feeds[address] = feedparser.parse(res[0])["feed"]["title"] title = feedparser.parse(res[0])["feed"]["title"]
# print(feeds) except:
else: title = '*** No Title ***'
address = parted_url.scheme + '://' + parted_url.netloc + path feeds[address] = title
res = await download_feed(address)
if res[1] == 200:
# print('1res[1]')
# print(res[1])
# print(feedparser.parse(res[0])["feed"]["title"])
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
# print(feeds)
if len(feeds) > 1: if len(feeds) > 1:
msg = "RSS URL discovery has found {} feeds:\n\n".format(len(feeds)) msg = "RSS URL discovery has found {} feeds:\n\n".format(len(feeds))
for feed in feeds: for feed in feeds:
@ -621,19 +624,19 @@ async def add_feed(db_file, url):
else: else:
return await database.add_feed(db_file, feed, url, res) return await database.add_feed(db_file, feed, url, res)
else: else:
return "Failed to get URL <{}>. HTTP Error {}".format(url, res[1]) return "Failed to get URL <{}>. Reason: {}".format(url, res[1])
else: else:
return "News source <{}> is already listed in the subscription list".format(url) ix = exist[0]
return "News source <{}> is already listed in the subscription list at index {}".format(url, ix)
def toggle_state(jid, state): def toggle_state(jid, state):
# print("toggle_state")
# time.sleep(1)
""" """
Set status of update Set status of update.
:param jid: jid of the user
:param state: boolean :param jid: JID (Jabber ID).
:return: :param state: True or False.
:return: Status message.
""" """
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
db_file = os.path.join(db_dir, r"{}.db".format(jid)) db_file = os.path.join(db_dir, r"{}.db".format(jid))

304
slixfeed/database.py
View file

@ -7,7 +7,6 @@ from sqlite3 import Error
import asyncio import asyncio
from datetime import date from datetime import date
import feedparser
# from eliot import start_action, to_file # from eliot import start_action, to_file
# # with start_action(action_type="list_subscriptions()", db=db_file): # # with start_action(action_type="list_subscriptions()", db=db_file):
@ -23,15 +22,12 @@ DBLOCK = asyncio.Lock()
CURSORS = {} CURSORS = {}
def create_connection(db_file): def create_connection(db_file):
# print("create_connection")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Create a database connection to the SQLite database Create a database connection to the SQLite database
specified by db_file specified by db_file.
:param db_file: database file
:return: Connection object or None :param db_file: Database filename.
:return: Connection object or None.
""" """
conn = None conn = None
try: try:
@ -43,10 +39,11 @@ def create_connection(db_file):
def create_tables(db_file): def create_tables(db_file):
# print("create_tables") """
# print("db_file") Create SQLite tables.
# print(db_file)
# time.sleep(1) :param db_file: Database filename.
"""
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
feeds_table_sql = """ feeds_table_sql = """
CREATE TABLE IF NOT EXISTS feeds ( CREATE TABLE IF NOT EXISTS feeds (
@ -68,18 +65,26 @@ def create_tables(db_file):
source text, source text,
read integer read integer
); """ ); """
# statistics_table_sql = """
# CREATE TABLE IF NOT EXISTS statistics (
# id integer PRIMARY KEY,
# title text NOT NULL,
# number integer
# ); """
c = conn.cursor() c = conn.cursor()
# c = get_cursor(db_file) # c = get_cursor(db_file)
c.execute(feeds_table_sql) c.execute(feeds_table_sql)
c.execute(entries_table_sql) c.execute(entries_table_sql)
# c.execute(statistics_table_sql)
def get_cursor(db_file): def get_cursor(db_file):
""" """
Allocate a cursor to connection per database. Allocate a cursor to connection per database.
:param db_file: database file
:return: Cursor :param db_file: Database filename.
:return: Cursor.
""" """
if db_file in CURSORS: if db_file in CURSORS:
return CURSORS[db_file] return CURSORS[db_file]
@ -91,15 +96,14 @@ def get_cursor(db_file):
async def add_feed(db_file, feed, url, res): async def add_feed(db_file, feed, url, res):
# print("add_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Add a new feed into the feeds table Add a new feed into the feeds table.
:param conn:
:param feed: :param db_file: Database filename.
:return: string :param feed: Parsed XML document.
:param url: URL.
:param res: XML document.
:return: Message.
""" """
#TODO consider async with DBLOCK #TODO consider async with DBLOCK
#conn = create_connection(db_file) #conn = create_connection(db_file)
@ -128,15 +132,12 @@ async def add_feed(db_file, feed, url, res):
async def remove_feed(db_file, ix): async def remove_feed(db_file, ix):
# print("remove_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Delete a feed by feed id Delete a feed by feed id.
:param conn:
:param id: id of the feed :param db_file: Database filename.
:return: string :param ix: Index of feed.
:return: Message.
""" """
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
async with DBLOCK: async with DBLOCK:
@ -158,16 +159,13 @@ async def remove_feed(db_file, ix):
async def check_feed_exist(db_file, url): async def check_feed_exist(db_file, url):
# print("is_feed_exist")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Check whether a feed exists Check whether a feed exists.
Query for feeds by url Query for feeds by given url.
:param conn:
:param url: :param db_file: Database filename.
:return: row :param url: URL.
:return: SQL row or None.
""" """
cur = get_cursor(db_file) cur = get_cursor(db_file)
sql = "SELECT id FROM feeds WHERE address = ?" sql = "SELECT id FROM feeds WHERE address = ?"
@ -175,11 +173,29 @@ async def check_feed_exist(db_file, url):
return cur.fetchone() return cur.fetchone()
async def get_unread_entries_number(db_file): async def get_number_of_items(db_file, str):
""" """
Check number of unread items Return number of entries or feeds.
:param db_file
:return: string :param cur: Cursor object.
:param str: "entries" or "feeds".
:return: Number of rows.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT count(id) FROM {}".format(str)
count = cur.execute(sql)
count = cur.fetchone()[0]
return count
async def get_number_of_entries_unread(db_file):
"""
Return number of unread items.
:param db_file: Database filename.
:param cur: Cursor object.
:return: Number of rows.
""" """
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
cur = conn.cursor() cur = conn.cursor()
@ -189,22 +205,16 @@ async def get_unread_entries_number(db_file):
return count return count
async def get_entry_unread(db_file):
async def get_unread(db_file):
# print("get_unread")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Check read status of entry Check read status of entry.
:param conn:
:param id: id of the entry :param db_file: Database filename.
:return: string :return: News item as message.
""" """
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
entry = []
cur = conn.cursor() cur = conn.cursor()
# cur = get_cursor(db_file) entry = []
sql = "SELECT id FROM entries WHERE read = 0" sql = "SELECT id FROM entries WHERE read = 0"
ix = cur.execute(sql).fetchone() ix = cur.execute(sql).fetchone()
if ix is None: if ix is None:
@ -222,36 +232,72 @@ async def get_unread(db_file):
cur.execute(sql, (ix,)) cur.execute(sql, (ix,))
link = cur.fetchone()[0] link = cur.fetchone()[0]
entry.append(link) entry.append(link)
entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2]) entry = "{}\n\n{}\n\n{}".format(entry[0], entry[1], entry[2])
# print(entry) # print(entry)
async with DBLOCK: async with DBLOCK:
await mark_as_read(cur, ix) await mark_as_read(cur, ix)
# async with DBLOCK:
# await update_statistics(db_file)
return entry return entry
async def mark_as_read(cur, ix): async def mark_as_read(cur, ix):
# print("mark_as_read", ix)
# time.sleep(1)
""" """
Set read status of entry Set read status of entry.
:param cur:
:param ix: index of the entry :param cur: Cursor object.
:param ix: Index of entry.
""" """
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?" sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
cur.execute(sql, (ix,)) cur.execute(sql, (ix,))
async def statistics(db_file):
"""
Return table statistics.
:param db_file: Database filename.
:return: News item as message.
"""
feeds = await get_number_of_items(db_file, 'feeds')
entries = await get_number_of_items(db_file, 'entries')
unread_entries = await get_number_of_entries_unread(db_file)
return "You have {} unread news items out of {} from {} news sources.".format(unread_entries, entries, feeds)
async def update_statistics(cur):
"""
Update table statistics.
:param cur: Cursor object.
"""
stat_dict = {}
stat_dict["feeds"] = await get_number_of_items(cur, 'feeds')
stat_dict["entries"] = await get_number_of_items(cur, 'entries')
stat_dict["unread"] = await get_number_of_entries_unread(cur=cur)
for i in stat_dict:
sql = "SELECT id FROM statistics WHERE title = ?"
cur.execute(sql, (i,))
if cur.fetchone():
sql = "UPDATE statistics SET number = :num WHERE title = :title"
cur.execute(sql, {"title": i, "num": stat_dict[i]})
else:
sql = "SELECT count(id) FROM statistics"
count = cur.execute(sql)
count = cur.fetchone()[0]
ix = count + 1
sql = "INSERT INTO statistics VALUES(?,?,?)"
cur.execute(sql, (ix, i, stat_dict[i]))
# TODO mark_all_read for entries of feed # TODO mark_all_read for entries of feed
async def toggle_status(db_file, ix): async def toggle_status(db_file, ix):
# print("toggle_status")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Set status of feed Toggle status of feed.
:param conn:
:param id: id of the feed :param db_file: Database filename.
:return: string :param ix: Index of entry.
:return: Message
""" """
async with DBLOCK: async with DBLOCK:
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
@ -279,12 +325,11 @@ async def toggle_status(db_file, ix):
async def set_date(cur, url): async def set_date(cur, url):
# print("set_date")
# time.sleep(1)
""" """
Set last update date of feed Set last update date of feed.
:param url: url of the feed
:return: :param cur: Cursor object.
:param url: URL.
""" """
today = date.today() today = date.today()
sql = "UPDATE feeds SET updated = :today WHERE address = :url" sql = "UPDATE feeds SET updated = :today WHERE address = :url"
@ -293,6 +338,9 @@ async def set_date(cur, url):
async def add_entry_and_set_date(db_file, source, entry): async def add_entry_and_set_date(db_file, source, entry):
"""
TODO
"""
async with DBLOCK: async with DBLOCK:
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
cur = conn.cursor() cur = conn.cursor()
@ -301,6 +349,9 @@ async def add_entry_and_set_date(db_file, source, entry):
async def update_source_status(db_file, status, source): async def update_source_status(db_file, status, source):
"""
TODO
"""
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url" sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
async with DBLOCK: async with DBLOCK:
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
@ -309,6 +360,9 @@ async def update_source_status(db_file, status, source):
async def update_source_validity(db_file, source, valid): async def update_source_validity(db_file, source, valid):
"""
TODO
"""
sql = "UPDATE feeds SET valid = :validity WHERE address = :url" sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
async with DBLOCK: async with DBLOCK:
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
@ -317,29 +371,25 @@ async def update_source_validity(db_file, source, valid):
async def add_entry(cur, entry): async def add_entry(cur, entry):
# print("add_entry")
# time.sleep(1)
""" """
Add a new entry into the entries table Add a new entry into the entries table.
:param conn:
:param cur: Cursor object.
:param entry: :param entry:
:return:
""" """
sql = """ INSERT INTO entries(title,summary,link,source,read) sql = """ INSERT INTO entries(title,summary,link,source,read)
VALUES(?,?,?,?,?) """ VALUES(?,?,?,?,?) """
# cur = conn.cursor()
cur.execute(sql, entry) cur.execute(sql, entry)
# This function doesn't work as expected with bbs and wiki feeds # This function doesn't work as expected with bbs and wiki feeds
async def remove_entry(db_file, source, length): async def remove_entry(db_file, source, length):
# print("remove_entry")
# time.sleep(1)
""" """
Maintain list of entries Maintain list of entries equal to feed.
Check the number returned by feed and delete Check the number returned by feed and delete
existing entries up to the same returned amount existing entries up to the same returned amount.
:param conn:
:param db_file: Database filename.
:param source: :param source:
:param length: :param length:
:return: :return:
@ -364,18 +414,17 @@ async def remove_entry(db_file, source, length):
ORDER BY id ORDER BY id
ASC LIMIT :limit)""" ASC LIMIT :limit)"""
cur.execute(sql, {"source": source, "limit": limit}) cur.execute(sql, {"source": source, "limit": limit})
print('### removed', limit, 'from', source)
async def remove_nonexistent_entries(db_file, feed, source): async def remove_nonexistent_entries(db_file, feed, source):
""" """
Remove entries that don't exist in feed' Remove entries that don't exist in a given parsed feed.
Check the entries returned from feed and delete Check the entries returned from feed and delete non
non existing entries existing entries
:param conn:
:param source: :param db_file: Database filename.
:param length: :param feed: URL of parsed feed.
:return: :param source: URL of associated feed.
""" """
async with DBLOCK: async with DBLOCK:
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
@ -420,12 +469,11 @@ async def remove_nonexistent_entries(db_file, feed, source):
async def get_subscriptions(db_file): async def get_subscriptions(db_file):
# print("get_subscriptions")
# time.sleep(1)
""" """
Query feeds Query table feeds.
:param conn:
:return: rows (tuple) :param db_file: Database filename.
:return: List of feeds.
""" """
with create_connection(db_file) as conn: with create_connection(db_file) as conn:
cur = conn.cursor() cur = conn.cursor()
@ -435,17 +483,12 @@ async def get_subscriptions(db_file):
async def list_subscriptions(db_file): async def list_subscriptions(db_file):
# print("list_subscriptions")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Query feeds Query table feeds and list items.
:param conn:
:return: rows (string) :param db_file: Database filename.
:return: List of feeds.
""" """
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file) cur = get_cursor(db_file)
sql = "SELECT name, address, updated, id, enabled FROM feeds" sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql) results = cur.execute(sql)
@ -464,28 +507,23 @@ async def list_subscriptions(db_file):
"To add feed, send a message as follows: \n" "To add feed, send a message as follows: \n"
"feed add URL \n" "feed add URL \n"
"Example: \n" "Example: \n"
"feed add https://reclaimthenet.org/feed/") "add https://reclaimthenet.org/feed/")
return msg return msg
async def last_entries(db_file, num): async def last_entries(db_file, num):
# print("last_entries")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Query feeds Query entries
:param conn:
:param num: integer :param db_file: Database filename.
:return: rows (string) :param num: Number
:return: List of recent N entries
""" """
num = int(num) num = int(num)
if num > 50: if num > 50:
num = 50 num = 50
elif num < 1: elif num < 1:
num = 1 num = 1
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file) cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num" sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
results = cur.execute(sql, (num,)) results = cur.execute(sql, (num,))
@ -498,21 +536,16 @@ async def last_entries(db_file, num):
async def search_entries(db_file, query): async def search_entries(db_file, query):
# print("search_entries")
# print("db_file")
# print(db_file)
# time.sleep(1)
""" """
Query feeds Query entries
:param conn:
:param query: string :param db_file: Database filename.
:return: rows (string) :param query: Search query
:return: Entries with specified keywords
""" """
if len(query) < 2: if len(query) < 2:
return "Please enter at least 2 characters to search" return "Please enter at least 2 characters to search"
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file) cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50" sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
results = cur.execute(sql, [f'%{query}%']) results = cur.execute(sql, [f'%{query}%'])
@ -530,15 +563,14 @@ async def search_entries(db_file, query):
async def check_entry_exist(db_file, title, link): async def check_entry_exist(db_file, title, link):
# print("check_entry")
# time.sleep(1)
""" """
Check whether an entry exists Check whether an entry exists.
Query entries by title and link Query entries by title and link.
:param conn:
:param link: :param db_file: Database filename.
:param title: :param link: Entry URL.
:return: row :param title: Entry title.
:return: SQL row or None.
""" """
cur = get_cursor(db_file) cur = get_cursor(db_file)
sql = "SELECT id FROM entries WHERE title = :title and link = :link" sql = "SELECT id FROM entries WHERE title = :title and link = :link"