sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

Add statistics

Browse source
This commit is contained in:
Schimon Jehudah 2023-10-04 12:37:31 +00:00
parent cf44241698
commit aa8c35d728
2 changed files with 351 additions and 316 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

323
slixfeed/__main__.py
View file

@ -43,8 +43,8 @@ import database
class Slixfeed(slixmpp.ClientXMPP):
"""
Slixmpp bot that will send updates of feeds it
receives.
Slixmpp news bot that will send updates
from feeds it receives.
"""
def __init__(self, jid, password):
slixmpp.ClientXMPP.__init__(self, jid, password)
@ -66,7 +66,6 @@ class Slixfeed(slixmpp.ClientXMPP):
self.add_event_handler("disconnected", self.reconnect)
async def start(self, event):
# print("start")
"""
Process the session_start event.
@ -83,8 +82,6 @@ class Slixfeed(slixmpp.ClientXMPP):
await self.get_roster()
async def message(self, msg):
# print("message")
# time.sleep(1)
"""
Process incoming message stanzas. Be aware that this also
includes MUC messages and error messages. It is usually
@ -99,49 +96,34 @@ class Slixfeed(slixmpp.ClientXMPP):
if msg['type'] in ('chat', 'normal'):
message = " ".join(msg['body'].split())
if message.lower().startswith('help'):
print("COMMAND: help")
print("ACCOUNT: " + str(msg['from']))
action = print_help()
# NOTE: Might not need it
elif message.lower().startswith('feed recent '):
print("COMMAND: feed recent")
print("ACCOUNT: " + str(msg['from']))
action = await initdb(msg['from'].bare, database.last_entries, message[12:])
elif message.lower().startswith('feed search '):
print("COMMAND: feed search")
print("ACCOUNT: " + str(msg['from']))
action = await initdb( msg['from'].bare, database.search_entries, message[12:])
elif message.lower().startswith('feed list'):
print("COMMAND: feed list")
print("ACCOUNT: " + str(msg['from']))
elif message.lower().startswith('recent '):
action = await initdb(msg['from'].bare, database.last_entries, message[7:])
elif message.lower().startswith('search '):
action = await initdb( msg['from'].bare, database.search_entries, message[7:])
elif message.lower().startswith('list'):
action = await initdb(msg['from'].bare, database.list_subscriptions)
elif message.lower().startswith('feed add '):
print("COMMAND: feed add")
print("ACCOUNT: " + str(msg['from']))
action = await initdb(msg['from'].bare, add_feed, message[9:])
elif message.lower().startswith('feed remove '):
print("COMMAND: feed remove")
print("ACCOUNT: " + str(msg['from']))
action = await initdb(msg['from'].bare, database.remove_feed, message[12:])
elif message.lower().startswith('feed status '):
print("COMMAND: feed status")
print("ACCOUNT: " + str(msg['from']))
action = await initdb(msg['from'].bare, database.toggle_status, message[12:])
elif message.lower().startswith('add '):
action = await initdb(msg['from'].bare, add_feed, message[4:])
elif message.lower().startswith('remove '):
action = await initdb(msg['from'].bare, database.remove_feed, message[7:])
elif message.lower().startswith('status '):
action = await initdb(msg['from'].bare, database.toggle_status, message[7:])
elif message.lower().startswith('unread'):
action = await initdb(msg['from'].bare, database.statistics)
elif message.lower().startswith('enable'):
print("COMMAND: enable")
print("ACCOUNT: " + str(msg['from']))
action = toggle_state(msg['from'].bare, True)
elif message.lower().startswith('disable'):
print("COMMAND: disable")
print("ACCOUNT: " + str(msg['from']))
action = toggle_state(msg['from'].bare, False)
else:
action = 'Unknown command. Press "help" for list of commands'
msg.reply(action).send()
print("COMMAND:", message)
print("ACCOUNT: " + str(msg['from']))
async def check_updates(self, event):
# print("check_updates")
# time.sleep(1)
while True:
print("Checking update")
db_dir = get_default_dbdir()
@ -156,13 +138,11 @@ class Slixfeed(slixmpp.ClientXMPP):
files = os.listdir(db_dir)
for file in files:
jid = file[:-3]
print("download_updates",jid)
await initdb(jid, download_updates)
# await asyncio.sleep(9)
await asyncio.sleep(90)
async def send_update(self, event):
# print("send_update")
# time.sleep(1)
while True:
db_dir = get_default_dbdir()
if not os.path.isdir(db_dir):
@ -178,21 +158,39 @@ class Slixfeed(slixmpp.ClientXMPP):
for file in files:
if not file.endswith('.db-jour.db'):
jid = file[:-3]
print("get_entry_unread",jid)
new = await initdb(
jid,
database.get_unread
database.get_entry_unread
)
if new:
# NOTE Consider send_message
msg = self.make_message(
msg = self.send_message(
mto=jid,
mbody=new,
mtype='chat'
)
msg.send()
unread = await initdb(
jid,
database.get_number_of_entries_unread
)
if unread:
msg_status = ('📰 News items:', str(unread))
msg_status = ' '.join(msg_status)
else:
msg_status = '🗞 No News'
print(msg_status, 'for', jid)
# Send status message
self.send_presence(
pstatus=msg_status,
pto=jid,
#pfrom=None
)
# await asyncio.sleep(15)
await asyncio.sleep(60 * 3)
@ -212,61 +210,53 @@ class Slixfeed(slixmpp.ClientXMPP):
for file in files:
jid = file[:-3]
unread = await initdb(
jid,
database.get_unread_entries_number
)
if unread:
msg_status = ('News', str(unread))
msg_status = ' '.join(msg_status)
else:
msg_status = 'No News'
print(msg_status, 'for', jid)
# NOTE Consider send_presence
sts = self.make_presence(
pstatus=msg_status,
pto=jid,
pfrom=jid,
pnick='Slixfeed'
)
sts.send()
await asyncio.sleep(60)
def print_help():
# print("print_help")
# time.sleep(1)
"""
Print help manual.
"""
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
"\n"
"DESCRIPTION: \n"
" Slixfeed is a news aggregator bot for online news feeds. \n"
" Supported filetypes: Atom, RDF and RSS. \n"
"\n"
"BASIC USAGE: \n"
" enable \n"
" Send updates. \n"
" disable \n"
" Stop sending updates. \n"
" batch N \n"
" Send N updates on ech interval. \n"
" interval N \n"
" Send an update each N minutes. \n"
" feed list \n"
" List subscriptions. \n"
"\n"
"EDIT OPTIONS: \n"
" feed add URL \n"
" add URL \n"
" Add URL to subscription list. \n"
" feed remove ID \n"
" remove ID \n"
" Remove feed from subscription list. \n"
" feed status ID \n"
" status ID \n"
" Toggle update status of feed. \n"
"\n"
"SEARCH OPTIONS: \n"
" feed search TEXT \n"
" search TEXT \n"
" Search news items by given keywords. \n"
" feed recent N \n"
" recent N \n"
" List recent N news items (up to 50 items). \n"
"\n"
"STATISTICS OPTIONS: \n"
" analyses \n"
" Show report and statistics of feeds. \n"
" obsolete \n"
" List feeds that are not available. \n"
" unread \n"
" Print number of unread news items. \n"
"\n"
"BACKUP OPTIONS: \n"
" export opml \n"
" Send an OPML file with your feeds. \n"
@ -287,13 +277,10 @@ def print_help():
return msg
# Function from buku
# https://github.com/jarun/buku
# Function from jarun/buku
# Arun Prakash Jana (jarun)
# Dmitry Marakasov (AMDmi3)
def get_default_dbdir():
# print("get_default_dbdir")
# time.sleep(1)
"""Determine the directory path where dbfile will be stored.
If $XDG_DATA_HOME is defined, use it
@ -301,10 +288,11 @@ def get_default_dbdir():
else if the platform is Windows, use %APPDATA%
else use the current directory.
Returns
-------
str
Path to database file.
:return: Path to database file.
Note
----
This code was taken from the buku project.
"""
# data_home = xdg.BaseDirectory.xdg_data_home
data_home = os.environ.get('XDG_DATA_HOME')
@ -324,8 +312,13 @@ def get_default_dbdir():
# TODO Perhaps this needs to be executed
# just once per program execution
async def initdb(jid, callback, message=None):
# print("initdb")
# time.sleep(1)
"""
Callback function to instantiate action on database.
:param jid: JID (Jabber ID).
:param callback: Function name.
:param massage: Optional kwarg when a message is a part or required argument.
"""
db_dir = get_default_dbdir()
if not os.path.isdir(db_dir):
os.mkdir(db_dir)
@ -340,10 +333,11 @@ async def initdb(jid, callback, message=None):
# NOTE I don't think there should be "return"
# because then we might stop scanning next URLs
async def download_updates(db_file):
# print("download_updates")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Chack feeds for new entries.
:param db_file: Database filename.
"""
urls = await database.get_subscriptions(db_file)
for url in urls:
@ -386,11 +380,9 @@ async def download_updates(db_file):
# TODO Place these couple of lines back down
# NOTE Need to correct the SQL statement to do so
entries = feed.entries
length = len(entries)
# breakpoint()
# length = len(entries)
# await database.remove_entry(db_file, source, length)
await database.remove_nonexistent_entries(db_file, feed, source)
# breakpoint()
new_entry = 0
for entry in entries:
@ -407,23 +399,9 @@ async def download_updates(db_file):
# print('source:', source)
exist = await database.check_entry_exist(db_file, title, link)
# breakpoint()
# if exist:
# print("//////// OLD ////////")
# print(source)
# print('ex:',exist)
# if entry.has_key("id"):
# print('id:',entry.id)
if not exist:
# breakpoint()
new_entry = new_entry + 1
# print("******** NEW ********")
# print('T',title)
# if entry.has_key("date"):
# print('D',entry.date)
# print('L',link)
# print('ex',exist)
# TODO Enhance summary
if entry.has_key("summary"):
summary = entry.summary
@ -433,45 +411,50 @@ async def download_updates(db_file):
summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
else:
summary = '*** No summary ***'
#print('~~~~~~summary not in entry')
entry = (title, summary, link, source, 0);
await database.add_entry_and_set_date(db_file, source, entry)
# print("### added", new_entry, "entries")
async def download_feed(url):
"""
Download content of given URL.
:param url: URL.
:return: Document or error message.
"""
# print("download_feed")
# time.sleep(1)
timeout = aiohttp.ClientTimeout(total=10)
async with aiohttp.ClientSession() as session:
# async with aiohttp.ClientSession(trust_env=True) as session:
# async with aiohttp.ClientSession(trust_env=True) as session:
try:
async with session.get(url, timeout=timeout) as response:
status = response.status
if response.status == 200:
try:
doc = await response.text()
# print (response.content_type)
return [doc, status]
except:
return [False, "The content of this document doesn't appear to be textual"]
else:
return [False, status]
return [False, "HTTP Error: " + str(status)]
except aiohttp.ClientError as e:
print('Error', str(e))
return [False, "error"]
return [False, "Error: " + str(e)]
except asyncio.TimeoutError as e:
print('Timeout', str(e))
return [False, "timeout"]
return [False, "Timeout"]
async def add_feed(db_file, url):
# print("add_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Check whether feed exist, otherwise process it
:param db_file:
:param url:
:return: string
Check whether feed exist, otherwise process it.
:param db_file: Database filename.
:param url: URL.
:return: Status message.
"""
exist = await database.check_feed_exist(db_file, url)
@ -483,10 +466,10 @@ async def add_feed(db_file, url):
bozo = ("WARNING: Bozo detected. Failed to load <{}>.".format(url))
print(bozo)
try:
# tree = etree.fromstring(res[0]) # etree -> html
# tree = etree.fromstring(res[0]) # etree is for xml
tree = html.fromstring(res[0])
except:
return "Failed to parse {} as feed".format(url)
return "Failed to parse URL <{}> as feed".format(url)
print("RSS Auto-Discovery Engaged")
xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]"""
@ -518,28 +501,48 @@ async def add_feed(db_file, url):
return await add_feed(db_file, url)
# Search for feeds by file extension and path
paths = ["/atom",
paths = [
"/app.php/feed", # phpbb
"/atom",
"/atom.php",
"/atom.xml",
"/content-feeds/",
"/external.php?type=RSS2",
"/feed", # good practice
"/feed.atom",
# "/feed.json",
"/feed.php",
"/feed.rdf",
"/feed.rss",
"/feed.xml",
"/feed/atom/",
"/feeds/news_feed",
"/feeds/rss/news.xml.php",
"/forum_rss.php",
"/index.php/feed",
"/index.php?type=atom;action=.xml", #smf
"/index.php?type=rss;action=.xml", #smf
"/index.rss",
"/latest.rss",
"/news",
"/news.xml",
"/news.xml.php",
"/news/feed",
"/posts.rss", # discourse
"/rdf",
"/rdf.php",
"/rdf.xml",
"/rss",
# "/rss.json",
"/rss.php",
"/rss.xml",
"/feed",
"/feed.atom",
"/feed.rdf",
"/feed.rss",
"/feed.xml",
"/news",
"/news/feed",
"?format=rss",
"/feeds/news_feed",
"/content-feeds/",
"/app.php/feed", # phpBB
"/posts.rss" # Discourse
] # More paths "rss.json", "feed.json"
"/timeline.rss",
"/xml/feed.rss",
# "?format=atom",
# "?format=rdf",
# "?format=rss",
# "?format=xml"
]
print("RSS Scan Mode Engaged")
feeds = {}
@ -551,16 +554,12 @@ async def add_feed(db_file, url):
for address in addresses:
address = address.xpath('@href')[0]
if address.startswith('/'):
address = parted_url.netloc + address
address = parted_url.scheme + '://' + parted_url.netloc + address
res = await download_feed(address)
# print(address)
if res[1] == 200:
# print(address)
try:
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
# print(feeds)
except:
# print('Not a feed')
continue
if len(feeds) > 1:
msg = "RSS URL scan has found {} feeds:\n\n".format(len(feeds))
@ -583,7 +582,18 @@ async def add_feed(db_file, url):
feeds = {}
parted_url = urlparse(url)
for path in paths:
# print(path)
address = parted_url.scheme + '://' + parted_url.netloc + path
res = await download_feed(address)
if res[1] == 200:
# print(feedparser.parse(res[0])["feed"]["title"])
# feeds[address] = feedparser.parse(res[0])["feed"]["title"]
try:
title = feedparser.parse(res[0])["feed"]["title"]
except:
title = '*** No Title ***'
feeds[address] = title
# Check whether URL has path (i.e. not root)
if parted_url.path.split('/')[1]:
paths.extend([".atom", ".feed", ".rdf", ".rss"]) if '.rss' not in paths else -1
# if paths.index('.rss'):
@ -591,20 +601,13 @@ async def add_feed(db_file, url):
address = parted_url.scheme + '://' + parted_url.netloc + '/' + parted_url.path.split('/')[1] + path
res = await download_feed(address)
if res[1] == 200:
# print('2res[1]')
# print(res[1])
# print(feedparser.parse(res[0])["feed"]["title"])
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
# print(feeds)
else:
address = parted_url.scheme + '://' + parted_url.netloc + path
res = await download_feed(address)
if res[1] == 200:
# print('1res[1]')
# print(res[1])
# print(feedparser.parse(res[0])["feed"]["title"])
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
# print(feeds)
print('ATTENTION')
print(address)
try:
title = feedparser.parse(res[0])["feed"]["title"]
except:
title = '*** No Title ***'
feeds[address] = title
if len(feeds) > 1:
msg = "RSS URL discovery has found {} feeds:\n\n".format(len(feeds))
for feed in feeds:
@ -621,19 +624,19 @@ async def add_feed(db_file, url):
else:
return await database.add_feed(db_file, feed, url, res)
else:
return "Failed to get URL <{}>. HTTP Error {}".format(url, res[1])
return "Failed to get URL <{}>. Reason: {}".format(url, res[1])
else:
return "News source <{}> is already listed in the subscription list".format(url)
ix = exist[0]
return "News source <{}> is already listed in the subscription list at index {}".format(url, ix)
def toggle_state(jid, state):
# print("toggle_state")
# time.sleep(1)
"""
Set status of update
:param jid: jid of the user
:param state: boolean
:return:
Set status of update.
:param jid: JID (Jabber ID).
:param state: True or False.
:return: Status message.
"""
db_dir = get_default_dbdir()
db_file = os.path.join(db_dir, r"{}.db".format(jid))

304
slixfeed/database.py
View file

@ -7,7 +7,6 @@ from sqlite3 import Error
import asyncio
from datetime import date
import feedparser
# from eliot import start_action, to_file
# # with start_action(action_type="list_subscriptions()", db=db_file):
@ -23,15 +22,12 @@ DBLOCK = asyncio.Lock()
CURSORS = {}
def create_connection(db_file):
# print("create_connection")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Create a database connection to the SQLite database
specified by db_file
:param db_file: database file
:return: Connection object or None
specified by db_file.
:param db_file: Database filename.
:return: Connection object or None.
"""
conn = None
try:
@ -43,10 +39,11 @@ def create_connection(db_file):
def create_tables(db_file):
# print("create_tables")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Create SQLite tables.
:param db_file: Database filename.
"""
with create_connection(db_file) as conn:
feeds_table_sql = """
CREATE TABLE IF NOT EXISTS feeds (
@ -68,18 +65,26 @@ def create_tables(db_file):
source text,
read integer
); """
# statistics_table_sql = """
# CREATE TABLE IF NOT EXISTS statistics (
# id integer PRIMARY KEY,
# title text NOT NULL,
# number integer
# ); """
c = conn.cursor()
# c = get_cursor(db_file)
c.execute(feeds_table_sql)
c.execute(entries_table_sql)
# c.execute(statistics_table_sql)
def get_cursor(db_file):
"""
Allocate a cursor to connection per database.
:param db_file: database file
:return: Cursor
:param db_file: Database filename.
:return: Cursor.
"""
if db_file in CURSORS:
return CURSORS[db_file]
@ -91,15 +96,14 @@ def get_cursor(db_file):
async def add_feed(db_file, feed, url, res):
# print("add_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Add a new feed into the feeds table
:param conn:
:param feed:
:return: string
Add a new feed into the feeds table.
:param db_file: Database filename.
:param feed: Parsed XML document.
:param url: URL.
:param res: XML document.
:return: Message.
"""
#TODO consider async with DBLOCK
#conn = create_connection(db_file)
@ -128,15 +132,12 @@ async def add_feed(db_file, feed, url, res):
async def remove_feed(db_file, ix):
# print("remove_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Delete a feed by feed id
:param conn:
:param id: id of the feed
:return: string
Delete a feed by feed id.
:param db_file: Database filename.
:param ix: Index of feed.
:return: Message.
"""
with create_connection(db_file) as conn:
async with DBLOCK:
@ -158,16 +159,13 @@ async def remove_feed(db_file, ix):
async def check_feed_exist(db_file, url):
# print("is_feed_exist")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Check whether a feed exists
Query for feeds by url
:param conn:
:param url:
:return: row
Check whether a feed exists.
Query for feeds by given url.
:param db_file: Database filename.
:param url: URL.
:return: SQL row or None.
"""
cur = get_cursor(db_file)
sql = "SELECT id FROM feeds WHERE address = ?"
@ -175,11 +173,29 @@ async def check_feed_exist(db_file, url):
return cur.fetchone()
async def get_unread_entries_number(db_file):
async def get_number_of_items(db_file, str):
"""
Check number of unread items
:param db_file
:return: string
Return number of entries or feeds.
:param cur: Cursor object.
:param str: "entries" or "feeds".
:return: Number of rows.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT count(id) FROM {}".format(str)
count = cur.execute(sql)
count = cur.fetchone()[0]
return count
async def get_number_of_entries_unread(db_file):
"""
Return number of unread items.
:param db_file: Database filename.
:param cur: Cursor object.
:return: Number of rows.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
@ -189,22 +205,16 @@ async def get_unread_entries_number(db_file):
return count
async def get_unread(db_file):
# print("get_unread")
# print("db_file")
# print(db_file)
# time.sleep(1)
async def get_entry_unread(db_file):
"""
Check read status of entry
:param conn:
:param id: id of the entry
:return: string
Check read status of entry.
:param db_file: Database filename.
:return: News item as message.
"""
with create_connection(db_file) as conn:
entry = []
cur = conn.cursor()
# cur = get_cursor(db_file)
entry = []
sql = "SELECT id FROM entries WHERE read = 0"
ix = cur.execute(sql).fetchone()
if ix is None:
@ -222,36 +232,72 @@ async def get_unread(db_file):
cur.execute(sql, (ix,))
link = cur.fetchone()[0]
entry.append(link)
entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2])
entry = "{}\n\n{}\n\n{}".format(entry[0], entry[1], entry[2])
# print(entry)
async with DBLOCK:
await mark_as_read(cur, ix)
# async with DBLOCK:
# await update_statistics(db_file)
return entry
async def mark_as_read(cur, ix):
# print("mark_as_read", ix)
# time.sleep(1)
"""
Set read status of entry
:param cur:
:param ix: index of the entry
Set read status of entry.
:param cur: Cursor object.
:param ix: Index of entry.
"""
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
cur.execute(sql, (ix,))
async def statistics(db_file):
"""
Return table statistics.
:param db_file: Database filename.
:return: News item as message.
"""
feeds = await get_number_of_items(db_file, 'feeds')
entries = await get_number_of_items(db_file, 'entries')
unread_entries = await get_number_of_entries_unread(db_file)
return "You have {} unread news items out of {} from {} news sources.".format(unread_entries, entries, feeds)
async def update_statistics(cur):
"""
Update table statistics.
:param cur: Cursor object.
"""
stat_dict = {}
stat_dict["feeds"] = await get_number_of_items(cur, 'feeds')
stat_dict["entries"] = await get_number_of_items(cur, 'entries')
stat_dict["unread"] = await get_number_of_entries_unread(cur=cur)
for i in stat_dict:
sql = "SELECT id FROM statistics WHERE title = ?"
cur.execute(sql, (i,))
if cur.fetchone():
sql = "UPDATE statistics SET number = :num WHERE title = :title"
cur.execute(sql, {"title": i, "num": stat_dict[i]})
else:
sql = "SELECT count(id) FROM statistics"
count = cur.execute(sql)
count = cur.fetchone()[0]
ix = count + 1
sql = "INSERT INTO statistics VALUES(?,?,?)"
cur.execute(sql, (ix, i, stat_dict[i]))
# TODO mark_all_read for entries of feed
async def toggle_status(db_file, ix):
# print("toggle_status")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Set status of feed
:param conn:
:param id: id of the feed
:return: string
Toggle status of feed.
:param db_file: Database filename.
:param ix: Index of entry.
:return: Message
"""
async with DBLOCK:
with create_connection(db_file) as conn:
@ -279,12 +325,11 @@ async def toggle_status(db_file, ix):
async def set_date(cur, url):
# print("set_date")
# time.sleep(1)
"""
Set last update date of feed
:param url: url of the feed
:return:
Set last update date of feed.
:param cur: Cursor object.
:param url: URL.
"""
today = date.today()
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
@ -293,6 +338,9 @@ async def set_date(cur, url):
async def add_entry_and_set_date(db_file, source, entry):
"""
TODO
"""
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
@ -301,6 +349,9 @@ async def add_entry_and_set_date(db_file, source, entry):
async def update_source_status(db_file, status, source):
"""
TODO
"""
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
async with DBLOCK:
with create_connection(db_file) as conn:
@ -309,6 +360,9 @@ async def update_source_status(db_file, status, source):
async def update_source_validity(db_file, source, valid):
"""
TODO
"""
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
async with DBLOCK:
with create_connection(db_file) as conn:
@ -317,29 +371,25 @@ async def update_source_validity(db_file, source, valid):
async def add_entry(cur, entry):
# print("add_entry")
# time.sleep(1)
"""
Add a new entry into the entries table
:param conn:
Add a new entry into the entries table.
:param cur: Cursor object.
:param entry:
:return:
"""
sql = """ INSERT INTO entries(title,summary,link,source,read)
VALUES(?,?,?,?,?) """
# cur = conn.cursor()
cur.execute(sql, entry)
# This function doesn't work as expected with bbs and wiki feeds
async def remove_entry(db_file, source, length):
# print("remove_entry")
# time.sleep(1)
"""
Maintain list of entries
Maintain list of entries equal to feed.
Check the number returned by feed and delete
existing entries up to the same returned amount
:param conn:
existing entries up to the same returned amount.
:param db_file: Database filename.
:param source:
:param length:
:return:
@ -364,18 +414,17 @@ async def remove_entry(db_file, source, length):
ORDER BY id
ASC LIMIT :limit)"""
cur.execute(sql, {"source": source, "limit": limit})
print('### removed', limit, 'from', source)
async def remove_nonexistent_entries(db_file, feed, source):
"""
Remove entries that don't exist in feed'
Check the entries returned from feed and delete
non existing entries
:param conn:
:param source:
:param length:
:return:
Remove entries that don't exist in a given parsed feed.
Check the entries returned from feed and delete non
existing entries
:param db_file: Database filename.
:param feed: URL of parsed feed.
:param source: URL of associated feed.
"""
async with DBLOCK:
with create_connection(db_file) as conn:
@ -420,12 +469,11 @@ async def remove_nonexistent_entries(db_file, feed, source):
async def get_subscriptions(db_file):
# print("get_subscriptions")
# time.sleep(1)
"""
Query feeds
:param conn:
:return: rows (tuple)
Query table feeds.
:param db_file: Database filename.
:return: List of feeds.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
@ -435,17 +483,12 @@ async def get_subscriptions(db_file):
async def list_subscriptions(db_file):
# print("list_subscriptions")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Query feeds
:param conn:
:return: rows (string)
Query table feeds and list items.
:param db_file: Database filename.
:return: List of feeds.
"""
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file)
sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql)
@ -464,28 +507,23 @@ async def list_subscriptions(db_file):
"To add feed, send a message as follows: \n"
"feed add URL \n"
"Example: \n"
"feed add https://reclaimthenet.org/feed/")
"add https://reclaimthenet.org/feed/")
return msg
async def last_entries(db_file, num):
# print("last_entries")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Query feeds
:param conn:
:param num: integer
:return: rows (string)
Query entries
:param db_file: Database filename.
:param num: Number
:return: List of recent N entries
"""
num = int(num)
if num > 50:
num = 50
elif num < 1:
num = 1
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
results = cur.execute(sql, (num,))
@ -498,21 +536,16 @@ async def last_entries(db_file, num):
async def search_entries(db_file, query):
# print("search_entries")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Query feeds
:param conn:
:param query: string
:return: rows (string)
Query entries
:param db_file: Database filename.
:param query: Search query
:return: Entries with specified keywords
"""
if len(query) < 2:
return "Please enter at least 2 characters to search"
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
results = cur.execute(sql, [f'%{query}%'])
@ -530,15 +563,14 @@ async def search_entries(db_file, query):
async def check_entry_exist(db_file, title, link):
# print("check_entry")
# time.sleep(1)
"""
Check whether an entry exists
Query entries by title and link
:param conn:
:param link:
:param title:
:return: row
Check whether an entry exists.
Query entries by title and link.
:param db_file: Database filename.
:param link: Entry URL.
:param title: Entry title.
:return: SQL row or None.
"""
cur = get_cursor(db_file)
sql = "SELECT id FROM entries WHERE title = :title and link = :link"