sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

Update slixfeed.py

Browse source
This commit is contained in:
Schimon Jehudah 2022-08-14 09:29:20 +00:00
parent b85bc9243f
commit 604aa73db1
1 changed files with 98 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
slixfeed.py
View file

@ -31,6 +31,8 @@ import time
# offline = False # offline = False
lock = asyncio.Lock()
class Slixfeed(slixmpp.ClientXMPP): class Slixfeed(slixmpp.ClientXMPP):
""" """
Slixmpp bot that will send updates of feeds it Slixmpp bot that will send updates of feeds it
@ -149,7 +151,8 @@ class Slixfeed(slixmpp.ClientXMPP):
async def send_updates(self, event): async def send_updates(self, event):
#while not offline: #while not offline:
while True: #while True:
async with lock:
print(time.strftime("%H:%M:%S")) print(time.strftime("%H:%M:%S"))
# print(offline) # print(offline)
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
@ -194,8 +197,9 @@ class Slixfeed(slixmpp.ClientXMPP):
# asyncio.ensure_future(send_updates(self)) # asyncio.ensure_future(send_updates(self))
async def check_updates(): async def check_updates(lock):
while True: #while True:
async with lock:
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
if not os.path.isdir(db_dir): if not os.path.isdir(db_dir):
msg = ("Slixfeed can not work without a database. \n" msg = ("Slixfeed can not work without a database. \n"
@ -216,7 +220,7 @@ async def check_updates():
) )
await asyncio.sleep(60 * 30) await asyncio.sleep(60 * 30)
asyncio.ensure_future(check_updates()) asyncio.ensure_future(check_updates(lock))
# async def tasks(): # async def tasks():
# # Begin scanning feeds # # Begin scanning feeds
@ -226,7 +230,7 @@ asyncio.ensure_future(check_updates())
async def tasks(jid, password): async def tasks(jid, password):
# Begin scanning feeds # Begin scanning feeds
await asyncio.gather( await asyncio.gather(
check_updates(), check_updates(lock),
Slixfeed(jid, password).send_updates() Slixfeed(jid, password).send_updates()
) )
@ -335,8 +339,11 @@ async def initdb(jid, message, callback):
create_table(conn, entries_table_sql) create_table(conn, entries_table_sql)
else: else:
print("Error! cannot create the database connection.") print("Error! cannot create the database connection.")
if message: if lock:
return await callback(conn, message) if message:
return await callback(conn, message, lock)
else:
return await callback(conn, lock)
else: else:
return await callback(conn) return await callback(conn)
@ -371,72 +378,70 @@ def create_table(conn, create_table_sql):
# def setup_info(jid): # def setup_info(jid):
# def start_process(jid): # def start_process(jid):
async def download_updates(conn): async def download_updates(conn, lock):
with conn: async with lock:
# cur = conn.cursor() with conn:
# get current date # cur = conn.cursor()
#today = date.today() # get current date
urls = await get_subscriptions(conn) #today = date.today()
for url in urls: urls = await get_subscriptions(conn)
#"".join(url) for url in urls:
source = url[0] #"".join(url)
res = await download_feed(conn, source) source = url[0]
cur = conn.cursor() res = await download_feed(conn, source)
sql = "UPDATE feeds SET status = :status WHERE address = :url" cur = conn.cursor()
cur.execute(sql, {"status": res[1], "url": source}) sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
conn.commit() cur.execute(sql, {"status": res[1], "scanned": date.today(), "url": source})
sql = "UPDATE feeds SET scanned = :scanned WHERE address = :url" conn.commit()
cur.execute(sql, {"scanned": date.today(), "url": source}) if res[0]:
conn.commit() try:
if res[0]: feed = feedparser.parse(res[0])
try: if feed.bozo:
feed = feedparser.parse(res[0]) bozo = ("WARNING: Bozo detected for feed <{}>. "
if feed.bozo: "For more information, visit "
bozo = ("WARNING: Bozo detected for feed <{}>. " "https://pythonhosted.org/feedparser/bozo.html"
"For more information, visit " .format(source))
"https://pythonhosted.org/feedparser/bozo.html" print(bozo)
.format(source)) valid = 0
print(bozo) else:
cur = conn.cursor() valid = 1
sql = "UPDATE feeds SET valid = 0 WHERE address = ?" sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
cur.execute(sql, {"validity": valid, "url": source})
conn.commit()
except (IncompleteReadError, IncompleteRead, error.URLError) as e:
print(e)
return
# TODO Place these couple of lines back down
# NOTE Need to correct the SQL statement to do so
entries = feed.entries
length = len(entries)
await remove_entry(conn, source, length)
for entry in entries:
if entry.has_key("title"):
title = entry.title
else: else:
sql = "UPDATE feeds SET valid = 1 WHERE address = ?" title = feed["feed"]["title"]
cur.execute(sql, (source,)) link = source if not entry.link else entry.link
conn.commit() exist = await check_entry(conn, title, link)
except (IncompleteReadError, IncompleteRead, error.URLError) as e: if not exist:
print(e) if entry.has_key("summary"):
return summary = entry.summary
# TODO Place these couple of lines back down # Remove HTML tags
# NOTE Need to correct the SQL statement to do so summary = BeautifulSoup(summary, "lxml").text
entries = feed.entries # TODO Limit text length
length = len(entries) summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
await remove_entry(conn, source, length) else:
for entry in entries: summary = '*** No summary ***'
if entry.has_key("title"): #print('~~~~~~summary not in entry')
title = entry.title entry = (title, summary, link, source, 0);
else: await add_entry(conn, entry)
title = feed["feed"]["title"] await set_date(conn, source)
link = source if not entry.link else entry.link #make_message
exist = await check_entry(conn, title, link) # message = title + '\n\n' + summary + '\n\nLink: ' + link
if not exist: # print(message)
if entry.has_key("summary"): # news.append(message)
summary = entry.summary # print(len(news))
# Remove HTML tags # return news
summary = BeautifulSoup(summary, "lxml").text
# TODO Limit text length
summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
else:
summary = '*** No summary ***'
#print('~~~~~~summary not in entry')
entry = (title, summary, link, source, 0);
await add_entry(conn, entry)
await set_date(conn, source)
#make_message
# message = title + '\n\n' + summary + '\n\nLink: ' + link
# print(message)
# news.append(message)
# print(len(news))
# return news
async def download_feed(conn, url): async def download_feed(conn, url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@ -465,13 +470,15 @@ async def check_feed(conn, url):
cur.execute(sql, (url,)) cur.execute(sql, (url,))
return cur.fetchone() return cur.fetchone()
async def add_feed(conn, url): async def add_feed(conn, url, lock):
""" """
Add a new feed into the feeds table Add a new feed into the feeds table
:param conn: :param conn:
:param feed: :param feed:
:return: string :return: string
""" """
#TODO consider async with lock
await lock.acquire()
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)") print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)")
@ -486,6 +493,7 @@ async def add_feed(conn, url):
VALUES(?,?,?,?) """ VALUES(?,?,?,?) """
cur.execute(sql, feed) cur.execute(sql, feed)
conn.commit() conn.commit()
lock.release()
bozo = ("WARNING: Bozo detected. Failed to load URL.") bozo = ("WARNING: Bozo detected. Failed to load URL.")
print(bozo) print(bozo)
return "Failed to parse URL as feed" return "Failed to parse URL as feed"
@ -496,12 +504,14 @@ async def add_feed(conn, url):
VALUES(?,?,?,?,?) """ VALUES(?,?,?,?,?) """
cur.execute(sql, feed) cur.execute(sql, feed)
conn.commit() conn.commit()
lock.release()
else: else:
feed = (url, 1, res[1], 0) feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid) sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """ VALUES(?,?,?,?) """
cur.execute(sql, feed) cur.execute(sql, feed)
conn.commit() conn.commit()
lock.release()
return "Failed to get URL. HTTP Error {}".format(res[1]) return "Failed to get URL. HTTP Error {}".format(res[1])
print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)") print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)")
# source = title if not '' else url # source = title if not '' else url
@ -512,7 +522,7 @@ async def add_feed(conn, url):
msg = "News source is already listed in the subscription list" msg = "News source is already listed in the subscription list"
return msg return msg
async def remove_feed(conn, id): async def remove_feed(conn, id, lock):
""" """
Delete a feed by feed id Delete a feed by feed id
:param conn: :param conn:
@ -521,6 +531,7 @@ async def remove_feed(conn, id):
""" """
# You have chose to remove feed (title, url) from your feed list. # You have chose to remove feed (title, url) from your feed list.
# Enter "delete" to confirm removal. # Enter "delete" to confirm removal.
await lock.acquire()
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from remove_feed(conn, id)") print(time.strftime("%H:%M:%S"), "conn.cursor() from remove_feed(conn, id)")
@ -534,6 +545,7 @@ async def remove_feed(conn, id):
sql = "DELETE FROM feeds WHERE id = ?" sql = "DELETE FROM feeds WHERE id = ?"
cur.execute(sql, (id,)) cur.execute(sql, (id,))
conn.commit() conn.commit()
lock.release()
print(time.strftime("%H:%M:%S"), "conn.commit() from remove_feed(conn, id)") print(time.strftime("%H:%M:%S"), "conn.commit() from remove_feed(conn, id)")
return """News source <{}> has been removed from subscriptions list return """News source <{}> has been removed from subscriptions list
""".format(url) """.format(url)
@ -604,13 +616,14 @@ async def feed_refresh(conn, id):
conn.commit() conn.commit()
# TODO mark_all_read for entries of feed # TODO mark_all_read for entries of feed
async def toggle_status(conn, id): async def toggle_status(conn, id, lock):
""" """
Set status of feed Set status of feed
:param conn: :param conn:
:param id: id of the feed :param id: id of the feed
:return: string :return: string
""" """
await lock.acquire()
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from toggle_status(conn, id)") print(time.strftime("%H:%M:%S"), "conn.cursor() from toggle_status(conn, id)")
@ -633,6 +646,7 @@ async def toggle_status(conn, id):
sql = "UPDATE feeds SET enabled = :status WHERE id = :id" sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": id}) cur.execute(sql, {"status": status, "id": id})
conn.commit() conn.commit()
lock.release()
print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)") print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)")
return notice return notice
@ -693,17 +707,19 @@ async def get_subscriptions(conn):
result = cur.execute(sql) result = cur.execute(sql)
return result return result
async def list_subscriptions(conn): async def list_subscriptions(conn, lock):
""" """
Query feeds Query feeds
:param conn: :param conn:
:return: rows (string) :return: rows (string)
""" """
await lock.acquire()
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from list_subscriptions(conn)") print(time.strftime("%H:%M:%S"), "conn.cursor() from list_subscriptions(conn)")
#sql = "SELECT id, address FROM feeds" #sql = "SELECT id, address FROM feeds"
sql = "SELECT name, address, updated, id, enabled FROM feeds" sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql) results = cur.execute(sql)
lock.release()
feeds_list = "List of subscriptions: \n" feeds_list = "List of subscriptions: \n"
counter = 0 counter = 0
for result in results: for result in results:
@ -722,7 +738,7 @@ async def list_subscriptions(conn):
"feed add https://reclaimthenet.org/feed/") "feed add https://reclaimthenet.org/feed/")
return msg return msg
async def last_entries(conn, num): async def last_entries(conn, num, lock):
""" """
Query feeds Query feeds
:param conn: :param conn:
@ -734,9 +750,11 @@ async def last_entries(conn, num):
num = 50 num = 50
elif num < 1: elif num < 1:
num = 1 num = 1
await lock.acquire()
cur = conn.cursor() cur = conn.cursor()
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT {}".format(num) sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT {}".format(num)
results = cur.execute(sql) results = cur.execute(sql)
lock.release()
titles_list = "Recent {} titles: \n".format(num) titles_list = "Recent {} titles: \n".format(num)
for result in results: for result in results:
# titles_list += """\nTitle: {} \nLink: {} # titles_list += """\nTitle: {} \nLink: {}
@ -744,7 +762,7 @@ async def last_entries(conn, num):
""".format(str(result[0]), str(result[1])) """.format(str(result[0]), str(result[1]))
return titles_list return titles_list
async def search_entries(conn, query): async def search_entries(conn, query, lock):
""" """
Query feeds Query feeds
:param conn: :param conn:
@ -753,10 +771,12 @@ async def search_entries(conn, query):
""" """
if len(query) < 2: if len(query) < 2:
return "Please enter at least 2 characters to search" return "Please enter at least 2 characters to search"
await lock.acquire()
cur = conn.cursor() cur = conn.cursor()
sql = "SELECT title, link FROM entries WHERE title LIKE '%{}%' LIMIT 50".format(query) sql = "SELECT title, link FROM entries WHERE title LIKE '%{}%' LIMIT 50".format(query)
# sql = "SELECT title, link FROM entries WHERE title OR link LIKE '%{}%'".format(query) # sql = "SELECT title, link FROM entries WHERE title OR link LIKE '%{}%'".format(query)
results = cur.execute(sql) results = cur.execute(sql)
lock.release()
results_list = "Search results for '{}': \n".format(query) results_list = "Search results for '{}': \n".format(query)
counter = 0 counter = 0
for result in results: for result in results: