thecoffemaker/Slixfeed
1
0
Fork 0
forked from sch/Slixfeed
Code Pull requests Activity

Update slixfeed.py

Browse source
This commit is contained in:
Schimon Jehudah 2022-08-17 13:32:25 +00:00
parent 692faed8ee
commit 0c32f95814
1 changed files with 162 additions and 172 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

334
slixfeed.py
View file

@ -397,66 +397,64 @@ def create_table(conn, create_table_sql):
# def setup_info(jid): # def setup_info(jid):
# def start_process(jid): # def start_process(jid):
async def download_updates(conn, lock): async def download_updates(conn, lock):
print("download_updates(conn, lock)") async with lock:
with conn: print("download_updates(conn, lock)")
# cur = conn.cursor() with conn:
# get current date # cur = conn.cursor()
#today = date.today() # get current date
urls = await get_subscriptions(conn) #today = date.today()
for url in urls: urls = await get_subscriptions(conn)
#"".join(url) for url in urls:
source = url[0] #"".join(url)
res = await download_feed(conn, source) source = url[0]
await lock.acquire() res = await download_feed(conn, source)
cur = conn.cursor() cur = conn.cursor()
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url" sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
cur.execute(sql, {"status": res[1], "scanned": date.today(), "url": source}) cur.execute(sql, {"status": res[1], "scanned": date.today(), "url": source})
conn.commit() conn.commit()
if res[0]: if res[0]:
try: try:
feed = feedparser.parse(res[0]) feed = feedparser.parse(res[0])
if feed.bozo: if feed.bozo:
bozo = ("WARNING: Bozo detected for feed <{}>. " bozo = ("WARNING: Bozo detected for feed <{}>. "
"For more information, visit " "For more information, visit "
"https://pythonhosted.org/feedparser/bozo.html" "https://pythonhosted.org/feedparser/bozo.html"
.format(source)) .format(source))
print(bozo) print(bozo)
valid = 0 valid = 0
else:
valid = 1
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
cur.execute(sql, {"validity": valid, "url": source})
conn.commit()
except (IncompleteReadError, IncompleteRead, error.URLError) as e:
print(e)
return
# TODO Place these couple of lines back down
# NOTE Need to correct the SQL statement to do so
entries = feed.entries
length = len(entries)
await remove_entry(conn, source, length)
for entry in entries:
if entry.has_key("title"):
title = entry.title
else: else:
valid = 1 title = feed["feed"]["title"]
sql = "UPDATE feeds SET valid = :validity WHERE address = :url" link = source if not entry.link else entry.link
cur.execute(sql, {"validity": valid, "url": source}) exist = await check_entry(conn, title, link)
conn.commit() if not exist:
except (IncompleteReadError, IncompleteRead, error.URLError) as e: if entry.has_key("summary"):
print(e) summary = entry.summary
lock.release() # Remove HTML tags
return summary = BeautifulSoup(summary, "lxml").text
# TODO Place these couple of lines back down # TODO Limit text length
# NOTE Need to correct the SQL statement to do so summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
entries = feed.entries else:
length = len(entries) summary = '*** No summary ***'
await remove_entry(conn, source, length) #print('~~~~~~summary not in entry')
for entry in entries: entry = (title, summary, link, source, 0);
if entry.has_key("title"): await add_entry(conn, entry)
title = entry.title await set_date(conn, source)
else:
title = feed["feed"]["title"]
link = source if not entry.link else entry.link
exist = await check_entry(conn, title, link)
if not exist:
if entry.has_key("summary"):
summary = entry.summary
# Remove HTML tags
summary = BeautifulSoup(summary, "lxml").text
# TODO Limit text length
summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
else:
summary = '*** No summary ***'
#print('~~~~~~summary not in entry')
entry = (title, summary, link, source, 0);
await add_entry(conn, entry)
await set_date(conn, source)
lock.release()
#make_message #make_message
# message = title + '\n\n' + summary + '\n\nLink: ' + link # message = title + '\n\n' + summary + '\n\nLink: ' + link
# print(message) # print(message)
@ -485,13 +483,12 @@ async def check_feed(conn, lock, url):
:param url: :param url:
:return: row :return: row
""" """
await lock.acquire() async with lock:
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from check_feed(conn, url)") print(time.strftime("%H:%M:%S"), "conn.cursor() from check_feed(conn, url)")
sql = "SELECT id FROM feeds WHERE address = ?" sql = "SELECT id FROM feeds WHERE address = ?"
cur.execute(sql, (url,)) cur.execute(sql, (url,))
lock.release() return cur.fetchone()
return cur.fetchone()
async def add_feed(conn, lock, url): async def add_feed(conn, lock, url):
""" """
@ -500,51 +497,48 @@ async def add_feed(conn, lock, url):
:param feed: :param feed:
:return: string :return: string
""" """
print("add_feed(conn, lock, url)") async with lock:
#TODO consider async with lock print("add_feed(conn, lock, url)")
#conn = create_connection(db_file) #TODO consider async with lock
cur = conn.cursor() #conn = create_connection(db_file)
print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)") cur = conn.cursor()
exist = await check_feed(conn, lock, url) print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)")
if not exist: exist = await check_feed(conn, lock, url)
res = await download_feed(conn, url) if not exist:
if res[0]: res = await download_feed(conn, url)
feed = feedparser.parse(res[0]) if res[0]:
await lock.acquire() feed = feedparser.parse(res[0])
if feed.bozo: if feed.bozo:
feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
bozo = ("WARNING: Bozo detected. Failed to load URL.")
print(bozo)
return "Failed to parse URL as feed"
else:
title = feed["feed"]["title"]
feed = (title, url, 1, res[1], 1)
sql = """INSERT INTO feeds(name,address,enabled,status,valid)
VALUES(?,?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
else:
feed = (url, 1, res[1], 0) feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid) sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """ VALUES(?,?,?,?) """
cur.execute(sql, feed) cur.execute(sql, feed)
conn.commit() conn.commit()
lock.release() return "Failed to get URL. HTTP Error {}".format(res[1])
bozo = ("WARNING: Bozo detected. Failed to load URL.") print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)")
print(bozo) # source = title if not '' else url
return "Failed to parse URL as feed" source = title if title else '<' + url + '>'
else: msg = """News source "{}" has been added to subscriptions list
title = feed["feed"]["title"] """.format(source)
feed = (title, url, 1, res[1], 1)
sql = """INSERT INTO feeds(name,address,enabled,status,valid)
VALUES(?,?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
lock.release()
else: else:
feed = (url, 1, res[1], 0) msg = "News source is already listed in the subscription list"
sql = """INSERT INTO feeds(address,enabled,status,valid) return msg
VALUES(?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
lock.release()
return "Failed to get URL. HTTP Error {}".format(res[1])
print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)")
# source = title if not '' else url
source = title if title else '<' + url + '>'
msg = """News source "{}" has been added to subscriptions list
""".format(source)
else:
msg = "News source is already listed in the subscription list"
return msg
async def remove_feed(conn, lock, id): async def remove_feed(conn, lock, id):
""" """
@ -553,27 +547,26 @@ async def remove_feed(conn, lock, id):
:param id: id of the feed :param id: id of the feed
:return: string :return: string
""" """
print("remove_feed(conn, lock, id)") async with lock:
# You have chose to remove feed (title, url) from your feed list. print("remove_feed(conn, lock, id)")
# Enter "delete" to confirm removal. # You have chose to remove feed (title, url) from your feed list.
await lock.acquire() # Enter "delete" to confirm removal.
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from remove_feed(conn, id)") print(time.strftime("%H:%M:%S"), "conn.cursor() from remove_feed(conn, id)")
sql = "SELECT address FROM feeds WHERE id = ?" sql = "SELECT address FROM feeds WHERE id = ?"
# NOTE [0][1][2] # NOTE [0][1][2]
url = cur.execute(sql, (id,)) url = cur.execute(sql, (id,))
for i in url: for i in url:
url = i[0] url = i[0]
sql = "DELETE FROM entries WHERE source = ?" sql = "DELETE FROM entries WHERE source = ?"
cur.execute(sql, (url,)) cur.execute(sql, (url,))
sql = "DELETE FROM feeds WHERE id = ?" sql = "DELETE FROM feeds WHERE id = ?"
cur.execute(sql, (id,)) cur.execute(sql, (id,))
conn.commit() conn.commit()
lock.release() print(time.strftime("%H:%M:%S"), "conn.commit() from remove_feed(conn, id)")
print(time.strftime("%H:%M:%S"), "conn.commit() from remove_feed(conn, id)") return """News source <{}> has been removed from subscriptions list
return """News source <{}> has been removed from subscriptions list """.format(url)
""".format(url)
async def get_unread(conn, lock): async def get_unread(conn, lock):
""" """
@ -620,29 +613,27 @@ async def mark_as_read(conn, lock, id):
:param conn: :param conn:
:param id: id of the entry :param id: id of the entry
""" """
cur = conn.cursor() async with lock:
print(time.strftime("%H:%M:%S"), "conn.cursor() from mark_as_read(conn, id)") cur = conn.cursor()
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?" print(time.strftime("%H:%M:%S"), "conn.cursor() from mark_as_read(conn, id)")
await lock.acquire() sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
cur.execute(sql, (id,)) cur.execute(sql, (id,))
conn.commit() conn.commit()
lock.release() print(time.strftime("%H:%M:%S"), "conn.commit() from mark_as_read(conn, id)")
print(time.strftime("%H:%M:%S"), "conn.commit() from mark_as_read(conn, id)") #conn.close()
#conn.close()
async def feed_refresh(conn, lock, id): async def feed_refresh(conn, lock, id):
cur = conn.cursor() async with lock:
sql = "SELECT address FROM feeds WHERE id = :id" cur = conn.cursor()
cur.execute(sql, (id,)) sql = "SELECT address FROM feeds WHERE id = :id"
url = cur.fetchone()[0] cur.execute(sql, (id,))
res = await download_feed(conn, url) url = cur.fetchone()[0]
feed = feedparser.parse(res[0]) res = await download_feed(conn, url)
title = feed["feed"]["title"] feed = feedparser.parse(res[0])
sql = "UPDATE feeds SET name = :name WHERE address = :url" title = feed["feed"]["title"]
await lock.acquire() sql = "UPDATE feeds SET name = :name WHERE address = :url"
cur.execute(sql, {"name": title, "url": url}) cur.execute(sql, {"name": title, "url": url})
conn.commit() conn.commit()
lock.release()
# TODO mark_all_read for entries of feed # TODO mark_all_read for entries of feed
async def toggle_status(conn, lock, id): async def toggle_status(conn, lock, id):
@ -653,32 +644,31 @@ async def toggle_status(conn, lock, id):
:return: string :return: string
""" """
print("toggle_status(conn, lock, id)") print("toggle_status(conn, lock, id)")
await lock.acquire() async with lock:
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from toggle_status(conn, id)") print(time.strftime("%H:%M:%S"), "conn.cursor() from toggle_status(conn, id)")
sql = "SELECT name FROM feeds WHERE id = :id" sql = "SELECT name FROM feeds WHERE id = :id"
cur.execute(sql, (id,)) cur.execute(sql, (id,))
title = cur.fetchone()[0] title = cur.fetchone()[0]
sql = "SELECT enabled FROM feeds WHERE id = ?" sql = "SELECT enabled FROM feeds WHERE id = ?"
# NOTE [0][1][2] # NOTE [0][1][2]
cur.execute(sql, (id,)) cur.execute(sql, (id,))
status = cur.fetchone()[0] status = cur.fetchone()[0]
# FIXME always set to 1 # FIXME always set to 1
# NOTE Maybe because is not integer # NOTE Maybe because is not integer
# TODO Reset feed table before further testing # TODO Reset feed table before further testing
if status == 1: if status == 1:
status = 0 status = 0
notice = "News updates for '{}' are now disabled".format(title) notice = "News updates for '{}' are now disabled".format(title)
else: else:
status = 1 status = 1
notice = "News updates for '{}' are now enabled".format(title) notice = "News updates for '{}' are now enabled".format(title)
sql = "UPDATE feeds SET enabled = :status WHERE id = :id" sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": id}) cur.execute(sql, {"status": status, "id": id})
conn.commit() conn.commit()
lock.release() print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)")
print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)") return notice
return notice
async def toggle_state(jid, state): async def toggle_state(jid, state):
""" """