thecoffemaker/Slixfeed
1
0
Fork 0
forked from sch/Slixfeed
Code Pull requests Activity

Update slixfeed.py

Browse source
This commit is contained in:
Schimon Jehudah 2022-08-09 10:24:36 +00:00
parent ecd414b228
commit b85bc9243f
1 changed files with 70 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
slixfeed.py
View file

@ -311,8 +311,11 @@ async def initdb(jid, message, callback):
id integer PRIMARY KEY, id integer PRIMARY KEY,
name text, name text,
address text NOT NULL, address text NOT NULL,
enabled integer NOT NULL,
scanned text,
updated text,
status integer, status integer,
updated text valid integer
); """ ); """
entries_table_sql = """ entries_table_sql = """
CREATE TABLE IF NOT EXISTS entries ( CREATE TABLE IF NOT EXISTS entries (
@ -377,17 +380,29 @@ async def download_updates(conn):
for url in urls: for url in urls:
#"".join(url) #"".join(url)
source = url[0] source = url[0]
html = await download_page(url[0]) res = await download_feed(conn, source)
print(url[0]) cur = conn.cursor()
if html: sql = "UPDATE feeds SET status = :status WHERE address = :url"
cur.execute(sql, {"status": res[1], "url": source})
conn.commit()
sql = "UPDATE feeds SET scanned = :scanned WHERE address = :url"
cur.execute(sql, {"scanned": date.today(), "url": source})
conn.commit()
if res[0]:
try: try:
feed = feedparser.parse(html) feed = feedparser.parse(res[0])
if feed.bozo: if feed.bozo:
bozo = ("WARNING: Bozo detected for feed <{}>. " bozo = ("WARNING: Bozo detected for feed <{}>. "
"For more information, visit " "For more information, visit "
"https://pythonhosted.org/feedparser/bozo.html" "https://pythonhosted.org/feedparser/bozo.html"
.format(source)) .format(source))
print(bozo) print(bozo)
cur = conn.cursor()
sql = "UPDATE feeds SET valid = 0 WHERE address = ?"
else:
sql = "UPDATE feeds SET valid = 1 WHERE address = ?"
cur.execute(sql, (source,))
conn.commit()
except (IncompleteReadError, IncompleteRead, error.URLError) as e: except (IncompleteReadError, IncompleteRead, error.URLError) as e:
print(e) print(e)
return return
@ -409,7 +424,7 @@ async def download_updates(conn):
# Remove HTML tags # Remove HTML tags
summary = BeautifulSoup(summary, "lxml").text summary = BeautifulSoup(summary, "lxml").text
# TODO Limit text length # TODO Limit text length
summary = summary.replace("\n\n", "\n") summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
else: else:
summary = '*** No summary ***' summary = '*** No summary ***'
#print('~~~~~~summary not in entry') #print('~~~~~~summary not in entry')
@ -423,14 +438,15 @@ async def download_updates(conn):
# print(len(news)) # print(len(news))
# return news # return news
async def download_page(url): async def download_feed(conn, url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.get(url) as response: async with session.get(url) as response:
status = response.status
if response.status == 200: if response.status == 200:
html = await response.text() doc = await response.text()
return html return [doc,status]
print("Status:", response.status) else:
print("Content-type:", response.headers['content-type']) return [False,status]
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.run_until_complete loop.run_until_complete
@ -461,17 +477,32 @@ async def add_feed(conn, url):
print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)") print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)")
exist = await check_feed(conn, url) exist = await check_feed(conn, url)
if not exist: if not exist:
feed = feedparser.parse(url) res = await download_feed(conn, url)
if res[0]:
feed = feedparser.parse(res[0])
if feed.bozo: if feed.bozo:
feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
bozo = ("WARNING: Bozo detected. Failed to load URL.") bozo = ("WARNING: Bozo detected. Failed to load URL.")
print(bozo) print(bozo)
return "Failed to parse URL as feed" return "Failed to parse URL as feed"
title = feedparser.parse(url)["feed"]["title"] else:
feed = (title, url, 1) title = feed["feed"]["title"]
sql = """INSERT INTO feeds(name,address,status) feed = (title, url, 1, res[1], 1)
VALUES(?,?,?) """ sql = """INSERT INTO feeds(name,address,enabled,status,valid)
VALUES(?,?,?,?,?) """
cur.execute(sql, feed) cur.execute(sql, feed)
conn.commit() conn.commit()
else:
feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
return "Failed to get URL. HTTP Error {}".format(res[1])
print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)") print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)")
# source = title if not '' else url # source = title if not '' else url
source = title if title else '<' + url + '>' source = title if title else '<' + url + '>'
@ -560,7 +591,19 @@ async def mark_as_read(conn, id):
print(time.strftime("%H:%M:%S"), "conn.commit() from mark_as_read(conn, id)") print(time.strftime("%H:%M:%S"), "conn.commit() from mark_as_read(conn, id)")
#conn.close() #conn.close()
# TODO test async def feed_refresh(conn, id):
cur = conn.cursor()
sql = "SELECT address FROM feeds WHERE id = :id"
cur.execute(sql, (id,))
url = cur.fetchone()[0]
res = await download_feed(conn, url)
feed = feedparser.parse(res[0])
title = feed["feed"]["title"]
sql = "UPDATE feeds SET name = :name WHERE address = :url"
cur.execute(sql, {"name": title, "url": url})
conn.commit()
# TODO mark_all_read for entries of feed
async def toggle_status(conn, id): async def toggle_status(conn, id):
""" """
Set status of feed Set status of feed
@ -574,7 +617,7 @@ async def toggle_status(conn, id):
sql = "SELECT name FROM feeds WHERE id = :id" sql = "SELECT name FROM feeds WHERE id = :id"
cur.execute(sql, (id,)) cur.execute(sql, (id,))
title = cur.fetchone()[0] title = cur.fetchone()[0]
sql = "SELECT status FROM feeds WHERE id = ?" sql = "SELECT enabled FROM feeds WHERE id = ?"
# NOTE [0][1][2] # NOTE [0][1][2]
cur.execute(sql, (id,)) cur.execute(sql, (id,))
status = cur.fetchone()[0] status = cur.fetchone()[0]
@ -587,7 +630,7 @@ async def toggle_status(conn, id):
else: else:
status = 1 status = 1
notice = "News updates for '{}' are now enabled".format(title) notice = "News updates for '{}' are now enabled".format(title)
sql = "UPDATE feeds SET status = :status WHERE id = :id" sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": id}) cur.execute(sql, {"status": status, "id": id})
conn.commit() conn.commit()
print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)") print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)")
@ -646,7 +689,7 @@ async def get_subscriptions(conn):
""" """
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from get_subscriptions(conn)") print(time.strftime("%H:%M:%S"), "conn.cursor() from get_subscriptions(conn)")
sql = "SELECT address FROM feeds WHERE status = 1" sql = "SELECT address FROM feeds WHERE enabled = 1"
result = cur.execute(sql) result = cur.execute(sql)
return result return result
@ -659,7 +702,7 @@ async def list_subscriptions(conn):
cur = conn.cursor() cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from list_subscriptions(conn)") print(time.strftime("%H:%M:%S"), "conn.cursor() from list_subscriptions(conn)")
#sql = "SELECT id, address FROM feeds" #sql = "SELECT id, address FROM feeds"
sql = "SELECT name, address, updated, id, status FROM feeds" sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql) results = cur.execute(sql)
feeds_list = "List of subscriptions: \n" feeds_list = "List of subscriptions: \n"
counter = 0 counter = 0