thecoffemaker/Slixfeed
1
0
Fork 0
forked from sch/Slixfeed
Code Pull requests Activity

Update slixfeed.py

Browse source
This commit is contained in:
Schimon Jehudah 2022-08-09 10:24:36 +00:00
parent ecd414b228
commit b85bc9243f
1 changed files with 70 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

85
slixfeed.py
View file

@ -311,8 +311,11 @@ async def initdb(jid, message, callback):
id integer PRIMARY KEY,
name text,
address text NOT NULL,
enabled integer NOT NULL,
scanned text,
updated text,
status integer,
updated text
valid integer
); """
entries_table_sql = """
CREATE TABLE IF NOT EXISTS entries (
@ -377,17 +380,29 @@ async def download_updates(conn):
for url in urls:
#"".join(url)
source = url[0]
html = await download_page(url[0])
print(url[0])
if html:
res = await download_feed(conn, source)
cur = conn.cursor()
sql = "UPDATE feeds SET status = :status WHERE address = :url"
cur.execute(sql, {"status": res[1], "url": source})
conn.commit()
sql = "UPDATE feeds SET scanned = :scanned WHERE address = :url"
cur.execute(sql, {"scanned": date.today(), "url": source})
conn.commit()
if res[0]:
try:
feed = feedparser.parse(html)
feed = feedparser.parse(res[0])
if feed.bozo:
bozo = ("WARNING: Bozo detected for feed <{}>. "
"For more information, visit "
"https://pythonhosted.org/feedparser/bozo.html"
.format(source))
print(bozo)
cur = conn.cursor()
sql = "UPDATE feeds SET valid = 0 WHERE address = ?"
else:
sql = "UPDATE feeds SET valid = 1 WHERE address = ?"
cur.execute(sql, (source,))
conn.commit()
except (IncompleteReadError, IncompleteRead, error.URLError) as e:
print(e)
return
@ -409,7 +424,7 @@ async def download_updates(conn):
# Remove HTML tags
summary = BeautifulSoup(summary, "lxml").text
# TODO Limit text length
summary = summary.replace("\n\n", "\n")
summary = summary.replace("\n\n", "\n")[:300] + " ‍⃨"
else:
summary = '*** No summary ***'
#print('~~~~~~summary not in entry')
@ -423,14 +438,15 @@ async def download_updates(conn):
# print(len(news))
# return news
async def download_page(url):
async def download_feed(conn, url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
status = response.status
if response.status == 200:
html = await response.text()
return html
print("Status:", response.status)
print("Content-type:", response.headers['content-type'])
doc = await response.text()
return [doc,status]
else:
return [False,status]
loop = asyncio.get_event_loop()
loop.run_until_complete
@ -461,17 +477,32 @@ async def add_feed(conn, url):
print(time.strftime("%H:%M:%S"), "conn.cursor() from add_feed(conn, url)")
exist = await check_feed(conn, url)
if not exist:
feed = feedparser.parse(url)
res = await download_feed(conn, url)
if res[0]:
feed = feedparser.parse(res[0])
if feed.bozo:
feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
bozo = ("WARNING: Bozo detected. Failed to load URL.")
print(bozo)
return "Failed to parse URL as feed"
title = feedparser.parse(url)["feed"]["title"]
feed = (title, url, 1)
sql = """INSERT INTO feeds(name,address,status)
VALUES(?,?,?) """
else:
title = feed["feed"]["title"]
feed = (title, url, 1, res[1], 1)
sql = """INSERT INTO feeds(name,address,enabled,status,valid)
VALUES(?,?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
else:
feed = (url, 1, res[1], 0)
sql = """INSERT INTO feeds(address,enabled,status,valid)
VALUES(?,?,?,?) """
cur.execute(sql, feed)
conn.commit()
return "Failed to get URL. HTTP Error {}".format(res[1])
print(time.strftime("%H:%M:%S"), "conn.commit() from add_feed(conn, url)")
# source = title if not '' else url
source = title if title else '<' + url + '>'
@ -560,7 +591,19 @@ async def mark_as_read(conn, id):
print(time.strftime("%H:%M:%S"), "conn.commit() from mark_as_read(conn, id)")
#conn.close()
# TODO test
async def feed_refresh(conn, id):
cur = conn.cursor()
sql = "SELECT address FROM feeds WHERE id = :id"
cur.execute(sql, (id,))
url = cur.fetchone()[0]
res = await download_feed(conn, url)
feed = feedparser.parse(res[0])
title = feed["feed"]["title"]
sql = "UPDATE feeds SET name = :name WHERE address = :url"
cur.execute(sql, {"name": title, "url": url})
conn.commit()
# TODO mark_all_read for entries of feed
async def toggle_status(conn, id):
"""
Set status of feed
@ -574,7 +617,7 @@ async def toggle_status(conn, id):
sql = "SELECT name FROM feeds WHERE id = :id"
cur.execute(sql, (id,))
title = cur.fetchone()[0]
sql = "SELECT status FROM feeds WHERE id = ?"
sql = "SELECT enabled FROM feeds WHERE id = ?"
# NOTE [0][1][2]
cur.execute(sql, (id,))
status = cur.fetchone()[0]
@ -587,7 +630,7 @@ async def toggle_status(conn, id):
else:
status = 1
notice = "News updates for '{}' are now enabled".format(title)
sql = "UPDATE feeds SET status = :status WHERE id = :id"
sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": id})
conn.commit()
print(time.strftime("%H:%M:%S"), "conn.commit() from toggle_status(conn, id)")
@ -646,7 +689,7 @@ async def get_subscriptions(conn):
"""
cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from get_subscriptions(conn)")
sql = "SELECT address FROM feeds WHERE status = 1"
sql = "SELECT address FROM feeds WHERE enabled = 1"
result = cur.execute(sql)
return result
@ -659,7 +702,7 @@ async def list_subscriptions(conn):
cur = conn.cursor()
print(time.strftime("%H:%M:%S"), "conn.cursor() from list_subscriptions(conn)")
#sql = "SELECT id, address FROM feeds"
sql = "SELECT name, address, updated, id, status FROM feeds"
sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql)
feeds_list = "List of subscriptions: \n"
counter = 0