forked from sch/Slixfeed
546 lines
16 KiB
Python
546 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import sqlite3
|
|
from sqlite3 import Error
|
|
|
|
import asyncio
|
|
|
|
from datetime import date
|
|
import feedparser
|
|
|
|
# from eliot import start_action, to_file
|
|
# # with start_action(action_type="list_subscriptions()", db=db_file):
|
|
# # with start_action(action_type="last_entries()", num=num):
|
|
# # with start_action(action_type="get_subscriptions()"):
|
|
# # with start_action(action_type="remove_entry()", source=source):
|
|
# # with start_action(action_type="search_entries()", query=query):
|
|
# # with start_action(action_type="check_entry()", link=link):
|
|
|
|
# aiosqlite
|
|
DBLOCK = asyncio.Lock()
|
|
|
|
CURSORS = {}
|
|
|
|
def create_connection(db_file):
|
|
# print("create_connection")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Create a database connection to the SQLite database
|
|
specified by db_file
|
|
:param db_file: database file
|
|
:return: Connection object or None
|
|
"""
|
|
conn = None
|
|
try:
|
|
conn = sqlite3.connect(db_file)
|
|
return conn
|
|
except Error as e:
|
|
print(e)
|
|
return conn
|
|
|
|
|
|
def create_tables(db_file):
|
|
# print("create_tables")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
with create_connection(db_file) as conn:
|
|
feeds_table_sql = """
|
|
CREATE TABLE IF NOT EXISTS feeds (
|
|
id integer PRIMARY KEY,
|
|
name text,
|
|
address text NOT NULL,
|
|
enabled integer NOT NULL,
|
|
scanned text,
|
|
updated text,
|
|
status integer,
|
|
valid integer
|
|
); """
|
|
entries_table_sql = """
|
|
CREATE TABLE IF NOT EXISTS entries (
|
|
id integer PRIMARY KEY,
|
|
title text NOT NULL,
|
|
summary text NOT NULL,
|
|
link text NOT NULL,
|
|
source text,
|
|
read integer
|
|
); """
|
|
|
|
c = conn.cursor()
|
|
# c = get_cursor(db_file)
|
|
c.execute(feeds_table_sql)
|
|
c.execute(entries_table_sql)
|
|
|
|
|
|
def get_cursor(db_file):
|
|
"""
|
|
Allocate a cursor to connection per database.
|
|
:param db_file: database file
|
|
:return: Cursor
|
|
"""
|
|
if db_file in CURSORS:
|
|
return CURSORS[db_file]
|
|
else:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
CURSORS[db_file] = cur
|
|
return CURSORS[db_file]
|
|
|
|
|
|
async def add_feed(db_file, feed, url, res):
|
|
# print("add_feed")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Add a new feed into the feeds table
|
|
:param conn:
|
|
:param feed:
|
|
:return: string
|
|
"""
|
|
#TODO consider async with DBLOCK
|
|
#conn = create_connection(db_file)
|
|
|
|
# with create_connection(db_file) as conn:
|
|
# #exist = await check_feed_exist(conn, url)
|
|
# exist = await check_feed_exist(db_file, url)
|
|
|
|
# if not exist:
|
|
# res = await main.download_feed(url)
|
|
# else:
|
|
# return "News source is already listed in the subscription list"
|
|
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
title = feed["feed"]["title"]
|
|
feed = (title, url, 1, res[1], 1)
|
|
sql = """INSERT INTO feeds(name,address,enabled,status,valid)
|
|
VALUES(?,?,?,?,?) """
|
|
cur.execute(sql, feed)
|
|
|
|
source = title if title else '<' + url + '>'
|
|
msg = 'News source "{}" has been added to subscription list'.format(source)
|
|
return msg
|
|
|
|
|
|
async def remove_feed(db_file, ix):
|
|
# print("remove_feed")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Delete a feed by feed id
|
|
:param conn:
|
|
:param id: id of the feed
|
|
:return: string
|
|
"""
|
|
with create_connection(db_file) as conn:
|
|
async with DBLOCK:
|
|
cur = conn.cursor()
|
|
try:
|
|
sql = "SELECT address FROM feeds WHERE id = ?"
|
|
url = cur.execute(sql, (ix,))
|
|
for i in url:
|
|
url = i[0]
|
|
# NOTE Should we move DBLOCK to this line? 2022-12-23
|
|
sql = "DELETE FROM entries WHERE source = ?"
|
|
cur.execute(sql, (url,))
|
|
sql = "DELETE FROM feeds WHERE id = ?"
|
|
cur.execute(sql, (ix,))
|
|
return """News source <{}> has been removed from subscription list
|
|
""".format(url)
|
|
except:
|
|
return """No news source with ID {}""".format(ix)
|
|
|
|
|
|
async def check_feed_exist(db_file, url):
|
|
# print("is_feed_exist")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Check whether a feed exists
|
|
Query for feeds by url
|
|
:param conn:
|
|
:param url:
|
|
:return: row
|
|
"""
|
|
cur = get_cursor(db_file)
|
|
sql = "SELECT id FROM feeds WHERE address = ?"
|
|
cur.execute(sql, (url,))
|
|
return cur.fetchone()
|
|
|
|
|
|
async def get_unread_entries_number(db_file):
|
|
"""
|
|
Check number of unread items
|
|
:param db_file
|
|
:return: string
|
|
"""
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
sql = "SELECT count(id) FROM entries WHERE read = 0"
|
|
count = cur.execute(sql)
|
|
count = cur.fetchone()[0]
|
|
return count
|
|
|
|
|
|
|
|
async def get_unread(db_file):
|
|
# print("get_unread")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Check read status of entry
|
|
:param conn:
|
|
:param id: id of the entry
|
|
:return: string
|
|
"""
|
|
with create_connection(db_file) as conn:
|
|
entry = []
|
|
cur = conn.cursor()
|
|
# cur = get_cursor(db_file)
|
|
sql = "SELECT id FROM entries WHERE read = 0"
|
|
ix = cur.execute(sql).fetchone()
|
|
if ix is None:
|
|
return False
|
|
ix = ix[0]
|
|
sql = "SELECT title FROM entries WHERE id = :id"
|
|
cur.execute(sql, (ix,))
|
|
title = cur.fetchone()[0]
|
|
entry.append(title)
|
|
sql = "SELECT summary FROM entries WHERE id = :id"
|
|
cur.execute(sql, (ix,))
|
|
summary = cur.fetchone()[0]
|
|
entry.append(summary)
|
|
sql = "SELECT link FROM entries WHERE id = :id"
|
|
cur.execute(sql, (ix,))
|
|
link = cur.fetchone()[0]
|
|
entry.append(link)
|
|
entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2])
|
|
# print(entry)
|
|
async with DBLOCK:
|
|
await mark_as_read(cur, ix)
|
|
return entry
|
|
|
|
|
|
async def mark_as_read(cur, ix):
|
|
# print("mark_as_read", ix)
|
|
# time.sleep(1)
|
|
"""
|
|
Set read status of entry
|
|
:param cur:
|
|
:param ix: index of the entry
|
|
"""
|
|
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
|
|
cur.execute(sql, (ix,))
|
|
|
|
|
|
# TODO mark_all_read for entries of feed
|
|
async def toggle_status(db_file, ix):
|
|
# print("toggle_status")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Set status of feed
|
|
:param conn:
|
|
:param id: id of the feed
|
|
:return: string
|
|
"""
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
#cur = get_cursor(db_file)
|
|
sql = "SELECT name FROM feeds WHERE id = :id"
|
|
cur.execute(sql, (ix,))
|
|
title = cur.fetchone()[0]
|
|
sql = "SELECT enabled FROM feeds WHERE id = ?"
|
|
# NOTE [0][1][2]
|
|
cur.execute(sql, (ix,))
|
|
status = cur.fetchone()[0]
|
|
# FIXME always set to 1
|
|
# NOTE Maybe because is not integer
|
|
# TODO Reset feed table before further testing
|
|
if status == 1:
|
|
status = 0
|
|
state = "disabled"
|
|
else:
|
|
status = 1
|
|
state = "enabled"
|
|
sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
|
|
cur.execute(sql, {"status": status, "id": ix})
|
|
return "Updates for '{}' are now {}".format(title, state)
|
|
|
|
|
|
async def set_date(cur, url):
|
|
# print("set_date")
|
|
# time.sleep(1)
|
|
"""
|
|
Set last update date of feed
|
|
:param url: url of the feed
|
|
:return:
|
|
"""
|
|
today = date.today()
|
|
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
|
|
# cur = conn.cursor()
|
|
cur.execute(sql, {"today": today, "url": url})
|
|
|
|
|
|
async def add_entry_and_set_date(db_file, source, entry):
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
await add_entry(cur, entry)
|
|
await set_date(cur, source)
|
|
|
|
|
|
async def update_source_status(db_file, status, source):
|
|
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
cur.execute(sql, {"status": status, "scanned": date.today(), "url": source})
|
|
|
|
|
|
async def update_source_validity(db_file, source, valid):
|
|
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
cur.execute(sql, {"validity": valid, "url": source})
|
|
|
|
|
|
async def add_entry(cur, entry):
|
|
# print("add_entry")
|
|
# time.sleep(1)
|
|
"""
|
|
Add a new entry into the entries table
|
|
:param conn:
|
|
:param entry:
|
|
:return:
|
|
"""
|
|
sql = """ INSERT INTO entries(title,summary,link,source,read)
|
|
VALUES(?,?,?,?,?) """
|
|
# cur = conn.cursor()
|
|
cur.execute(sql, entry)
|
|
|
|
|
|
# This function doesn't work as expected with bbs and wiki feeds
|
|
async def remove_entry(db_file, source, length):
|
|
# print("remove_entry")
|
|
# time.sleep(1)
|
|
"""
|
|
Maintain list of entries
|
|
Check the number returned by feed and delete
|
|
existing entries up to the same returned amount
|
|
:param conn:
|
|
:param source:
|
|
:param length:
|
|
:return:
|
|
"""
|
|
# FIXED
|
|
# Dino empty titles are not counted https://dino.im/index.xml
|
|
# SOLVED
|
|
# Add text if is empty
|
|
# title = '*** No title ***' if not entry.title else entry.title
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
sql = "SELECT count(id) FROM entries WHERE source = ?"
|
|
count = cur.execute(sql, (source,))
|
|
count = cur.fetchone()[0]
|
|
limit = count - length
|
|
if limit:
|
|
limit = limit;
|
|
sql = """DELETE FROM entries WHERE id IN (
|
|
SELECT id FROM entries
|
|
WHERE source = :source
|
|
ORDER BY id
|
|
ASC LIMIT :limit)"""
|
|
cur.execute(sql, {"source": source, "limit": limit})
|
|
print('### removed', limit, 'from', source)
|
|
|
|
|
|
async def remove_nonexistent_entries(db_file, feed, source):
|
|
"""
|
|
Remove entries that don't exist in feed'
|
|
Check the entries returned from feed and delete
|
|
non existing entries
|
|
:param conn:
|
|
:param source:
|
|
:param length:
|
|
:return:
|
|
"""
|
|
async with DBLOCK:
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
sql = "SELECT id, title, link FROM entries WHERE source = ?"
|
|
cur.execute(sql, (source,))
|
|
entries_db = cur.fetchall()
|
|
# print('entries_db')
|
|
# print(entries_db)
|
|
for entry_db in entries_db:
|
|
# entry_db[1] = id
|
|
# entry_db[2] = title
|
|
# entry_db[3] = link
|
|
exist = False
|
|
# print("check-db")
|
|
for entry_feed in feed.entries:
|
|
# print("check-feed")
|
|
# TODO better check and don't repeat code
|
|
if entry_feed.has_key("title"):
|
|
title = entry_feed.title
|
|
else:
|
|
title = feed["feed"]["title"]
|
|
|
|
if entry_feed.has_key("link"):
|
|
link = entry_feed.link
|
|
else:
|
|
link = source
|
|
# TODO better check and don't repeat code
|
|
if entry_db[1] == title and entry_db[2] == link:
|
|
# print('exist')
|
|
# print(title)
|
|
exist = True
|
|
break
|
|
if not exist:
|
|
# print('>>> not exist')
|
|
# print(entry_db[1])
|
|
# TODO Send to table archive
|
|
# TODO Also make a regular/routine check for sources that have been changed (though that can only happen when manually editing)
|
|
sql = "DELETE FROM entries WHERE id = ?"
|
|
cur.execute(sql, (entry_db[0],))
|
|
# breakpoint()
|
|
|
|
|
|
async def get_subscriptions(db_file):
|
|
# print("get_subscriptions")
|
|
# time.sleep(1)
|
|
"""
|
|
Query feeds
|
|
:param conn:
|
|
:return: rows (tuple)
|
|
"""
|
|
with create_connection(db_file) as conn:
|
|
cur = conn.cursor()
|
|
sql = "SELECT address FROM feeds WHERE enabled = 1"
|
|
cur.execute(sql)
|
|
return cur.fetchall()
|
|
|
|
|
|
async def list_subscriptions(db_file):
|
|
# print("list_subscriptions")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Query feeds
|
|
:param conn:
|
|
:return: rows (string)
|
|
"""
|
|
with create_connection(db_file) as conn:
|
|
# cur = conn.cursor()
|
|
cur = get_cursor(db_file)
|
|
sql = "SELECT name, address, updated, id, enabled FROM feeds"
|
|
results = cur.execute(sql)
|
|
|
|
feeds_list = "List of subscriptions: \n"
|
|
counter = 0
|
|
for result in results:
|
|
counter += 1
|
|
feeds_list += """\n{} \n{} \nLast updated: {} \nID: {} [{}]
|
|
""".format(str(result[0]), str(result[1]), str(result[2]),
|
|
str(result[3]), str(result[4]))
|
|
if counter:
|
|
return feeds_list + "\n Total of {} subscriptions".format(counter)
|
|
else:
|
|
msg = ("List of subscriptions is empty. \n"
|
|
"To add feed, send a message as follows: \n"
|
|
"feed add URL \n"
|
|
"Example: \n"
|
|
"feed add https://reclaimthenet.org/feed/")
|
|
return msg
|
|
|
|
|
|
async def last_entries(db_file, num):
|
|
# print("last_entries")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Query feeds
|
|
:param conn:
|
|
:param num: integer
|
|
:return: rows (string)
|
|
"""
|
|
num = int(num)
|
|
if num > 50:
|
|
num = 50
|
|
elif num < 1:
|
|
num = 1
|
|
with create_connection(db_file) as conn:
|
|
# cur = conn.cursor()
|
|
cur = get_cursor(db_file)
|
|
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
|
|
results = cur.execute(sql, (num,))
|
|
|
|
|
|
titles_list = "Recent {} titles: \n".format(num)
|
|
for result in results:
|
|
titles_list += "\n{} \n{}".format(str(result[0]), str(result[1]))
|
|
return titles_list
|
|
|
|
|
|
async def search_entries(db_file, query):
|
|
# print("search_entries")
|
|
# print("db_file")
|
|
# print(db_file)
|
|
# time.sleep(1)
|
|
"""
|
|
Query feeds
|
|
:param conn:
|
|
:param query: string
|
|
:return: rows (string)
|
|
"""
|
|
if len(query) < 2:
|
|
return "Please enter at least 2 characters to search"
|
|
|
|
with create_connection(db_file) as conn:
|
|
# cur = conn.cursor()
|
|
cur = get_cursor(db_file)
|
|
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
|
|
results = cur.execute(sql, [f'%{query}%'])
|
|
|
|
results_list = "Search results for '{}': \n".format(query)
|
|
counter = 0
|
|
for result in results:
|
|
counter += 1
|
|
results_list += """\n{} \n{}
|
|
""".format(str(result[0]), str(result[1]))
|
|
if counter:
|
|
return results_list + "\n Total of {} results".format(counter)
|
|
else:
|
|
return "No results found for: {}".format(query)
|
|
|
|
|
|
async def check_entry_exist(db_file, title, link):
|
|
# print("check_entry")
|
|
# time.sleep(1)
|
|
"""
|
|
Check whether an entry exists
|
|
Query entries by title and link
|
|
:param conn:
|
|
:param link:
|
|
:param title:
|
|
:return: row
|
|
"""
|
|
cur = get_cursor(db_file)
|
|
sql = "SELECT id FROM entries WHERE title = :title and link = :link"
|
|
cur.execute(sql, {"title": title, "link": link})
|
|
return cur.fetchone()
|