Slixfeed/slixfeed/database.py
Schimon Jehudah ef5f38c031 Add Settings
2023-10-10 13:42:36 +00:00

675 lines
20 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sqlite3
import asyncio
from sqlite3 import Error
from datetime import date
import settings
# from eliot import start_action, to_file
# # with start_action(action_type="list_subscriptions()", db=db_file):
# # with start_action(action_type="last_entries()", num=num):
# # with start_action(action_type="get_subscriptions()"):
# # with start_action(action_type="remove_entry()", source=source):
# # with start_action(action_type="search_entries()", query=query):
# # with start_action(action_type="check_entry()", link=link):
# aiosqlite
DBLOCK = asyncio.Lock()
CURSORS = {}
def create_connection(db_file):
"""
Create a database connection to the SQLite database
specified by db_file.
:param db_file: Database filename.
:return: Connection object or None.
"""
conn = None
try:
conn = sqlite3.connect(db_file)
return conn
except Error as e:
print(e)
return conn
def create_tables(db_file):
"""
Create SQLite tables.
:param db_file: Database filename.
"""
with create_connection(db_file) as conn:
feeds_table_sql = """
CREATE TABLE IF NOT EXISTS feeds (
id integer PRIMARY KEY,
name text,
address text NOT NULL,
enabled integer NOT NULL,
scanned text,
updated text,
status integer,
valid integer
); """
entries_table_sql = """
CREATE TABLE IF NOT EXISTS entries (
id integer PRIMARY KEY,
title text NOT NULL,
summary text NOT NULL,
link text NOT NULL,
source text,
read integer
); """
# statistics_table_sql = """
# CREATE TABLE IF NOT EXISTS statistics (
# id integer PRIMARY KEY,
# title text NOT NULL,
# number integer
# ); """
settings_table_sql = """
CREATE TABLE IF NOT EXISTS settings (
id integer PRIMARY KEY,
key text NOT NULL,
value integer
); """
cur = conn.cursor()
# cur = get_cursor(db_file)
cur.execute(feeds_table_sql)
cur.execute(entries_table_sql)
# cur.execute(statistics_table_sql)
cur.execute(settings_table_sql)
def get_cursor(db_file):
"""
Allocate a cursor to connection per database.
:param db_file: Database filename.
:return: Cursor.
"""
if db_file in CURSORS:
return CURSORS[db_file]
else:
with create_connection(db_file) as conn:
cur = conn.cursor()
CURSORS[db_file] = cur
return CURSORS[db_file]
async def add_feed(db_file, feed, url, res):
"""
Add a new feed into the feeds table.
:param db_file: Database filename.
:param feed: Parsed XML document.
:param url: URL.
:param res: XML document.
:return: Message.
"""
#TODO consider async with DBLOCK
#conn = create_connection(db_file)
# with create_connection(db_file) as conn:
# #exist = await check_feed_exist(conn, url)
# exist = await check_feed_exist(db_file, url)
# if not exist:
# res = await main.download_feed(url)
# else:
# return "News source is already listed in the subscription list"
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
title = feed["feed"]["title"]
feed = (title, url, 1, res[1], 1)
sql = """INSERT INTO feeds(name, address, enabled, status, valid)
VALUES(?, ?, ?, ?, ?) """
cur.execute(sql, feed)
source = title if title else '<' + url + '>'
msg = '> {}\nNews source "{}" has been added to subscription list'.format(url, source)
return msg
async def remove_feed(db_file, ix):
"""
Delete a feed by feed id.
:param db_file: Database filename.
:param ix: Index of feed.
:return: Message.
"""
with create_connection(db_file) as conn:
async with DBLOCK:
cur = conn.cursor()
try:
sql = "SELECT address FROM feeds WHERE id = ?"
url = cur.execute(sql, (ix,))
for i in url:
url = i[0]
# NOTE Should we move DBLOCK to this line? 2022-12-23
sql = "DELETE FROM entries WHERE source = ?"
cur.execute(sql, (url,))
sql = "DELETE FROM feeds WHERE id = ?"
cur.execute(sql, (ix,))
msg = """News source <{}> has been removed from subscription list
""".format(url)
except:
msg = """No news source with ID {}""".format(ix)
return msg
async def check_feed_exist(db_file, url):
"""
Check whether a feed exists.
Query for feeds by given url.
:param db_file: Database filename.
:param url: URL.
:return: Index ID and Name or None.
"""
cur = get_cursor(db_file)
sql = "SELECT id, name FROM feeds WHERE address = ?"
cur.execute(sql, (url,))
result = cur.fetchone()
return result
async def get_number_of_items(db_file, str):
"""
Return number of entries or feeds.
:param cur: Cursor object.
:param str: "entries" or "feeds".
:return: Number of rows.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT count(id) FROM {}".format(str)
count = cur.execute(sql)
count = cur.fetchone()[0]
return count
async def get_number_of_entries_unread(db_file):
"""
Return number of unread items.
:param db_file: Database filename.
:param cur: Cursor object.
:return: Number of rows.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT count(id) FROM entries WHERE read = 0"
count = cur.execute(sql)
count = cur.fetchone()[0]
return count
async def get_entry_unread(db_file):
"""
Check read status of entry.
:param db_file: Database filename.
:return: News item as message.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
entry = []
sql = "SELECT id FROM entries WHERE read = 0"
ix = cur.execute(sql).fetchone()
if ix is None:
return False
ix = ix[0]
sql = "SELECT title FROM entries WHERE id = :id"
cur.execute(sql, (ix,))
title = cur.fetchone()[0]
entry.append(title)
sql = "SELECT summary FROM entries WHERE id = :id"
cur.execute(sql, (ix,))
summary = cur.fetchone()[0]
entry.append(summary)
sql = "SELECT link FROM entries WHERE id = :id"
cur.execute(sql, (ix,))
link = cur.fetchone()[0]
entry.append(link)
entry = "{}\n\n{}\n\n{}".format(entry[0], entry[1], entry[2])
# print(entry)
async with DBLOCK:
await mark_as_read(cur, ix)
# async with DBLOCK:
# await update_statistics(db_file)
return entry
async def mark_as_read(cur, ix):
"""
Set read status of entry.
:param cur: Cursor object.
:param ix: Index of entry.
"""
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
cur.execute(sql, (ix,))
async def statistics(db_file):
"""
Return table statistics.
:param db_file: Database filename.
:return: News item as message.
"""
feeds = await get_number_of_items(db_file, 'feeds')
entries = await get_number_of_items(db_file, 'entries')
unread_entries = await get_number_of_entries_unread(db_file)
msg = "You have {} unread news items out of {} from {} news sources.".format(unread_entries, entries, feeds)
return msg
async def update_statistics(cur):
"""
Update table statistics.
:param cur: Cursor object.
"""
stat_dict = {}
stat_dict["feeds"] = await get_number_of_items(cur, 'feeds')
stat_dict["entries"] = await get_number_of_items(cur, 'entries')
stat_dict["unread"] = await get_number_of_entries_unread(cur=cur)
for i in stat_dict:
sql = "SELECT id FROM statistics WHERE title = ?"
cur.execute(sql, (i,))
if cur.fetchone():
sql = "UPDATE statistics SET number = :num WHERE title = :title"
cur.execute(sql, {"title": i, "num": stat_dict[i]})
else:
sql = "SELECT count(id) FROM statistics"
count = cur.execute(sql)
count = cur.fetchone()[0]
ix = count + 1
sql = "INSERT INTO statistics VALUES(?,?,?)"
cur.execute(sql, (ix, i, stat_dict[i]))
# TODO mark_all_read for entries of feed
async def toggle_status(db_file, ix):
"""
Toggle status of feed.
:param db_file: Database filename.
:param ix: Index of entry.
:return: Message
"""
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
#cur = get_cursor(db_file)
sql = "SELECT name FROM feeds WHERE id = :id"
cur.execute(sql, (ix,))
title = cur.fetchone()[0]
sql = "SELECT enabled FROM feeds WHERE id = ?"
# NOTE [0][1][2]
cur.execute(sql, (ix,))
status = cur.fetchone()[0]
# FIXME always set to 1
# NOTE Maybe because is not integer
# TODO Reset feed table before further testing
if status == 1:
status = 0
state = "disabled"
else:
status = 1
state = "enabled"
sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": ix})
msg = "Updates for '{}' are now {}".format(title, state)
return msg
async def set_date(cur, url):
"""
Set last update date of feed.
:param cur: Cursor object.
:param url: URL.
"""
today = date.today()
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
# cur = conn.cursor()
cur.execute(sql, {"today": today, "url": url})
async def add_entry_and_set_date(db_file, source, entry):
"""
TODO
"""
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
await add_entry(cur, entry)
await set_date(cur, source)
async def update_source_status(db_file, status, source):
"""
TODO
"""
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
cur.execute(sql, {"status": status, "scanned": date.today(), "url": source})
async def update_source_validity(db_file, source, valid):
"""
TODO
"""
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
cur.execute(sql, {"validity": valid, "url": source})
async def add_entry(cur, entry):
"""
Add a new entry into the entries table.
:param cur: Cursor object.
:param entry:
"""
sql = """ INSERT INTO entries(title, summary, link, source, read)
VALUES(?, ?, ?, ?, ?) """
cur.execute(sql, entry)
# This function doesn't work as expected with bbs and wiki feeds
async def remove_entry(db_file, source, length):
"""
Maintain list of entries equal to feed.
Check the number returned by feed and delete
existing entries up to the same returned amount.
:param db_file: Database filename.
:param source:
:param length:
:return:
"""
# FIXED
# Dino empty titles are not counted https://dino.im/index.xml
# SOLVED
# Add text if is empty
# title = '*** No title ***' if not entry.title else entry.title
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT count(id) FROM entries WHERE source = ?"
count = cur.execute(sql, (source,))
count = cur.fetchone()[0]
limit = count - length
if limit:
limit = limit;
sql = """DELETE FROM entries WHERE id IN (
SELECT id FROM entries
WHERE source = :source
ORDER BY id
ASC LIMIT :limit)"""
cur.execute(sql, {"source": source, "limit": limit})
async def remove_nonexistent_entries(db_file, feed, source):
"""
Remove entries that don't exist in a given parsed feed.
Check the entries returned from feed and delete non
existing entries
:param db_file: Database filename.
:param feed: URL of parsed feed.
:param source: URL of associated feed.
"""
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT id, title, link FROM entries WHERE source = ?"
cur.execute(sql, (source,))
entries_db = cur.fetchall()
for entry_db in entries_db:
exist = False
for entry_feed in feed.entries:
# TODO better check and don't repeat code
if entry_feed.has_key("title"):
title = entry_feed.title
else:
title = feed["feed"]["title"]
if entry_feed.has_key("link"):
link = entry_feed.link
else:
link = source
# TODO better check and don't repeat code
if entry_db[1] == title and entry_db[2] == link:
exist = True
break
if not exist:
# TODO Send to table archive
# TODO Also make a regular/routine check for sources that have been changed (though that can only happen when manually editing)
sql = "DELETE FROM entries WHERE id = ?"
cur.execute(sql, (entry_db[0],))
async def get_subscriptions(db_file):
"""
Query table feeds.
:param db_file: Database filename.
:return: List of feeds.
"""
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT address FROM feeds WHERE enabled = 1"
cur.execute(sql)
result = cur.fetchall()
return result
async def list_subscriptions(db_file):
"""
Query table feeds and list items.
:param db_file: Database filename.
:return: List of feeds.
"""
cur = get_cursor(db_file)
sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql)
feeds_list = "List of subscriptions: \n"
counter = 0
for result in results:
counter += 1
feeds_list += """\n{} \n{} \nLast updated: {} \nID: {} [{}]
""".format(str(result[0]), str(result[1]), str(result[2]),
str(result[3]), str(result[4]))
if counter:
return feeds_list + "\n Total of {} subscriptions".format(counter)
else:
msg = ("List of subscriptions is empty. \n"
"To add feed, send a message as follows: \n"
"feed add URL \n"
"Example: \n"
"add https://reclaimthenet.org/feed/")
return msg
async def last_entries(db_file, num):
"""
Query entries
:param db_file: Database filename.
:param num: Number
:return: List of recent N entries
"""
num = int(num)
if num > 50:
num = 50
elif num < 1:
num = 1
cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
results = cur.execute(sql, (num,))
titles_list = "Recent {} titles: \n".format(num)
for result in results:
titles_list += "\n{} \n{}".format(str(result[0]), str(result[1]))
return titles_list
async def search_entries(db_file, query):
"""
Query entries
:param db_file: Database filename.
:param query: Search query
:return: Entries with specified keywords
"""
if len(query) < 2:
return "Please enter at least 2 characters to search"
cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
results = cur.execute(sql, [f'%{query}%'])
results_list = "Search results for '{}': \n".format(query)
counter = 0
for result in results:
counter += 1
results_list += """\n{} \n{}
""".format(str(result[0]), str(result[1]))
if counter:
return results_list + "\n Total of {} results".format(counter)
else:
return "No results found for: {}".format(query)
async def check_entry_exist(db_file, title, link):
"""
Check whether an entry exists.
Query entries by title and link.
:param db_file: Database filename.
:param link: Entry URL.
:param title: Entry title.
:return: Index ID or None.
"""
cur = get_cursor(db_file)
sql = "SELECT id FROM entries WHERE title = :title and link = :link"
cur.execute(sql, {"title": title, "link": link})
result = cur.fetchone()
return result
# TODO dictionary
# settings = {
# "enabled" : {
# "message": "Updates are {}".format(status),
# "value": val
# },
# "interval" : {
# "message": "Updates will be sent every {} minutes".format(val),
# "value": val
# },
# "quantom" : {
# "message": "Every updates will contain {} news items".format(val),
# "value": val
# }
# }
async def set_settings_value(db_file, key_value):
"""
Set settings value.
:param db_file: Database filename.
:param key_value: List of key ("enabled", "interval", "quantum") and value (Integer).
:return: Message.
"""
if isinstance(key_value, list):
key = key_value[0]
val = key_value[1]
elif key_value == "enable":
key = "enabled"
val = 1
else:
key = "enabled"
val = 0
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
await set_settings_value_default(cur, key)
sql = "UPDATE settings SET value = :value WHERE key = :key"
cur.execute(sql, {"key": key, "value": val})
if key == 'quantum':
msg = "Every update will contain {} news items.".format(val)
elif key == 'interval':
msg = "Updates will be sent every {} minutes.".format(val)
else:
msg = "Updates are {}d.".format(key_value)
return msg
async def set_settings_value_default(cur, key):
# async def set_settings_value_default(cur):
# keys = ["enabled", "interval", "quantum"]
# for i in keys:
# sql = "SELECT id FROM settings WHERE key = ?"
# cur.execute(sql, (i,))
# if not cur.fetchone():
# val = await settings.get_value_default(i)
# sql = "INSERT INTO settings(key,value) VALUES(?,?)"
# cur.execute(sql, (i, val))
sql = "SELECT id FROM settings WHERE key = ?"
cur.execute(sql, (key,))
if not cur.fetchone():
val = await settings.get_value_default(key)
sql = "INSERT INTO settings(key,value) VALUES(?,?)"
cur.execute(sql, (key, val))
return val
async def get_settings_value(db_file, key):
"""
Get settings value.
:param db_file: Database filename.
:param key: "enabled", "interval", "quantum".
"""
# try:
# with create_connection(db_file) as conn:
# cur = conn.cursor()
# sql = "SELECT value FROM settings WHERE key = ?"
# cur.execute(sql, (key,))
# result = cur.fetchone()
# except:
# result = await settings.get_value_default(key)
# if not result:
# result = await settings.get_value_default(key)
# return result
with create_connection(db_file) as conn:
try:
cur = conn.cursor()
sql = "SELECT value FROM settings WHERE key = ?"
cur.execute(sql, (key,))
result = cur.fetchone()[0]
except:
result = await set_settings_value_default(cur, key)
if not result:
result = await set_settings_value_default(cur, key)
return result