Slixfeed/slixfeed/database.py
2023-07-16 15:23:44 +00:00

502 lines
16 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import sqlite3
from sqlite3 import Error
import asyncio
from datetime import date
import feedparser
from eliot import start_action, to_file
# aiosqlite
DBLOCK = asyncio.Lock()
CURSORS = {}
def create_connection(db_file):
# print("create_connection")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Create a database connection to the SQLite database
specified by db_file
:param db_file: database file
:return: Connection object or None
"""
with start_action(action_type="create_connection()", db=db_file):
conn = None
try:
conn = sqlite3.connect(db_file)
return conn
except Error as e:
print(e)
return conn
def create_tables(db_file):
# print("create_tables")
# print("db_file")
# print(db_file)
# time.sleep(1)
with start_action(action_type="create_tables()", db=db_file):
with create_connection(db_file) as conn:
feeds_table_sql = """
CREATE TABLE IF NOT EXISTS feeds (
id integer PRIMARY KEY,
name text,
address text NOT NULL,
enabled integer NOT NULL,
scanned text,
updated text,
status integer,
valid integer
); """
entries_table_sql = """
CREATE TABLE IF NOT EXISTS entries (
id integer PRIMARY KEY,
title text NOT NULL,
summary text NOT NULL,
link text NOT NULL,
source text,
read integer
); """
c = conn.cursor()
# c = get_cursor(db_file)
c.execute(feeds_table_sql)
c.execute(entries_table_sql)
def get_cursor(db_file):
"""
Allocate a cursor to connection per database.
:param db_file: database file
:return: Cursor
"""
with start_action(action_type="get_cursor()", db=db_file):
if db_file in CURSORS:
return CURSORS[db_file]
else:
with create_connection(db_file) as conn:
cur = conn.cursor()
CURSORS[db_file] = cur
return CURSORS[db_file]
async def add_feed(db_file, url, res):
# print("add_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Add a new feed into the feeds table
:param conn:
:param feed:
:return: string
"""
with start_action(action_type="add_feed()", url=url):
#TODO consider async with DBLOCK
#conn = create_connection(db_file)
# with create_connection(db_file) as conn:
# #exist = await check_feed(conn, url)
# exist = await check_feed(db_file, url)
# if not exist:
# res = await main.download_feed(url)
# else:
# return "News source is already listed in the subscription list"
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
if res[0]:
feed = feedparser.parse(res[0])
if feed.bozo:
feed = (url, 1, res[1], 0)
#sql = """INSERT INTO feeds(address,enabled,status,valid)
# VALUES(?,?,?,?) """
#cur.execute(sql, feed)
bozo = ("WARNING: Bozo detected. Failed to load URL.")
print(bozo)
return "Failed to parse URL as feed"
else:
title = feed["feed"]["title"]
feed = (title, url, 1, res[1], 1)
sql = """INSERT INTO feeds(name,address,enabled,status,valid)
VALUES(?,?,?,?,?) """
cur.execute(sql, feed)
else:
feed = (url, 1, res[1], 0)
#sql = "INSERT INTO feeds(address,enabled,status,valid) VALUES(?,?,?,?) "
#cur.execute(sql, feed)
return "Failed to get URL. HTTP Error {}".format(res[1])
source = title if title else '<' + url + '>'
msg = 'News source "{}" has been added to subscription list'.format(source)
return msg
async def remove_feed(db_file, ix):
# print("remove_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Delete a feed by feed id
:param conn:
:param id: id of the feed
:return: string
"""
with start_action(action_type="remove_feed()", id=ix):
with create_connection(db_file) as conn:
with DBLOCK:
cur = conn.cursor()
sql = "SELECT address FROM feeds WHERE id = ?"
url = cur.execute(sql, (ix,))
for i in url:
url = i[0]
# NOTE Should we move DBLOCK to this line? 2022-12-23
sql = "DELETE FROM entries WHERE source = ?"
cur.execute(sql, (url,))
sql = "DELETE FROM feeds WHERE id = ?"
cur.execute(sql, (ix,))
return """News source <{}> has been removed from subscription list
""".format(url)
async def check_feed(db_file, url):
# print("check_feed")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Check whether a feed exists
Query for feeds by url
:param conn:
:param url:
:return: row
"""
with start_action(action_type="check_feed()", url=url):
cur = get_cursor(db_file)
sql = "SELECT id FROM feeds WHERE address = ?"
cur.execute(sql, (url,))
return cur.fetchone()
async def get_unread(db_file):
# print("get_unread")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Check read status of entry
:param conn:
:param id: id of the entry
:return: string
"""
with start_action(action_type="get_unread()", db=db_file):
with create_connection(db_file) as conn:
entry = []
cur = conn.cursor()
# cur = get_cursor(db_file)
sql = "SELECT id FROM entries WHERE read = 0"
ix = cur.execute(sql).fetchone()
if ix is None:
return False
ix = ix[0]
sql = "SELECT title FROM entries WHERE id = :id"
cur.execute(sql, (ix,))
title = cur.fetchone()[0]
entry.append(title)
sql = "SELECT summary FROM entries WHERE id = :id"
cur.execute(sql, (ix,))
summary = cur.fetchone()[0]
entry.append(summary)
sql = "SELECT link FROM entries WHERE id = :id"
cur.execute(sql, (ix,))
link = cur.fetchone()[0]
entry.append(link)
entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2])
# print(entry)
async with DBLOCK:
await mark_as_read(cur, ix)
return entry
async def mark_as_read(cur, ix):
# print("mark_as_read", ix)
# time.sleep(1)
"""
Set read status of entry
:param cur:
:param ix: index of the entry
"""
with start_action(action_type="mark_as_read()", id=ix):
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
cur.execute(sql, (ix,))
# TODO mark_all_read for entries of feed
async def toggle_status(db_file, ix):
# print("toggle_status")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Set status of feed
:param conn:
:param id: id of the feed
:return: string
"""
with start_action(action_type="toggle_status()", db=db_file):
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
#cur = get_cursor(db_file)
sql = "SELECT name FROM feeds WHERE id = :id"
cur.execute(sql, (ix,))
title = cur.fetchone()[0]
sql = "SELECT enabled FROM feeds WHERE id = ?"
# NOTE [0][1][2]
cur.execute(sql, (ix,))
status = cur.fetchone()[0]
# FIXME always set to 1
# NOTE Maybe because is not integer
# TODO Reset feed table before further testing
if status == 1:
status = 0
notice = "News updates for '{}' are now disabled".format(title)
else:
status = 1
notice = "News updates for '{}' are now enabled".format(title)
sql = "UPDATE feeds SET enabled = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": ix})
return notice
async def set_date(cur, url):
# print("set_date")
# time.sleep(1)
"""
Set last update date of feed
:param url: url of the feed
:return:
"""
with start_action(action_type="set_date()", url=url):
today = date.today()
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
# cur = conn.cursor()
cur.execute(sql, {"today": today, "url": url})
async def add_entry_and_set_date(db_file, source, entry):
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
await add_entry(cur, entry)
await set_date(cur, source)
async def update_source_status(db_file, status, source):
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
cur.execute(sql, {"status": status, "scanned": date.today(), "url": source})
async def update_source_validity(db_file, source, valid):
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
cur.execute(sql, {"validity": valid, "url": source})
async def add_entry(cur, entry):
# print("add_entry")
# time.sleep(1)
"""
Add a new entry into the entries table
:param conn:
:param entry:
:return:
"""
with start_action(action_type="add_entry()", entry=entry):
sql = """ INSERT INTO entries(title,summary,link,source,read)
VALUES(?,?,?,?,?) """
# cur = conn.cursor()
cur.execute(sql, entry)
async def remove_entry(db_file, source, length):
# print("remove_entry")
# time.sleep(1)
"""
Maintain list of entries
Check the number returned by feed and delete
existing entries up to the same returned amount
:param conn:
:param source:
:param length:
:return:
"""
with start_action(action_type="remove_entry()", source=source):
# FIXED
# Dino empty titles are not counted https://dino.im/index.xml
# SOLVED
# Add text if is empty
# title = '*** No title ***' if not entry.title else entry.title
async with DBLOCK:
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT count(id) FROM entries WHERE source = ?"
count = cur.execute(sql, (source,))
count = cur.fetchone()[0]
limit = count - length
if limit:
limit = limit;
sql = """DELETE FROM entries WHERE id IN (
SELECT id FROM entries
WHERE source = :source
ORDER BY id
ASC LIMIT :limit)"""
cur.execute(sql, {"source": source, "limit": limit})
async def get_subscriptions(db_file):
# print("get_subscriptions")
# time.sleep(1)
"""
Query feeds
:param conn:
:return: rows (tuple)
"""
with start_action(action_type="get_subscriptions()"):
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT address FROM feeds WHERE enabled = 1"
cur.execute(sql)
return cur.fetchall()
async def list_subscriptions(db_file):
# print("list_subscriptions")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Query feeds
:param conn:
:return: rows (string)
"""
with start_action(action_type="list_subscriptions()", db=db_file):
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file)
sql = "SELECT name, address, updated, id, enabled FROM feeds"
results = cur.execute(sql)
feeds_list = "List of subscriptions: \n"
counter = 0
for result in results:
counter += 1
feeds_list += """\n{} \n{} \nLast updated: {} \nID: {} [{}]
""".format(str(result[0]), str(result[1]), str(result[2]),
str(result[3]), str(result[4]))
if counter:
return feeds_list + "\n Total of {} subscriptions".format(counter)
else:
msg = ("List of subscriptions is empty. \n"
"To add feed, send a message as follows: \n"
"feed add URL \n"
"Example: \n"
"feed add https://reclaimthenet.org/feed/")
return msg
async def last_entries(db_file, num):
# print("last_entries")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Query feeds
:param conn:
:param num: integer
:return: rows (string)
"""
with start_action(action_type="last_entries()", num=num):
num = int(num)
if num > 50:
num = 50
elif num < 1:
num = 1
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
results = cur.execute(sql, (num,))
titles_list = "Recent {} titles: \n".format(num)
for result in results:
titles_list += "\n{} \n{}".format(str(result[0]), str(result[1]))
return titles_list
async def search_entries(db_file, query):
# print("search_entries")
# print("db_file")
# print(db_file)
# time.sleep(1)
"""
Query feeds
:param conn:
:param query: string
:return: rows (string)
"""
with start_action(action_type="search_entries()", query=query):
if len(query) < 2:
return "Please enter at least 2 characters to search"
with create_connection(db_file) as conn:
# cur = conn.cursor()
cur = get_cursor(db_file)
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
results = cur.execute(sql, [f'%{query}%'])
results_list = "Search results for '{}': \n".format(query)
counter = 0
for result in results:
counter += 1
results_list += """\n{} \n{}
""".format(str(result[0]), str(result[1]))
if counter:
return results_list + "\n Total of {} results".format(counter)
else:
return "No results found for: {}".format(query)
async def check_entry(db_file, title, link):
# print("check_entry")
# time.sleep(1)
"""
Check whether an entry exists
Query entries by title and link
:param conn:
:param link:
:param title:
:return: row
"""
with start_action(action_type="check_entry()", link=link):
with create_connection(db_file) as conn:
cur = conn.cursor()
sql = "SELECT id FROM entries WHERE title = :title and link = :link"
cur.execute(sql, {"title": title, "link": link})
return cur.fetchone()