From ec7abb5e902be11584163b5f30c0102116618e03 Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Sun, 16 Jul 2023 15:23:44 +0000 Subject: [PATCH] Fix database lock error (Laura) --- slixfeed/database.py | 502 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100644 slixfeed/database.py diff --git a/slixfeed/database.py b/slixfeed/database.py new file mode 100644 index 0000000..2d1dcbf --- /dev/null +++ b/slixfeed/database.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import sqlite3 +from sqlite3 import Error + +import asyncio + +from datetime import date +import feedparser + +from eliot import start_action, to_file + +# aiosqlite +DBLOCK = asyncio.Lock() + +CURSORS = {} + +def create_connection(db_file): + # print("create_connection") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Create a database connection to the SQLite database + specified by db_file + :param db_file: database file + :return: Connection object or None + """ + with start_action(action_type="create_connection()", db=db_file): + conn = None + try: + conn = sqlite3.connect(db_file) + return conn + except Error as e: + print(e) + return conn + + +def create_tables(db_file): + # print("create_tables") + # print("db_file") + # print(db_file) + # time.sleep(1) + with start_action(action_type="create_tables()", db=db_file): + with create_connection(db_file) as conn: + feeds_table_sql = """ + CREATE TABLE IF NOT EXISTS feeds ( + id integer PRIMARY KEY, + name text, + address text NOT NULL, + enabled integer NOT NULL, + scanned text, + updated text, + status integer, + valid integer + ); """ + entries_table_sql = """ + CREATE TABLE IF NOT EXISTS entries ( + id integer PRIMARY KEY, + title text NOT NULL, + summary text NOT NULL, + link text NOT NULL, + source text, + read integer + ); """ + + c = conn.cursor() + # c = get_cursor(db_file) + c.execute(feeds_table_sql) + c.execute(entries_table_sql) + + +def get_cursor(db_file): + """ + Allocate a cursor to connection per database. + :param db_file: database file + :return: Cursor + """ + with start_action(action_type="get_cursor()", db=db_file): + if db_file in CURSORS: + return CURSORS[db_file] + else: + with create_connection(db_file) as conn: + cur = conn.cursor() + CURSORS[db_file] = cur + return CURSORS[db_file] + + +async def add_feed(db_file, url, res): + # print("add_feed") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Add a new feed into the feeds table + :param conn: + :param feed: + :return: string + """ + with start_action(action_type="add_feed()", url=url): + #TODO consider async with DBLOCK + #conn = create_connection(db_file) + + # with create_connection(db_file) as conn: + # #exist = await check_feed(conn, url) + # exist = await check_feed(db_file, url) + + # if not exist: + # res = await main.download_feed(url) + # else: + # return "News source is already listed in the subscription list" + + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + if res[0]: + feed = feedparser.parse(res[0]) + if feed.bozo: + feed = (url, 1, res[1], 0) + #sql = """INSERT INTO feeds(address,enabled,status,valid) + # VALUES(?,?,?,?) """ + #cur.execute(sql, feed) + bozo = ("WARNING: Bozo detected. Failed to load URL.") + print(bozo) + return "Failed to parse URL as feed" + else: + title = feed["feed"]["title"] + feed = (title, url, 1, res[1], 1) + sql = """INSERT INTO feeds(name,address,enabled,status,valid) + VALUES(?,?,?,?,?) """ + cur.execute(sql, feed) + else: + feed = (url, 1, res[1], 0) + #sql = "INSERT INTO feeds(address,enabled,status,valid) VALUES(?,?,?,?) " + #cur.execute(sql, feed) + return "Failed to get URL. HTTP Error {}".format(res[1]) + + source = title if title else '<' + url + '>' + msg = 'News source "{}" has been added to subscription list'.format(source) + return msg + + +async def remove_feed(db_file, ix): + # print("remove_feed") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Delete a feed by feed id + :param conn: + :param id: id of the feed + :return: string + """ + with start_action(action_type="remove_feed()", id=ix): + with create_connection(db_file) as conn: + with DBLOCK: + cur = conn.cursor() + sql = "SELECT address FROM feeds WHERE id = ?" + url = cur.execute(sql, (ix,)) + for i in url: + url = i[0] + # NOTE Should we move DBLOCK to this line? 2022-12-23 + sql = "DELETE FROM entries WHERE source = ?" + cur.execute(sql, (url,)) + sql = "DELETE FROM feeds WHERE id = ?" + cur.execute(sql, (ix,)) + return """News source <{}> has been removed from subscription list + """.format(url) + + +async def check_feed(db_file, url): + # print("check_feed") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Check whether a feed exists + Query for feeds by url + :param conn: + :param url: + :return: row + """ + with start_action(action_type="check_feed()", url=url): + cur = get_cursor(db_file) + sql = "SELECT id FROM feeds WHERE address = ?" + cur.execute(sql, (url,)) + return cur.fetchone() + + +async def get_unread(db_file): + # print("get_unread") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Check read status of entry + :param conn: + :param id: id of the entry + :return: string + """ + with start_action(action_type="get_unread()", db=db_file): + with create_connection(db_file) as conn: + entry = [] + cur = conn.cursor() + # cur = get_cursor(db_file) + sql = "SELECT id FROM entries WHERE read = 0" + ix = cur.execute(sql).fetchone() + if ix is None: + return False + ix = ix[0] + sql = "SELECT title FROM entries WHERE id = :id" + cur.execute(sql, (ix,)) + title = cur.fetchone()[0] + entry.append(title) + sql = "SELECT summary FROM entries WHERE id = :id" + cur.execute(sql, (ix,)) + summary = cur.fetchone()[0] + entry.append(summary) + sql = "SELECT link FROM entries WHERE id = :id" + cur.execute(sql, (ix,)) + link = cur.fetchone()[0] + entry.append(link) + entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2]) + # print(entry) + async with DBLOCK: + await mark_as_read(cur, ix) + return entry + + +async def mark_as_read(cur, ix): + # print("mark_as_read", ix) + # time.sleep(1) + """ + Set read status of entry + :param cur: + :param ix: index of the entry + """ + with start_action(action_type="mark_as_read()", id=ix): + sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?" + cur.execute(sql, (ix,)) + + +# TODO mark_all_read for entries of feed +async def toggle_status(db_file, ix): + # print("toggle_status") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Set status of feed + :param conn: + :param id: id of the feed + :return: string + """ + with start_action(action_type="toggle_status()", db=db_file): + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + #cur = get_cursor(db_file) + sql = "SELECT name FROM feeds WHERE id = :id" + cur.execute(sql, (ix,)) + title = cur.fetchone()[0] + sql = "SELECT enabled FROM feeds WHERE id = ?" + # NOTE [0][1][2] + cur.execute(sql, (ix,)) + status = cur.fetchone()[0] + # FIXME always set to 1 + # NOTE Maybe because is not integer + # TODO Reset feed table before further testing + if status == 1: + status = 0 + notice = "News updates for '{}' are now disabled".format(title) + else: + status = 1 + notice = "News updates for '{}' are now enabled".format(title) + sql = "UPDATE feeds SET enabled = :status WHERE id = :id" + cur.execute(sql, {"status": status, "id": ix}) + return notice + + +async def set_date(cur, url): + # print("set_date") + # time.sleep(1) + """ + Set last update date of feed + :param url: url of the feed + :return: + """ + with start_action(action_type="set_date()", url=url): + today = date.today() + sql = "UPDATE feeds SET updated = :today WHERE address = :url" + # cur = conn.cursor() + cur.execute(sql, {"today": today, "url": url}) + + +async def add_entry_and_set_date(db_file, source, entry): + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + await add_entry(cur, entry) + await set_date(cur, source) + + +async def update_source_status(db_file, status, source): + sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url" + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + cur.execute(sql, {"status": status, "scanned": date.today(), "url": source}) + + +async def update_source_validity(db_file, source, valid): + sql = "UPDATE feeds SET valid = :validity WHERE address = :url" + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + cur.execute(sql, {"validity": valid, "url": source}) + + +async def add_entry(cur, entry): + # print("add_entry") + # time.sleep(1) + """ + Add a new entry into the entries table + :param conn: + :param entry: + :return: + """ + with start_action(action_type="add_entry()", entry=entry): + sql = """ INSERT INTO entries(title,summary,link,source,read) + VALUES(?,?,?,?,?) """ + # cur = conn.cursor() + cur.execute(sql, entry) + + +async def remove_entry(db_file, source, length): + # print("remove_entry") + # time.sleep(1) + """ + Maintain list of entries + Check the number returned by feed and delete + existing entries up to the same returned amount + :param conn: + :param source: + :param length: + :return: + """ + with start_action(action_type="remove_entry()", source=source): + # FIXED + # Dino empty titles are not counted https://dino.im/index.xml + # SOLVED + # Add text if is empty + # title = '*** No title ***' if not entry.title else entry.title + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = "SELECT count(id) FROM entries WHERE source = ?" + count = cur.execute(sql, (source,)) + count = cur.fetchone()[0] + limit = count - length + if limit: + limit = limit; + sql = """DELETE FROM entries WHERE id IN ( + SELECT id FROM entries + WHERE source = :source + ORDER BY id + ASC LIMIT :limit)""" + cur.execute(sql, {"source": source, "limit": limit}) + + +async def get_subscriptions(db_file): + # print("get_subscriptions") + # time.sleep(1) + """ + Query feeds + :param conn: + :return: rows (tuple) + """ + with start_action(action_type="get_subscriptions()"): + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = "SELECT address FROM feeds WHERE enabled = 1" + cur.execute(sql) + return cur.fetchall() + + +async def list_subscriptions(db_file): + # print("list_subscriptions") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Query feeds + :param conn: + :return: rows (string) + """ + with start_action(action_type="list_subscriptions()", db=db_file): + with create_connection(db_file) as conn: + # cur = conn.cursor() + cur = get_cursor(db_file) + sql = "SELECT name, address, updated, id, enabled FROM feeds" + results = cur.execute(sql) + + feeds_list = "List of subscriptions: \n" + counter = 0 + for result in results: + counter += 1 + feeds_list += """\n{} \n{} \nLast updated: {} \nID: {} [{}] + """.format(str(result[0]), str(result[1]), str(result[2]), + str(result[3]), str(result[4])) + if counter: + return feeds_list + "\n Total of {} subscriptions".format(counter) + else: + msg = ("List of subscriptions is empty. \n" + "To add feed, send a message as follows: \n" + "feed add URL \n" + "Example: \n" + "feed add https://reclaimthenet.org/feed/") + return msg + + +async def last_entries(db_file, num): + # print("last_entries") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Query feeds + :param conn: + :param num: integer + :return: rows (string) + """ + with start_action(action_type="last_entries()", num=num): + num = int(num) + if num > 50: + num = 50 + elif num < 1: + num = 1 + with create_connection(db_file) as conn: + # cur = conn.cursor() + cur = get_cursor(db_file) + sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num" + results = cur.execute(sql, (num,)) + + + titles_list = "Recent {} titles: \n".format(num) + for result in results: + titles_list += "\n{} \n{}".format(str(result[0]), str(result[1])) + return titles_list + + +async def search_entries(db_file, query): + # print("search_entries") + # print("db_file") + # print(db_file) + # time.sleep(1) + """ + Query feeds + :param conn: + :param query: string + :return: rows (string) + """ + with start_action(action_type="search_entries()", query=query): + if len(query) < 2: + return "Please enter at least 2 characters to search" + + with create_connection(db_file) as conn: + # cur = conn.cursor() + cur = get_cursor(db_file) + sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50" + results = cur.execute(sql, [f'%{query}%']) + + results_list = "Search results for '{}': \n".format(query) + counter = 0 + for result in results: + counter += 1 + results_list += """\n{} \n{} + """.format(str(result[0]), str(result[1])) + if counter: + return results_list + "\n Total of {} results".format(counter) + else: + return "No results found for: {}".format(query) + + +async def check_entry(db_file, title, link): + # print("check_entry") + # time.sleep(1) + """ + Check whether an entry exists + Query entries by title and link + :param conn: + :param link: + :param title: + :return: row + """ + with start_action(action_type="check_entry()", link=link): + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = "SELECT id FROM entries WHERE title = :title and link = :link" + cur.execute(sql, {"title": title, "link": link}) + return cur.fetchone()