forked from sch/Slixfeed
Add statistics
This commit is contained in:
Schimon Jehudah
parent
cf44241698
commit
aa8c35d728
2 changed files with 351 additions and 316 deletions
|
|
@ -43,8 +43,8 @@ import database
|
|||
|
||||
class Slixfeed(slixmpp.ClientXMPP):
|
||||
"""
|
||||
Slixmpp bot that will send updates of feeds it
|
||||
receives.
|
||||
Slixmpp news bot that will send updates
|
||||
from feeds it receives.
|
||||
"""
|
||||
def __init__(self, jid, password):
|
||||
slixmpp.ClientXMPP.__init__(self, jid, password)
|
||||
|
|
@ -66,7 +66,6 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
self.add_event_handler("disconnected", self.reconnect)
|
||||
|
||||
async def start(self, event):
|
||||
# print("start")
|
||||
"""
|
||||
Process the session_start event.
|
||||
|
||||
|
|
@ -83,8 +82,6 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
await self.get_roster()
|
||||
|
||||
async def message(self, msg):
|
||||
# print("message")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Process incoming message stanzas. Be aware that this also
|
||||
includes MUC messages and error messages. It is usually
|
||||
|
|
@ -99,49 +96,34 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
if msg['type'] in ('chat', 'normal'):
|
||||
message = " ".join(msg['body'].split())
|
||||
if message.lower().startswith('help'):
|
||||
print("COMMAND: help")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = print_help()
|
||||
# NOTE: Might not need it
|
||||
elif message.lower().startswith('feed recent '):
|
||||
print("COMMAND: feed recent")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = await initdb(msg['from'].bare, database.last_entries, message[12:])
|
||||
elif message.lower().startswith('feed search '):
|
||||
print("COMMAND: feed search")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = await initdb( msg['from'].bare, database.search_entries, message[12:])
|
||||
elif message.lower().startswith('feed list'):
|
||||
print("COMMAND: feed list")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
elif message.lower().startswith('recent '):
|
||||
action = await initdb(msg['from'].bare, database.last_entries, message[7:])
|
||||
elif message.lower().startswith('search '):
|
||||
action = await initdb( msg['from'].bare, database.search_entries, message[7:])
|
||||
elif message.lower().startswith('list'):
|
||||
action = await initdb(msg['from'].bare, database.list_subscriptions)
|
||||
elif message.lower().startswith('feed add '):
|
||||
print("COMMAND: feed add")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = await initdb(msg['from'].bare, add_feed, message[9:])
|
||||
elif message.lower().startswith('feed remove '):
|
||||
print("COMMAND: feed remove")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = await initdb(msg['from'].bare, database.remove_feed, message[12:])
|
||||
elif message.lower().startswith('feed status '):
|
||||
print("COMMAND: feed status")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = await initdb(msg['from'].bare, database.toggle_status, message[12:])
|
||||
elif message.lower().startswith('add '):
|
||||
action = await initdb(msg['from'].bare, add_feed, message[4:])
|
||||
elif message.lower().startswith('remove '):
|
||||
action = await initdb(msg['from'].bare, database.remove_feed, message[7:])
|
||||
elif message.lower().startswith('status '):
|
||||
action = await initdb(msg['from'].bare, database.toggle_status, message[7:])
|
||||
elif message.lower().startswith('unread'):
|
||||
action = await initdb(msg['from'].bare, database.statistics)
|
||||
elif message.lower().startswith('enable'):
|
||||
print("COMMAND: enable")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = toggle_state(msg['from'].bare, True)
|
||||
elif message.lower().startswith('disable'):
|
||||
print("COMMAND: disable")
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
action = toggle_state(msg['from'].bare, False)
|
||||
else:
|
||||
action = 'Unknown command. Press "help" for list of commands'
|
||||
msg.reply(action).send()
|
||||
|
||||
print("COMMAND:", message)
|
||||
print("ACCOUNT: " + str(msg['from']))
|
||||
|
||||
async def check_updates(self, event):
|
||||
# print("check_updates")
|
||||
# time.sleep(1)
|
||||
while True:
|
||||
print("Checking update")
|
||||
db_dir = get_default_dbdir()
|
||||
|
|
@ -156,13 +138,11 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
files = os.listdir(db_dir)
|
||||
for file in files:
|
||||
jid = file[:-3]
|
||||
print("download_updates",jid)
|
||||
await initdb(jid, download_updates)
|
||||
# await asyncio.sleep(9)
|
||||
await asyncio.sleep(90)
|
||||
|
||||
async def send_update(self, event):
|
||||
# print("send_update")
|
||||
# time.sleep(1)
|
||||
while True:
|
||||
db_dir = get_default_dbdir()
|
||||
if not os.path.isdir(db_dir):
|
||||
|
|
@ -178,21 +158,39 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
for file in files:
|
||||
if not file.endswith('.db-jour.db'):
|
||||
jid = file[:-3]
|
||||
print("get_entry_unread",jid)
|
||||
|
||||
new = await initdb(
|
||||
jid,
|
||||
database.get_unread
|
||||
database.get_entry_unread
|
||||
)
|
||||
|
||||
if new:
|
||||
# NOTE Consider send_message
|
||||
msg = self.make_message(
|
||||
msg = self.send_message(
|
||||
mto=jid,
|
||||
mbody=new,
|
||||
mtype='chat'
|
||||
)
|
||||
|
||||
msg.send()
|
||||
unread = await initdb(
|
||||
jid,
|
||||
database.get_number_of_entries_unread
|
||||
)
|
||||
|
||||
if unread:
|
||||
msg_status = ('📰 News items:', str(unread))
|
||||
msg_status = ' '.join(msg_status)
|
||||
else:
|
||||
msg_status = '🗞 No News'
|
||||
|
||||
print(msg_status, 'for', jid)
|
||||
|
||||
# Send status message
|
||||
self.send_presence(
|
||||
pstatus=msg_status,
|
||||
pto=jid,
|
||||
#pfrom=None
|
||||
)
|
||||
|
||||
# await asyncio.sleep(15)
|
||||
await asyncio.sleep(60 * 3)
|
||||
|
|
@ -212,61 +210,53 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
for file in files:
|
||||
jid = file[:-3]
|
||||
|
||||
unread = await initdb(
|
||||
jid,
|
||||
database.get_unread_entries_number
|
||||
)
|
||||
|
||||
if unread:
|
||||
msg_status = ('News', str(unread))
|
||||
msg_status = ' '.join(msg_status)
|
||||
else:
|
||||
msg_status = 'No News'
|
||||
print(msg_status, 'for', jid)
|
||||
|
||||
# NOTE Consider send_presence
|
||||
sts = self.make_presence(
|
||||
pstatus=msg_status,
|
||||
pto=jid,
|
||||
pfrom=jid,
|
||||
pnick='Slixfeed'
|
||||
)
|
||||
|
||||
sts.send()
|
||||
|
||||
await asyncio.sleep(60)
|
||||
|
||||
|
||||
def print_help():
|
||||
# print("print_help")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Print help manual.
|
||||
"""
|
||||
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
|
||||
"\n"
|
||||
"DESCRIPTION: \n"
|
||||
" Slixfeed is a news aggregator bot for online news feeds. \n"
|
||||
" Supported filetypes: Atom, RDF and RSS. \n"
|
||||
"\n"
|
||||
"BASIC USAGE: \n"
|
||||
" enable \n"
|
||||
" Send updates. \n"
|
||||
" disable \n"
|
||||
" Stop sending updates. \n"
|
||||
" batch N \n"
|
||||
" Send N updates on ech interval. \n"
|
||||
" interval N \n"
|
||||
" Send an update each N minutes. \n"
|
||||
" feed list \n"
|
||||
" List subscriptions. \n"
|
||||
"\n"
|
||||
"EDIT OPTIONS: \n"
|
||||
" feed add URL \n"
|
||||
" add URL \n"
|
||||
" Add URL to subscription list. \n"
|
||||
" feed remove ID \n"
|
||||
" remove ID \n"
|
||||
" Remove feed from subscription list. \n"
|
||||
" feed status ID \n"
|
||||
" status ID \n"
|
||||
" Toggle update status of feed. \n"
|
||||
"\n"
|
||||
"SEARCH OPTIONS: \n"
|
||||
" feed search TEXT \n"
|
||||
" search TEXT \n"
|
||||
" Search news items by given keywords. \n"
|
||||
" feed recent N \n"
|
||||
" recent N \n"
|
||||
" List recent N news items (up to 50 items). \n"
|
||||
"\n"
|
||||
"STATISTICS OPTIONS: \n"
|
||||
" analyses \n"
|
||||
" Show report and statistics of feeds. \n"
|
||||
" obsolete \n"
|
||||
" List feeds that are not available. \n"
|
||||
" unread \n"
|
||||
" Print number of unread news items. \n"
|
||||
"\n"
|
||||
"BACKUP OPTIONS: \n"
|
||||
" export opml \n"
|
||||
" Send an OPML file with your feeds. \n"
|
||||
|
|
@ -287,13 +277,10 @@ def print_help():
|
|||
return msg
|
||||
|
||||
|
||||
# Function from buku
|
||||
# https://github.com/jarun/buku
|
||||
# Function from jarun/buku
|
||||
# Arun Prakash Jana (jarun)
|
||||
# Dmitry Marakasov (AMDmi3)
|
||||
def get_default_dbdir():
|
||||
# print("get_default_dbdir")
|
||||
# time.sleep(1)
|
||||
"""Determine the directory path where dbfile will be stored.
|
||||
|
||||
If $XDG_DATA_HOME is defined, use it
|
||||
|
|
@ -301,10 +288,11 @@ def get_default_dbdir():
|
|||
else if the platform is Windows, use %APPDATA%
|
||||
else use the current directory.
|
||||
|
||||
Returns
|
||||
-------
|
||||
str
|
||||
Path to database file.
|
||||
:return: Path to database file.
|
||||
|
||||
Note
|
||||
----
|
||||
This code was taken from the buku project.
|
||||
"""
|
||||
# data_home = xdg.BaseDirectory.xdg_data_home
|
||||
data_home = os.environ.get('XDG_DATA_HOME')
|
||||
|
|
@ -324,8 +312,13 @@ def get_default_dbdir():
|
|||
# TODO Perhaps this needs to be executed
|
||||
# just once per program execution
|
||||
async def initdb(jid, callback, message=None):
|
||||
# print("initdb")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Callback function to instantiate action on database.
|
||||
|
||||
:param jid: JID (Jabber ID).
|
||||
:param callback: Function name.
|
||||
:param massage: Optional kwarg when a message is a part or required argument.
|
||||
"""
|
||||
db_dir = get_default_dbdir()
|
||||
if not os.path.isdir(db_dir):
|
||||
os.mkdir(db_dir)
|
||||
|
|
@ -340,10 +333,11 @@ async def initdb(jid, callback, message=None):
|
|||
# NOTE I don't think there should be "return"
|
||||
# because then we might stop scanning next URLs
|
||||
async def download_updates(db_file):
|
||||
# print("download_updates")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Chack feeds for new entries.
|
||||
|
||||
:param db_file: Database filename.
|
||||
"""
|
||||
urls = await database.get_subscriptions(db_file)
|
||||
|
||||
for url in urls:
|
||||
|
|
@ -386,11 +380,9 @@ async def download_updates(db_file):
|
|||
# TODO Place these couple of lines back down
|
||||
# NOTE Need to correct the SQL statement to do so
|
||||
entries = feed.entries
|
||||
length = len(entries)
|
||||
# breakpoint()
|
||||
# length = len(entries)
|
||||
# await database.remove_entry(db_file, source, length)
|
||||
await database.remove_nonexistent_entries(db_file, feed, source)
|
||||
# breakpoint()
|
||||
|
||||
new_entry = 0
|
||||
for entry in entries:
|
||||
|
|
@ -407,23 +399,9 @@ async def download_updates(db_file):
|
|||
# print('source:', source)
|
||||
|
||||
exist = await database.check_entry_exist(db_file, title, link)
|
||||
# breakpoint()
|
||||
# if exist:
|
||||
# print("//////// OLD ////////")
|
||||
# print(source)
|
||||
# print('ex:',exist)
|
||||
# if entry.has_key("id"):
|
||||
# print('id:',entry.id)
|
||||
|
||||
if not exist:
|
||||
# breakpoint()
|
||||
new_entry = new_entry + 1
|
||||
# print("******** NEW ********")
|
||||
# print('T',title)
|
||||
# if entry.has_key("date"):
|
||||
# print('D',entry.date)
|
||||
# print('L',link)
|
||||
# print('ex',exist)
|
||||
# TODO Enhance summary
|
||||
if entry.has_key("summary"):
|
||||
summary = entry.summary
|
||||
|
|
@ -433,45 +411,50 @@ async def download_updates(db_file):
|
|||
summary = summary.replace("\n\n", "\n")[:300] + " ⃨"
|
||||
else:
|
||||
summary = '*** No summary ***'
|
||||
#print('~~~~~~summary not in entry')
|
||||
entry = (title, summary, link, source, 0);
|
||||
await database.add_entry_and_set_date(db_file, source, entry)
|
||||
# print("### added", new_entry, "entries")
|
||||
|
||||
|
||||
async def download_feed(url):
|
||||
"""
|
||||
Download content of given URL.
|
||||
|
||||
:param url: URL.
|
||||
:return: Document or error message.
|
||||
"""
|
||||
# print("download_feed")
|
||||
# time.sleep(1)
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# async with aiohttp.ClientSession(trust_env=True) as session:
|
||||
# async with aiohttp.ClientSession(trust_env=True) as session:
|
||||
try:
|
||||
async with session.get(url, timeout=timeout) as response:
|
||||
status = response.status
|
||||
if response.status == 200:
|
||||
doc = await response.text()
|
||||
# print (response.content_type)
|
||||
return [doc, status]
|
||||
try:
|
||||
doc = await response.text()
|
||||
# print (response.content_type)
|
||||
return [doc, status]
|
||||
except:
|
||||
return [False, "The content of this document doesn't appear to be textual"]
|
||||
else:
|
||||
return [False, status]
|
||||
return [False, "HTTP Error: " + str(status)]
|
||||
except aiohttp.ClientError as e:
|
||||
print('Error', str(e))
|
||||
return [False, "error"]
|
||||
return [False, "Error: " + str(e)]
|
||||
except asyncio.TimeoutError as e:
|
||||
print('Timeout', str(e))
|
||||
return [False, "timeout"]
|
||||
return [False, "Timeout"]
|
||||
|
||||
|
||||
async def add_feed(db_file, url):
|
||||
# print("add_feed")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Check whether feed exist, otherwise process it
|
||||
:param db_file:
|
||||
:param url:
|
||||
:return: string
|
||||
Check whether feed exist, otherwise process it.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param url: URL.
|
||||
:return: Status message.
|
||||
"""
|
||||
exist = await database.check_feed_exist(db_file, url)
|
||||
|
||||
|
|
@ -483,10 +466,10 @@ async def add_feed(db_file, url):
|
|||
bozo = ("WARNING: Bozo detected. Failed to load <{}>.".format(url))
|
||||
print(bozo)
|
||||
try:
|
||||
# tree = etree.fromstring(res[0]) # etree -> html
|
||||
# tree = etree.fromstring(res[0]) # etree is for xml
|
||||
tree = html.fromstring(res[0])
|
||||
except:
|
||||
return "Failed to parse {} as feed".format(url)
|
||||
return "Failed to parse URL <{}> as feed".format(url)
|
||||
|
||||
print("RSS Auto-Discovery Engaged")
|
||||
xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]"""
|
||||
|
|
@ -518,28 +501,48 @@ async def add_feed(db_file, url):
|
|||
return await add_feed(db_file, url)
|
||||
|
||||
# Search for feeds by file extension and path
|
||||
paths = ["/atom",
|
||||
"/atom.php",
|
||||
"/atom.xml",
|
||||
"/rdf",
|
||||
"/rdf.php",
|
||||
"/rdf.xml",
|
||||
"/rss",
|
||||
"/rss.php",
|
||||
"/rss.xml",
|
||||
"/feed",
|
||||
"/feed.atom",
|
||||
"/feed.rdf",
|
||||
"/feed.rss",
|
||||
"/feed.xml",
|
||||
"/news",
|
||||
"/news/feed",
|
||||
"?format=rss",
|
||||
"/feeds/news_feed",
|
||||
"/content-feeds/",
|
||||
"/app.php/feed", # phpBB
|
||||
"/posts.rss" # Discourse
|
||||
] # More paths "rss.json", "feed.json"
|
||||
paths = [
|
||||
"/app.php/feed", # phpbb
|
||||
"/atom",
|
||||
"/atom.php",
|
||||
"/atom.xml",
|
||||
"/content-feeds/",
|
||||
"/external.php?type=RSS2",
|
||||
"/feed", # good practice
|
||||
"/feed.atom",
|
||||
# "/feed.json",
|
||||
"/feed.php",
|
||||
"/feed.rdf",
|
||||
"/feed.rss",
|
||||
"/feed.xml",
|
||||
"/feed/atom/",
|
||||
"/feeds/news_feed",
|
||||
"/feeds/rss/news.xml.php",
|
||||
"/forum_rss.php",
|
||||
"/index.php/feed",
|
||||
"/index.php?type=atom;action=.xml", #smf
|
||||
"/index.php?type=rss;action=.xml", #smf
|
||||
"/index.rss",
|
||||
"/latest.rss",
|
||||
"/news",
|
||||
"/news.xml",
|
||||
"/news.xml.php",
|
||||
"/news/feed",
|
||||
"/posts.rss", # discourse
|
||||
"/rdf",
|
||||
"/rdf.php",
|
||||
"/rdf.xml",
|
||||
"/rss",
|
||||
# "/rss.json",
|
||||
"/rss.php",
|
||||
"/rss.xml",
|
||||
"/timeline.rss",
|
||||
"/xml/feed.rss",
|
||||
# "?format=atom",
|
||||
# "?format=rdf",
|
||||
# "?format=rss",
|
||||
# "?format=xml"
|
||||
]
|
||||
|
||||
print("RSS Scan Mode Engaged")
|
||||
feeds = {}
|
||||
|
|
@ -551,16 +554,12 @@ async def add_feed(db_file, url):
|
|||
for address in addresses:
|
||||
address = address.xpath('@href')[0]
|
||||
if address.startswith('/'):
|
||||
address = parted_url.netloc + address
|
||||
address = parted_url.scheme + '://' + parted_url.netloc + address
|
||||
res = await download_feed(address)
|
||||
# print(address)
|
||||
if res[1] == 200:
|
||||
# print(address)
|
||||
try:
|
||||
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
||||
# print(feeds)
|
||||
except:
|
||||
# print('Not a feed')
|
||||
continue
|
||||
if len(feeds) > 1:
|
||||
msg = "RSS URL scan has found {} feeds:\n\n".format(len(feeds))
|
||||
|
|
@ -583,7 +582,18 @@ async def add_feed(db_file, url):
|
|||
feeds = {}
|
||||
parted_url = urlparse(url)
|
||||
for path in paths:
|
||||
# print(path)
|
||||
address = parted_url.scheme + '://' + parted_url.netloc + path
|
||||
res = await download_feed(address)
|
||||
if res[1] == 200:
|
||||
# print(feedparser.parse(res[0])["feed"]["title"])
|
||||
# feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
||||
try:
|
||||
title = feedparser.parse(res[0])["feed"]["title"]
|
||||
except:
|
||||
title = '*** No Title ***'
|
||||
feeds[address] = title
|
||||
|
||||
# Check whether URL has path (i.e. not root)
|
||||
if parted_url.path.split('/')[1]:
|
||||
paths.extend([".atom", ".feed", ".rdf", ".rss"]) if '.rss' not in paths else -1
|
||||
# if paths.index('.rss'):
|
||||
|
|
@ -591,20 +601,13 @@ async def add_feed(db_file, url):
|
|||
address = parted_url.scheme + '://' + parted_url.netloc + '/' + parted_url.path.split('/')[1] + path
|
||||
res = await download_feed(address)
|
||||
if res[1] == 200:
|
||||
# print('2res[1]')
|
||||
# print(res[1])
|
||||
# print(feedparser.parse(res[0])["feed"]["title"])
|
||||
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
||||
# print(feeds)
|
||||
else:
|
||||
address = parted_url.scheme + '://' + parted_url.netloc + path
|
||||
res = await download_feed(address)
|
||||
if res[1] == 200:
|
||||
# print('1res[1]')
|
||||
# print(res[1])
|
||||
# print(feedparser.parse(res[0])["feed"]["title"])
|
||||
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
||||
# print(feeds)
|
||||
print('ATTENTION')
|
||||
print(address)
|
||||
try:
|
||||
title = feedparser.parse(res[0])["feed"]["title"]
|
||||
except:
|
||||
title = '*** No Title ***'
|
||||
feeds[address] = title
|
||||
if len(feeds) > 1:
|
||||
msg = "RSS URL discovery has found {} feeds:\n\n".format(len(feeds))
|
||||
for feed in feeds:
|
||||
|
|
@ -621,19 +624,19 @@ async def add_feed(db_file, url):
|
|||
else:
|
||||
return await database.add_feed(db_file, feed, url, res)
|
||||
else:
|
||||
return "Failed to get URL <{}>. HTTP Error {}".format(url, res[1])
|
||||
return "Failed to get URL <{}>. Reason: {}".format(url, res[1])
|
||||
else:
|
||||
return "News source <{}> is already listed in the subscription list".format(url)
|
||||
ix = exist[0]
|
||||
return "News source <{}> is already listed in the subscription list at index {}".format(url, ix)
|
||||
|
||||
|
||||
def toggle_state(jid, state):
|
||||
# print("toggle_state")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Set status of update
|
||||
:param jid: jid of the user
|
||||
:param state: boolean
|
||||
:return:
|
||||
Set status of update.
|
||||
|
||||
:param jid: JID (Jabber ID).
|
||||
:param state: True or False.
|
||||
:return: Status message.
|
||||
"""
|
||||
db_dir = get_default_dbdir()
|
||||
db_file = os.path.join(db_dir, r"{}.db".format(jid))
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ from sqlite3 import Error
|
|||
import asyncio
|
||||
|
||||
from datetime import date
|
||||
import feedparser
|
||||
|
||||
# from eliot import start_action, to_file
|
||||
# # with start_action(action_type="list_subscriptions()", db=db_file):
|
||||
|
|
@ -23,15 +22,12 @@ DBLOCK = asyncio.Lock()
|
|||
CURSORS = {}
|
||||
|
||||
def create_connection(db_file):
|
||||
# print("create_connection")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Create a database connection to the SQLite database
|
||||
specified by db_file
|
||||
:param db_file: database file
|
||||
:return: Connection object or None
|
||||
specified by db_file.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:return: Connection object or None.
|
||||
"""
|
||||
conn = None
|
||||
try:
|
||||
|
|
@ -43,10 +39,11 @@ def create_connection(db_file):
|
|||
|
||||
|
||||
def create_tables(db_file):
|
||||
# print("create_tables")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Create SQLite tables.
|
||||
|
||||
:param db_file: Database filename.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
feeds_table_sql = """
|
||||
CREATE TABLE IF NOT EXISTS feeds (
|
||||
|
|
@ -68,18 +65,26 @@ def create_tables(db_file):
|
|||
source text,
|
||||
read integer
|
||||
); """
|
||||
# statistics_table_sql = """
|
||||
# CREATE TABLE IF NOT EXISTS statistics (
|
||||
# id integer PRIMARY KEY,
|
||||
# title text NOT NULL,
|
||||
# number integer
|
||||
# ); """
|
||||
|
||||
c = conn.cursor()
|
||||
# c = get_cursor(db_file)
|
||||
c.execute(feeds_table_sql)
|
||||
c.execute(entries_table_sql)
|
||||
# c.execute(statistics_table_sql)
|
||||
|
||||
|
||||
def get_cursor(db_file):
|
||||
"""
|
||||
Allocate a cursor to connection per database.
|
||||
:param db_file: database file
|
||||
:return: Cursor
|
||||
|
||||
:param db_file: Database filename.
|
||||
:return: Cursor.
|
||||
"""
|
||||
if db_file in CURSORS:
|
||||
return CURSORS[db_file]
|
||||
|
|
@ -91,15 +96,14 @@ def get_cursor(db_file):
|
|||
|
||||
|
||||
async def add_feed(db_file, feed, url, res):
|
||||
# print("add_feed")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Add a new feed into the feeds table
|
||||
:param conn:
|
||||
:param feed:
|
||||
:return: string
|
||||
Add a new feed into the feeds table.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param feed: Parsed XML document.
|
||||
:param url: URL.
|
||||
:param res: XML document.
|
||||
:return: Message.
|
||||
"""
|
||||
#TODO consider async with DBLOCK
|
||||
#conn = create_connection(db_file)
|
||||
|
|
@ -128,15 +132,12 @@ async def add_feed(db_file, feed, url, res):
|
|||
|
||||
|
||||
async def remove_feed(db_file, ix):
|
||||
# print("remove_feed")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Delete a feed by feed id
|
||||
:param conn:
|
||||
:param id: id of the feed
|
||||
:return: string
|
||||
Delete a feed by feed id.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param ix: Index of feed.
|
||||
:return: Message.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
async with DBLOCK:
|
||||
|
|
@ -158,16 +159,13 @@ async def remove_feed(db_file, ix):
|
|||
|
||||
|
||||
async def check_feed_exist(db_file, url):
|
||||
# print("is_feed_exist")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Check whether a feed exists
|
||||
Query for feeds by url
|
||||
:param conn:
|
||||
:param url:
|
||||
:return: row
|
||||
Check whether a feed exists.
|
||||
Query for feeds by given url.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param url: URL.
|
||||
:return: SQL row or None.
|
||||
"""
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT id FROM feeds WHERE address = ?"
|
||||
|
|
@ -175,11 +173,29 @@ async def check_feed_exist(db_file, url):
|
|||
return cur.fetchone()
|
||||
|
||||
|
||||
async def get_unread_entries_number(db_file):
|
||||
async def get_number_of_items(db_file, str):
|
||||
"""
|
||||
Check number of unread items
|
||||
:param db_file
|
||||
:return: string
|
||||
Return number of entries or feeds.
|
||||
|
||||
:param cur: Cursor object.
|
||||
:param str: "entries" or "feeds".
|
||||
:return: Number of rows.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
cur = conn.cursor()
|
||||
sql = "SELECT count(id) FROM {}".format(str)
|
||||
count = cur.execute(sql)
|
||||
count = cur.fetchone()[0]
|
||||
return count
|
||||
|
||||
|
||||
async def get_number_of_entries_unread(db_file):
|
||||
"""
|
||||
Return number of unread items.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param cur: Cursor object.
|
||||
:return: Number of rows.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
cur = conn.cursor()
|
||||
|
|
@ -187,24 +203,18 @@ async def get_unread_entries_number(db_file):
|
|||
count = cur.execute(sql)
|
||||
count = cur.fetchone()[0]
|
||||
return count
|
||||
|
||||
|
||||
|
||||
async def get_unread(db_file):
|
||||
# print("get_unread")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
async def get_entry_unread(db_file):
|
||||
"""
|
||||
Check read status of entry
|
||||
:param conn:
|
||||
:param id: id of the entry
|
||||
:return: string
|
||||
Check read status of entry.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:return: News item as message.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
entry = []
|
||||
cur = conn.cursor()
|
||||
# cur = get_cursor(db_file)
|
||||
entry = []
|
||||
sql = "SELECT id FROM entries WHERE read = 0"
|
||||
ix = cur.execute(sql).fetchone()
|
||||
if ix is None:
|
||||
|
|
@ -222,36 +232,72 @@ async def get_unread(db_file):
|
|||
cur.execute(sql, (ix,))
|
||||
link = cur.fetchone()[0]
|
||||
entry.append(link)
|
||||
entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2])
|
||||
entry = "{}\n\n{}\n\n{}".format(entry[0], entry[1], entry[2])
|
||||
# print(entry)
|
||||
async with DBLOCK:
|
||||
await mark_as_read(cur, ix)
|
||||
# async with DBLOCK:
|
||||
# await update_statistics(db_file)
|
||||
return entry
|
||||
|
||||
|
||||
async def mark_as_read(cur, ix):
|
||||
# print("mark_as_read", ix)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Set read status of entry
|
||||
:param cur:
|
||||
:param ix: index of the entry
|
||||
Set read status of entry.
|
||||
|
||||
:param cur: Cursor object.
|
||||
:param ix: Index of entry.
|
||||
"""
|
||||
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
|
||||
cur.execute(sql, (ix,))
|
||||
|
||||
|
||||
async def statistics(db_file):
|
||||
"""
|
||||
Return table statistics.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:return: News item as message.
|
||||
"""
|
||||
feeds = await get_number_of_items(db_file, 'feeds')
|
||||
entries = await get_number_of_items(db_file, 'entries')
|
||||
unread_entries = await get_number_of_entries_unread(db_file)
|
||||
return "You have {} unread news items out of {} from {} news sources.".format(unread_entries, entries, feeds)
|
||||
|
||||
|
||||
async def update_statistics(cur):
|
||||
"""
|
||||
Update table statistics.
|
||||
|
||||
:param cur: Cursor object.
|
||||
"""
|
||||
stat_dict = {}
|
||||
stat_dict["feeds"] = await get_number_of_items(cur, 'feeds')
|
||||
stat_dict["entries"] = await get_number_of_items(cur, 'entries')
|
||||
stat_dict["unread"] = await get_number_of_entries_unread(cur=cur)
|
||||
for i in stat_dict:
|
||||
sql = "SELECT id FROM statistics WHERE title = ?"
|
||||
cur.execute(sql, (i,))
|
||||
if cur.fetchone():
|
||||
sql = "UPDATE statistics SET number = :num WHERE title = :title"
|
||||
cur.execute(sql, {"title": i, "num": stat_dict[i]})
|
||||
else:
|
||||
sql = "SELECT count(id) FROM statistics"
|
||||
count = cur.execute(sql)
|
||||
count = cur.fetchone()[0]
|
||||
ix = count + 1
|
||||
sql = "INSERT INTO statistics VALUES(?,?,?)"
|
||||
cur.execute(sql, (ix, i, stat_dict[i]))
|
||||
|
||||
|
||||
# TODO mark_all_read for entries of feed
|
||||
async def toggle_status(db_file, ix):
|
||||
# print("toggle_status")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Set status of feed
|
||||
:param conn:
|
||||
:param id: id of the feed
|
||||
:return: string
|
||||
Toggle status of feed.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param ix: Index of entry.
|
||||
:return: Message
|
||||
"""
|
||||
async with DBLOCK:
|
||||
with create_connection(db_file) as conn:
|
||||
|
|
@ -279,12 +325,11 @@ async def toggle_status(db_file, ix):
|
|||
|
||||
|
||||
async def set_date(cur, url):
|
||||
# print("set_date")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Set last update date of feed
|
||||
:param url: url of the feed
|
||||
:return:
|
||||
Set last update date of feed.
|
||||
|
||||
:param cur: Cursor object.
|
||||
:param url: URL.
|
||||
"""
|
||||
today = date.today()
|
||||
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
|
||||
|
|
@ -293,6 +338,9 @@ async def set_date(cur, url):
|
|||
|
||||
|
||||
async def add_entry_and_set_date(db_file, source, entry):
|
||||
"""
|
||||
TODO
|
||||
"""
|
||||
async with DBLOCK:
|
||||
with create_connection(db_file) as conn:
|
||||
cur = conn.cursor()
|
||||
|
|
@ -301,6 +349,9 @@ async def add_entry_and_set_date(db_file, source, entry):
|
|||
|
||||
|
||||
async def update_source_status(db_file, status, source):
|
||||
"""
|
||||
TODO
|
||||
"""
|
||||
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
|
||||
async with DBLOCK:
|
||||
with create_connection(db_file) as conn:
|
||||
|
|
@ -309,6 +360,9 @@ async def update_source_status(db_file, status, source):
|
|||
|
||||
|
||||
async def update_source_validity(db_file, source, valid):
|
||||
"""
|
||||
TODO
|
||||
"""
|
||||
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
|
||||
async with DBLOCK:
|
||||
with create_connection(db_file) as conn:
|
||||
|
|
@ -317,29 +371,25 @@ async def update_source_validity(db_file, source, valid):
|
|||
|
||||
|
||||
async def add_entry(cur, entry):
|
||||
# print("add_entry")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Add a new entry into the entries table
|
||||
:param conn:
|
||||
Add a new entry into the entries table.
|
||||
|
||||
:param cur: Cursor object.
|
||||
:param entry:
|
||||
:return:
|
||||
"""
|
||||
sql = """ INSERT INTO entries(title,summary,link,source,read)
|
||||
VALUES(?,?,?,?,?) """
|
||||
# cur = conn.cursor()
|
||||
cur.execute(sql, entry)
|
||||
|
||||
|
||||
# This function doesn't work as expected with bbs and wiki feeds
|
||||
async def remove_entry(db_file, source, length):
|
||||
# print("remove_entry")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Maintain list of entries
|
||||
Maintain list of entries equal to feed.
|
||||
Check the number returned by feed and delete
|
||||
existing entries up to the same returned amount
|
||||
:param conn:
|
||||
existing entries up to the same returned amount.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param source:
|
||||
:param length:
|
||||
:return:
|
||||
|
|
@ -364,18 +414,17 @@ async def remove_entry(db_file, source, length):
|
|||
ORDER BY id
|
||||
ASC LIMIT :limit)"""
|
||||
cur.execute(sql, {"source": source, "limit": limit})
|
||||
print('### removed', limit, 'from', source)
|
||||
|
||||
|
||||
async def remove_nonexistent_entries(db_file, feed, source):
|
||||
"""
|
||||
Remove entries that don't exist in feed'
|
||||
Check the entries returned from feed and delete
|
||||
non existing entries
|
||||
:param conn:
|
||||
:param source:
|
||||
:param length:
|
||||
:return:
|
||||
Remove entries that don't exist in a given parsed feed.
|
||||
Check the entries returned from feed and delete non
|
||||
existing entries
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param feed: URL of parsed feed.
|
||||
:param source: URL of associated feed.
|
||||
"""
|
||||
async with DBLOCK:
|
||||
with create_connection(db_file) as conn:
|
||||
|
|
@ -420,12 +469,11 @@ async def remove_nonexistent_entries(db_file, feed, source):
|
|||
|
||||
|
||||
async def get_subscriptions(db_file):
|
||||
# print("get_subscriptions")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Query feeds
|
||||
:param conn:
|
||||
:return: rows (tuple)
|
||||
Query table feeds.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:return: List of feeds.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
cur = conn.cursor()
|
||||
|
|
@ -435,20 +483,15 @@ async def get_subscriptions(db_file):
|
|||
|
||||
|
||||
async def list_subscriptions(db_file):
|
||||
# print("list_subscriptions")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Query feeds
|
||||
:param conn:
|
||||
:return: rows (string)
|
||||
Query table feeds and list items.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:return: List of feeds.
|
||||
"""
|
||||
with create_connection(db_file) as conn:
|
||||
# cur = conn.cursor()
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT name, address, updated, id, enabled FROM feeds"
|
||||
results = cur.execute(sql)
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT name, address, updated, id, enabled FROM feeds"
|
||||
results = cur.execute(sql)
|
||||
|
||||
feeds_list = "List of subscriptions: \n"
|
||||
counter = 0
|
||||
|
|
@ -464,31 +507,26 @@ async def list_subscriptions(db_file):
|
|||
"To add feed, send a message as follows: \n"
|
||||
"feed add URL \n"
|
||||
"Example: \n"
|
||||
"feed add https://reclaimthenet.org/feed/")
|
||||
"add https://reclaimthenet.org/feed/")
|
||||
return msg
|
||||
|
||||
|
||||
async def last_entries(db_file, num):
|
||||
# print("last_entries")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Query feeds
|
||||
:param conn:
|
||||
:param num: integer
|
||||
:return: rows (string)
|
||||
Query entries
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param num: Number
|
||||
:return: List of recent N entries
|
||||
"""
|
||||
num = int(num)
|
||||
if num > 50:
|
||||
num = 50
|
||||
elif num < 1:
|
||||
num = 1
|
||||
with create_connection(db_file) as conn:
|
||||
# cur = conn.cursor()
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
|
||||
results = cur.execute(sql, (num,))
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
|
||||
results = cur.execute(sql, (num,))
|
||||
|
||||
|
||||
titles_list = "Recent {} titles: \n".format(num)
|
||||
|
|
@ -498,24 +536,19 @@ async def last_entries(db_file, num):
|
|||
|
||||
|
||||
async def search_entries(db_file, query):
|
||||
# print("search_entries")
|
||||
# print("db_file")
|
||||
# print(db_file)
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Query feeds
|
||||
:param conn:
|
||||
:param query: string
|
||||
:return: rows (string)
|
||||
Query entries
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param query: Search query
|
||||
:return: Entries with specified keywords
|
||||
"""
|
||||
if len(query) < 2:
|
||||
return "Please enter at least 2 characters to search"
|
||||
|
||||
with create_connection(db_file) as conn:
|
||||
# cur = conn.cursor()
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
|
||||
results = cur.execute(sql, [f'%{query}%'])
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
|
||||
results = cur.execute(sql, [f'%{query}%'])
|
||||
|
||||
results_list = "Search results for '{}': \n".format(query)
|
||||
counter = 0
|
||||
|
|
@ -530,15 +563,14 @@ async def search_entries(db_file, query):
|
|||
|
||||
|
||||
async def check_entry_exist(db_file, title, link):
|
||||
# print("check_entry")
|
||||
# time.sleep(1)
|
||||
"""
|
||||
Check whether an entry exists
|
||||
Query entries by title and link
|
||||
:param conn:
|
||||
:param link:
|
||||
:param title:
|
||||
:return: row
|
||||
Check whether an entry exists.
|
||||
Query entries by title and link.
|
||||
|
||||
:param db_file: Database filename.
|
||||
:param link: Entry URL.
|
||||
:param title: Entry title.
|
||||
:return: SQL row or None.
|
||||
"""
|
||||
cur = get_cursor(db_file)
|
||||
sql = "SELECT id FROM entries WHERE title = :title and link = :link"
|
||||
|
|
|
|||
Loading…
Reference in a new issue