Add statistics
This commit is contained in:
Schimon Jehudah
parent
cf44241698
commit
aa8c35d728
2 changed files with 351 additions and 316 deletions
|
|
@ -43,8 +43,8 @@ import database
|
||||||
|
|
||||||
class Slixfeed(slixmpp.ClientXMPP):
|
class Slixfeed(slixmpp.ClientXMPP):
|
||||||
"""
|
"""
|
||||||
Slixmpp bot that will send updates of feeds it
|
Slixmpp news bot that will send updates
|
||||||
receives.
|
from feeds it receives.
|
||||||
"""
|
"""
|
||||||
def __init__(self, jid, password):
|
def __init__(self, jid, password):
|
||||||
slixmpp.ClientXMPP.__init__(self, jid, password)
|
slixmpp.ClientXMPP.__init__(self, jid, password)
|
||||||
|
|
@ -66,7 +66,6 @@ class Slixfeed(slixmpp.ClientXMPP):
|
||||||
self.add_event_handler("disconnected", self.reconnect)
|
self.add_event_handler("disconnected", self.reconnect)
|
||||||
|
|
||||||
async def start(self, event):
|
async def start(self, event):
|
||||||
# print("start")
|
|
||||||
"""
|
"""
|
||||||
Process the session_start event.
|
Process the session_start event.
|
||||||
|
|
||||||
|
|
@ -83,8 +82,6 @@ class Slixfeed(slixmpp.ClientXMPP):
|
||||||
await self.get_roster()
|
await self.get_roster()
|
||||||
|
|
||||||
async def message(self, msg):
|
async def message(self, msg):
|
||||||
# print("message")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Process incoming message stanzas. Be aware that this also
|
Process incoming message stanzas. Be aware that this also
|
||||||
includes MUC messages and error messages. It is usually
|
includes MUC messages and error messages. It is usually
|
||||||
|
|
@ -99,49 +96,34 @@ class Slixfeed(slixmpp.ClientXMPP):
|
||||||
if msg['type'] in ('chat', 'normal'):
|
if msg['type'] in ('chat', 'normal'):
|
||||||
message = " ".join(msg['body'].split())
|
message = " ".join(msg['body'].split())
|
||||||
if message.lower().startswith('help'):
|
if message.lower().startswith('help'):
|
||||||
print("COMMAND: help")
|
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
|
||||||
action = print_help()
|
action = print_help()
|
||||||
# NOTE: Might not need it
|
# NOTE: Might not need it
|
||||||
elif message.lower().startswith('feed recent '):
|
elif message.lower().startswith('recent '):
|
||||||
print("COMMAND: feed recent")
|
action = await initdb(msg['from'].bare, database.last_entries, message[7:])
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
elif message.lower().startswith('search '):
|
||||||
action = await initdb(msg['from'].bare, database.last_entries, message[12:])
|
action = await initdb( msg['from'].bare, database.search_entries, message[7:])
|
||||||
elif message.lower().startswith('feed search '):
|
elif message.lower().startswith('list'):
|
||||||
print("COMMAND: feed search")
|
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
|
||||||
action = await initdb( msg['from'].bare, database.search_entries, message[12:])
|
|
||||||
elif message.lower().startswith('feed list'):
|
|
||||||
print("COMMAND: feed list")
|
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
|
||||||
action = await initdb(msg['from'].bare, database.list_subscriptions)
|
action = await initdb(msg['from'].bare, database.list_subscriptions)
|
||||||
elif message.lower().startswith('feed add '):
|
elif message.lower().startswith('add '):
|
||||||
print("COMMAND: feed add")
|
action = await initdb(msg['from'].bare, add_feed, message[4:])
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
elif message.lower().startswith('remove '):
|
||||||
action = await initdb(msg['from'].bare, add_feed, message[9:])
|
action = await initdb(msg['from'].bare, database.remove_feed, message[7:])
|
||||||
elif message.lower().startswith('feed remove '):
|
elif message.lower().startswith('status '):
|
||||||
print("COMMAND: feed remove")
|
action = await initdb(msg['from'].bare, database.toggle_status, message[7:])
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
elif message.lower().startswith('unread'):
|
||||||
action = await initdb(msg['from'].bare, database.remove_feed, message[12:])
|
action = await initdb(msg['from'].bare, database.statistics)
|
||||||
elif message.lower().startswith('feed status '):
|
|
||||||
print("COMMAND: feed status")
|
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
|
||||||
action = await initdb(msg['from'].bare, database.toggle_status, message[12:])
|
|
||||||
elif message.lower().startswith('enable'):
|
elif message.lower().startswith('enable'):
|
||||||
print("COMMAND: enable")
|
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
|
||||||
action = toggle_state(msg['from'].bare, True)
|
action = toggle_state(msg['from'].bare, True)
|
||||||
elif message.lower().startswith('disable'):
|
elif message.lower().startswith('disable'):
|
||||||
print("COMMAND: disable")
|
|
||||||
print("ACCOUNT: " + str(msg['from']))
|
|
||||||
action = toggle_state(msg['from'].bare, False)
|
action = toggle_state(msg['from'].bare, False)
|
||||||
else:
|
else:
|
||||||
action = 'Unknown command. Press "help" for list of commands'
|
action = 'Unknown command. Press "help" for list of commands'
|
||||||
msg.reply(action).send()
|
msg.reply(action).send()
|
||||||
|
|
||||||
|
print("COMMAND:", message)
|
||||||
|
print("ACCOUNT: " + str(msg['from']))
|
||||||
|
|
||||||
async def check_updates(self, event):
|
async def check_updates(self, event):
|
||||||
# print("check_updates")
|
|
||||||
# time.sleep(1)
|
|
||||||
while True:
|
while True:
|
||||||
print("Checking update")
|
print("Checking update")
|
||||||
db_dir = get_default_dbdir()
|
db_dir = get_default_dbdir()
|
||||||
|
|
@ -156,13 +138,11 @@ class Slixfeed(slixmpp.ClientXMPP):
|
||||||
files = os.listdir(db_dir)
|
files = os.listdir(db_dir)
|
||||||
for file in files:
|
for file in files:
|
||||||
jid = file[:-3]
|
jid = file[:-3]
|
||||||
|
print("download_updates",jid)
|
||||||
await initdb(jid, download_updates)
|
await initdb(jid, download_updates)
|
||||||
# await asyncio.sleep(9)
|
|
||||||
await asyncio.sleep(90)
|
await asyncio.sleep(90)
|
||||||
|
|
||||||
async def send_update(self, event):
|
async def send_update(self, event):
|
||||||
# print("send_update")
|
|
||||||
# time.sleep(1)
|
|
||||||
while True:
|
while True:
|
||||||
db_dir = get_default_dbdir()
|
db_dir = get_default_dbdir()
|
||||||
if not os.path.isdir(db_dir):
|
if not os.path.isdir(db_dir):
|
||||||
|
|
@ -178,21 +158,39 @@ class Slixfeed(slixmpp.ClientXMPP):
|
||||||
for file in files:
|
for file in files:
|
||||||
if not file.endswith('.db-jour.db'):
|
if not file.endswith('.db-jour.db'):
|
||||||
jid = file[:-3]
|
jid = file[:-3]
|
||||||
|
print("get_entry_unread",jid)
|
||||||
|
|
||||||
new = await initdb(
|
new = await initdb(
|
||||||
jid,
|
jid,
|
||||||
database.get_unread
|
database.get_entry_unread
|
||||||
)
|
)
|
||||||
|
|
||||||
if new:
|
if new:
|
||||||
# NOTE Consider send_message
|
msg = self.send_message(
|
||||||
msg = self.make_message(
|
|
||||||
mto=jid,
|
mto=jid,
|
||||||
mbody=new,
|
mbody=new,
|
||||||
mtype='chat'
|
mtype='chat'
|
||||||
)
|
)
|
||||||
|
|
||||||
msg.send()
|
unread = await initdb(
|
||||||
|
jid,
|
||||||
|
database.get_number_of_entries_unread
|
||||||
|
)
|
||||||
|
|
||||||
|
if unread:
|
||||||
|
msg_status = ('📰 News items:', str(unread))
|
||||||
|
msg_status = ' '.join(msg_status)
|
||||||
|
else:
|
||||||
|
msg_status = '🗞 No News'
|
||||||
|
|
||||||
|
print(msg_status, 'for', jid)
|
||||||
|
|
||||||
|
# Send status message
|
||||||
|
self.send_presence(
|
||||||
|
pstatus=msg_status,
|
||||||
|
pto=jid,
|
||||||
|
#pfrom=None
|
||||||
|
)
|
||||||
|
|
||||||
# await asyncio.sleep(15)
|
# await asyncio.sleep(15)
|
||||||
await asyncio.sleep(60 * 3)
|
await asyncio.sleep(60 * 3)
|
||||||
|
|
@ -212,61 +210,53 @@ class Slixfeed(slixmpp.ClientXMPP):
|
||||||
for file in files:
|
for file in files:
|
||||||
jid = file[:-3]
|
jid = file[:-3]
|
||||||
|
|
||||||
unread = await initdb(
|
|
||||||
jid,
|
|
||||||
database.get_unread_entries_number
|
|
||||||
)
|
|
||||||
|
|
||||||
if unread:
|
|
||||||
msg_status = ('News', str(unread))
|
|
||||||
msg_status = ' '.join(msg_status)
|
|
||||||
else:
|
|
||||||
msg_status = 'No News'
|
|
||||||
print(msg_status, 'for', jid)
|
|
||||||
|
|
||||||
# NOTE Consider send_presence
|
|
||||||
sts = self.make_presence(
|
|
||||||
pstatus=msg_status,
|
|
||||||
pto=jid,
|
|
||||||
pfrom=jid,
|
|
||||||
pnick='Slixfeed'
|
|
||||||
)
|
|
||||||
|
|
||||||
sts.send()
|
|
||||||
|
|
||||||
await asyncio.sleep(60)
|
await asyncio.sleep(60)
|
||||||
|
|
||||||
|
|
||||||
def print_help():
|
def print_help():
|
||||||
# print("print_help")
|
"""
|
||||||
# time.sleep(1)
|
Print help manual.
|
||||||
|
"""
|
||||||
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
|
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
|
||||||
"\n"
|
"\n"
|
||||||
"DESCRIPTION: \n"
|
"DESCRIPTION: \n"
|
||||||
" Slixfeed is a news aggregator bot for online news feeds. \n"
|
" Slixfeed is a news aggregator bot for online news feeds. \n"
|
||||||
|
" Supported filetypes: Atom, RDF and RSS. \n"
|
||||||
"\n"
|
"\n"
|
||||||
"BASIC USAGE: \n"
|
"BASIC USAGE: \n"
|
||||||
" enable \n"
|
" enable \n"
|
||||||
" Send updates. \n"
|
" Send updates. \n"
|
||||||
" disable \n"
|
" disable \n"
|
||||||
" Stop sending updates. \n"
|
" Stop sending updates. \n"
|
||||||
|
" batch N \n"
|
||||||
|
" Send N updates on ech interval. \n"
|
||||||
|
" interval N \n"
|
||||||
|
" Send an update each N minutes. \n"
|
||||||
" feed list \n"
|
" feed list \n"
|
||||||
" List subscriptions. \n"
|
" List subscriptions. \n"
|
||||||
"\n"
|
"\n"
|
||||||
"EDIT OPTIONS: \n"
|
"EDIT OPTIONS: \n"
|
||||||
" feed add URL \n"
|
" add URL \n"
|
||||||
" Add URL to subscription list. \n"
|
" Add URL to subscription list. \n"
|
||||||
" feed remove ID \n"
|
" remove ID \n"
|
||||||
" Remove feed from subscription list. \n"
|
" Remove feed from subscription list. \n"
|
||||||
" feed status ID \n"
|
" status ID \n"
|
||||||
" Toggle update status of feed. \n"
|
" Toggle update status of feed. \n"
|
||||||
"\n"
|
"\n"
|
||||||
"SEARCH OPTIONS: \n"
|
"SEARCH OPTIONS: \n"
|
||||||
" feed search TEXT \n"
|
" search TEXT \n"
|
||||||
" Search news items by given keywords. \n"
|
" Search news items by given keywords. \n"
|
||||||
" feed recent N \n"
|
" recent N \n"
|
||||||
" List recent N news items (up to 50 items). \n"
|
" List recent N news items (up to 50 items). \n"
|
||||||
"\n"
|
"\n"
|
||||||
|
"STATISTICS OPTIONS: \n"
|
||||||
|
" analyses \n"
|
||||||
|
" Show report and statistics of feeds. \n"
|
||||||
|
" obsolete \n"
|
||||||
|
" List feeds that are not available. \n"
|
||||||
|
" unread \n"
|
||||||
|
" Print number of unread news items. \n"
|
||||||
|
"\n"
|
||||||
"BACKUP OPTIONS: \n"
|
"BACKUP OPTIONS: \n"
|
||||||
" export opml \n"
|
" export opml \n"
|
||||||
" Send an OPML file with your feeds. \n"
|
" Send an OPML file with your feeds. \n"
|
||||||
|
|
@ -287,13 +277,10 @@ def print_help():
|
||||||
return msg
|
return msg
|
||||||
|
|
||||||
|
|
||||||
# Function from buku
|
# Function from jarun/buku
|
||||||
# https://github.com/jarun/buku
|
|
||||||
# Arun Prakash Jana (jarun)
|
# Arun Prakash Jana (jarun)
|
||||||
# Dmitry Marakasov (AMDmi3)
|
# Dmitry Marakasov (AMDmi3)
|
||||||
def get_default_dbdir():
|
def get_default_dbdir():
|
||||||
# print("get_default_dbdir")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""Determine the directory path where dbfile will be stored.
|
"""Determine the directory path where dbfile will be stored.
|
||||||
|
|
||||||
If $XDG_DATA_HOME is defined, use it
|
If $XDG_DATA_HOME is defined, use it
|
||||||
|
|
@ -301,10 +288,11 @@ def get_default_dbdir():
|
||||||
else if the platform is Windows, use %APPDATA%
|
else if the platform is Windows, use %APPDATA%
|
||||||
else use the current directory.
|
else use the current directory.
|
||||||
|
|
||||||
Returns
|
:return: Path to database file.
|
||||||
-------
|
|
||||||
str
|
Note
|
||||||
Path to database file.
|
----
|
||||||
|
This code was taken from the buku project.
|
||||||
"""
|
"""
|
||||||
# data_home = xdg.BaseDirectory.xdg_data_home
|
# data_home = xdg.BaseDirectory.xdg_data_home
|
||||||
data_home = os.environ.get('XDG_DATA_HOME')
|
data_home = os.environ.get('XDG_DATA_HOME')
|
||||||
|
|
@ -324,8 +312,13 @@ def get_default_dbdir():
|
||||||
# TODO Perhaps this needs to be executed
|
# TODO Perhaps this needs to be executed
|
||||||
# just once per program execution
|
# just once per program execution
|
||||||
async def initdb(jid, callback, message=None):
|
async def initdb(jid, callback, message=None):
|
||||||
# print("initdb")
|
"""
|
||||||
# time.sleep(1)
|
Callback function to instantiate action on database.
|
||||||
|
|
||||||
|
:param jid: JID (Jabber ID).
|
||||||
|
:param callback: Function name.
|
||||||
|
:param massage: Optional kwarg when a message is a part or required argument.
|
||||||
|
"""
|
||||||
db_dir = get_default_dbdir()
|
db_dir = get_default_dbdir()
|
||||||
if not os.path.isdir(db_dir):
|
if not os.path.isdir(db_dir):
|
||||||
os.mkdir(db_dir)
|
os.mkdir(db_dir)
|
||||||
|
|
@ -340,10 +333,11 @@ async def initdb(jid, callback, message=None):
|
||||||
# NOTE I don't think there should be "return"
|
# NOTE I don't think there should be "return"
|
||||||
# because then we might stop scanning next URLs
|
# because then we might stop scanning next URLs
|
||||||
async def download_updates(db_file):
|
async def download_updates(db_file):
|
||||||
# print("download_updates")
|
"""
|
||||||
# print("db_file")
|
Chack feeds for new entries.
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
:param db_file: Database filename.
|
||||||
|
"""
|
||||||
urls = await database.get_subscriptions(db_file)
|
urls = await database.get_subscriptions(db_file)
|
||||||
|
|
||||||
for url in urls:
|
for url in urls:
|
||||||
|
|
@ -386,11 +380,9 @@ async def download_updates(db_file):
|
||||||
# TODO Place these couple of lines back down
|
# TODO Place these couple of lines back down
|
||||||
# NOTE Need to correct the SQL statement to do so
|
# NOTE Need to correct the SQL statement to do so
|
||||||
entries = feed.entries
|
entries = feed.entries
|
||||||
length = len(entries)
|
# length = len(entries)
|
||||||
# breakpoint()
|
|
||||||
# await database.remove_entry(db_file, source, length)
|
# await database.remove_entry(db_file, source, length)
|
||||||
await database.remove_nonexistent_entries(db_file, feed, source)
|
await database.remove_nonexistent_entries(db_file, feed, source)
|
||||||
# breakpoint()
|
|
||||||
|
|
||||||
new_entry = 0
|
new_entry = 0
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
|
|
@ -407,23 +399,9 @@ async def download_updates(db_file):
|
||||||
# print('source:', source)
|
# print('source:', source)
|
||||||
|
|
||||||
exist = await database.check_entry_exist(db_file, title, link)
|
exist = await database.check_entry_exist(db_file, title, link)
|
||||||
# breakpoint()
|
|
||||||
# if exist:
|
|
||||||
# print("//////// OLD ////////")
|
|
||||||
# print(source)
|
|
||||||
# print('ex:',exist)
|
|
||||||
# if entry.has_key("id"):
|
|
||||||
# print('id:',entry.id)
|
|
||||||
|
|
||||||
if not exist:
|
if not exist:
|
||||||
# breakpoint()
|
|
||||||
new_entry = new_entry + 1
|
new_entry = new_entry + 1
|
||||||
# print("******** NEW ********")
|
|
||||||
# print('T',title)
|
|
||||||
# if entry.has_key("date"):
|
|
||||||
# print('D',entry.date)
|
|
||||||
# print('L',link)
|
|
||||||
# print('ex',exist)
|
|
||||||
# TODO Enhance summary
|
# TODO Enhance summary
|
||||||
if entry.has_key("summary"):
|
if entry.has_key("summary"):
|
||||||
summary = entry.summary
|
summary = entry.summary
|
||||||
|
|
@ -433,45 +411,50 @@ async def download_updates(db_file):
|
||||||
summary = summary.replace("\n\n", "\n")[:300] + " ⃨"
|
summary = summary.replace("\n\n", "\n")[:300] + " ⃨"
|
||||||
else:
|
else:
|
||||||
summary = '*** No summary ***'
|
summary = '*** No summary ***'
|
||||||
#print('~~~~~~summary not in entry')
|
|
||||||
entry = (title, summary, link, source, 0);
|
entry = (title, summary, link, source, 0);
|
||||||
await database.add_entry_and_set_date(db_file, source, entry)
|
await database.add_entry_and_set_date(db_file, source, entry)
|
||||||
# print("### added", new_entry, "entries")
|
# print("### added", new_entry, "entries")
|
||||||
|
|
||||||
|
|
||||||
async def download_feed(url):
|
async def download_feed(url):
|
||||||
|
"""
|
||||||
|
Download content of given URL.
|
||||||
|
|
||||||
|
:param url: URL.
|
||||||
|
:return: Document or error message.
|
||||||
|
"""
|
||||||
# print("download_feed")
|
# print("download_feed")
|
||||||
# time.sleep(1)
|
# time.sleep(1)
|
||||||
timeout = aiohttp.ClientTimeout(total=10)
|
timeout = aiohttp.ClientTimeout(total=10)
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
# async with aiohttp.ClientSession(trust_env=True) as session:
|
# async with aiohttp.ClientSession(trust_env=True) as session:
|
||||||
try:
|
try:
|
||||||
async with session.get(url, timeout=timeout) as response:
|
async with session.get(url, timeout=timeout) as response:
|
||||||
status = response.status
|
status = response.status
|
||||||
if response.status == 200:
|
if response.status == 200:
|
||||||
doc = await response.text()
|
try:
|
||||||
# print (response.content_type)
|
doc = await response.text()
|
||||||
return [doc, status]
|
# print (response.content_type)
|
||||||
|
return [doc, status]
|
||||||
|
except:
|
||||||
|
return [False, "The content of this document doesn't appear to be textual"]
|
||||||
else:
|
else:
|
||||||
return [False, status]
|
return [False, "HTTP Error: " + str(status)]
|
||||||
except aiohttp.ClientError as e:
|
except aiohttp.ClientError as e:
|
||||||
print('Error', str(e))
|
print('Error', str(e))
|
||||||
return [False, "error"]
|
return [False, "Error: " + str(e)]
|
||||||
except asyncio.TimeoutError as e:
|
except asyncio.TimeoutError as e:
|
||||||
print('Timeout', str(e))
|
print('Timeout', str(e))
|
||||||
return [False, "timeout"]
|
return [False, "Timeout"]
|
||||||
|
|
||||||
|
|
||||||
async def add_feed(db_file, url):
|
async def add_feed(db_file, url):
|
||||||
# print("add_feed")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Check whether feed exist, otherwise process it
|
Check whether feed exist, otherwise process it.
|
||||||
:param db_file:
|
|
||||||
:param url:
|
:param db_file: Database filename.
|
||||||
:return: string
|
:param url: URL.
|
||||||
|
:return: Status message.
|
||||||
"""
|
"""
|
||||||
exist = await database.check_feed_exist(db_file, url)
|
exist = await database.check_feed_exist(db_file, url)
|
||||||
|
|
||||||
|
|
@ -483,10 +466,10 @@ async def add_feed(db_file, url):
|
||||||
bozo = ("WARNING: Bozo detected. Failed to load <{}>.".format(url))
|
bozo = ("WARNING: Bozo detected. Failed to load <{}>.".format(url))
|
||||||
print(bozo)
|
print(bozo)
|
||||||
try:
|
try:
|
||||||
# tree = etree.fromstring(res[0]) # etree -> html
|
# tree = etree.fromstring(res[0]) # etree is for xml
|
||||||
tree = html.fromstring(res[0])
|
tree = html.fromstring(res[0])
|
||||||
except:
|
except:
|
||||||
return "Failed to parse {} as feed".format(url)
|
return "Failed to parse URL <{}> as feed".format(url)
|
||||||
|
|
||||||
print("RSS Auto-Discovery Engaged")
|
print("RSS Auto-Discovery Engaged")
|
||||||
xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]"""
|
xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]"""
|
||||||
|
|
@ -518,28 +501,48 @@ async def add_feed(db_file, url):
|
||||||
return await add_feed(db_file, url)
|
return await add_feed(db_file, url)
|
||||||
|
|
||||||
# Search for feeds by file extension and path
|
# Search for feeds by file extension and path
|
||||||
paths = ["/atom",
|
paths = [
|
||||||
"/atom.php",
|
"/app.php/feed", # phpbb
|
||||||
"/atom.xml",
|
"/atom",
|
||||||
"/rdf",
|
"/atom.php",
|
||||||
"/rdf.php",
|
"/atom.xml",
|
||||||
"/rdf.xml",
|
"/content-feeds/",
|
||||||
"/rss",
|
"/external.php?type=RSS2",
|
||||||
"/rss.php",
|
"/feed", # good practice
|
||||||
"/rss.xml",
|
"/feed.atom",
|
||||||
"/feed",
|
# "/feed.json",
|
||||||
"/feed.atom",
|
"/feed.php",
|
||||||
"/feed.rdf",
|
"/feed.rdf",
|
||||||
"/feed.rss",
|
"/feed.rss",
|
||||||
"/feed.xml",
|
"/feed.xml",
|
||||||
"/news",
|
"/feed/atom/",
|
||||||
"/news/feed",
|
"/feeds/news_feed",
|
||||||
"?format=rss",
|
"/feeds/rss/news.xml.php",
|
||||||
"/feeds/news_feed",
|
"/forum_rss.php",
|
||||||
"/content-feeds/",
|
"/index.php/feed",
|
||||||
"/app.php/feed", # phpBB
|
"/index.php?type=atom;action=.xml", #smf
|
||||||
"/posts.rss" # Discourse
|
"/index.php?type=rss;action=.xml", #smf
|
||||||
] # More paths "rss.json", "feed.json"
|
"/index.rss",
|
||||||
|
"/latest.rss",
|
||||||
|
"/news",
|
||||||
|
"/news.xml",
|
||||||
|
"/news.xml.php",
|
||||||
|
"/news/feed",
|
||||||
|
"/posts.rss", # discourse
|
||||||
|
"/rdf",
|
||||||
|
"/rdf.php",
|
||||||
|
"/rdf.xml",
|
||||||
|
"/rss",
|
||||||
|
# "/rss.json",
|
||||||
|
"/rss.php",
|
||||||
|
"/rss.xml",
|
||||||
|
"/timeline.rss",
|
||||||
|
"/xml/feed.rss",
|
||||||
|
# "?format=atom",
|
||||||
|
# "?format=rdf",
|
||||||
|
# "?format=rss",
|
||||||
|
# "?format=xml"
|
||||||
|
]
|
||||||
|
|
||||||
print("RSS Scan Mode Engaged")
|
print("RSS Scan Mode Engaged")
|
||||||
feeds = {}
|
feeds = {}
|
||||||
|
|
@ -551,16 +554,12 @@ async def add_feed(db_file, url):
|
||||||
for address in addresses:
|
for address in addresses:
|
||||||
address = address.xpath('@href')[0]
|
address = address.xpath('@href')[0]
|
||||||
if address.startswith('/'):
|
if address.startswith('/'):
|
||||||
address = parted_url.netloc + address
|
address = parted_url.scheme + '://' + parted_url.netloc + address
|
||||||
res = await download_feed(address)
|
res = await download_feed(address)
|
||||||
# print(address)
|
|
||||||
if res[1] == 200:
|
if res[1] == 200:
|
||||||
# print(address)
|
|
||||||
try:
|
try:
|
||||||
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
||||||
# print(feeds)
|
|
||||||
except:
|
except:
|
||||||
# print('Not a feed')
|
|
||||||
continue
|
continue
|
||||||
if len(feeds) > 1:
|
if len(feeds) > 1:
|
||||||
msg = "RSS URL scan has found {} feeds:\n\n".format(len(feeds))
|
msg = "RSS URL scan has found {} feeds:\n\n".format(len(feeds))
|
||||||
|
|
@ -583,7 +582,18 @@ async def add_feed(db_file, url):
|
||||||
feeds = {}
|
feeds = {}
|
||||||
parted_url = urlparse(url)
|
parted_url = urlparse(url)
|
||||||
for path in paths:
|
for path in paths:
|
||||||
# print(path)
|
address = parted_url.scheme + '://' + parted_url.netloc + path
|
||||||
|
res = await download_feed(address)
|
||||||
|
if res[1] == 200:
|
||||||
|
# print(feedparser.parse(res[0])["feed"]["title"])
|
||||||
|
# feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
||||||
|
try:
|
||||||
|
title = feedparser.parse(res[0])["feed"]["title"]
|
||||||
|
except:
|
||||||
|
title = '*** No Title ***'
|
||||||
|
feeds[address] = title
|
||||||
|
|
||||||
|
# Check whether URL has path (i.e. not root)
|
||||||
if parted_url.path.split('/')[1]:
|
if parted_url.path.split('/')[1]:
|
||||||
paths.extend([".atom", ".feed", ".rdf", ".rss"]) if '.rss' not in paths else -1
|
paths.extend([".atom", ".feed", ".rdf", ".rss"]) if '.rss' not in paths else -1
|
||||||
# if paths.index('.rss'):
|
# if paths.index('.rss'):
|
||||||
|
|
@ -591,20 +601,13 @@ async def add_feed(db_file, url):
|
||||||
address = parted_url.scheme + '://' + parted_url.netloc + '/' + parted_url.path.split('/')[1] + path
|
address = parted_url.scheme + '://' + parted_url.netloc + '/' + parted_url.path.split('/')[1] + path
|
||||||
res = await download_feed(address)
|
res = await download_feed(address)
|
||||||
if res[1] == 200:
|
if res[1] == 200:
|
||||||
# print('2res[1]')
|
print('ATTENTION')
|
||||||
# print(res[1])
|
print(address)
|
||||||
# print(feedparser.parse(res[0])["feed"]["title"])
|
try:
|
||||||
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
title = feedparser.parse(res[0])["feed"]["title"]
|
||||||
# print(feeds)
|
except:
|
||||||
else:
|
title = '*** No Title ***'
|
||||||
address = parted_url.scheme + '://' + parted_url.netloc + path
|
feeds[address] = title
|
||||||
res = await download_feed(address)
|
|
||||||
if res[1] == 200:
|
|
||||||
# print('1res[1]')
|
|
||||||
# print(res[1])
|
|
||||||
# print(feedparser.parse(res[0])["feed"]["title"])
|
|
||||||
feeds[address] = feedparser.parse(res[0])["feed"]["title"]
|
|
||||||
# print(feeds)
|
|
||||||
if len(feeds) > 1:
|
if len(feeds) > 1:
|
||||||
msg = "RSS URL discovery has found {} feeds:\n\n".format(len(feeds))
|
msg = "RSS URL discovery has found {} feeds:\n\n".format(len(feeds))
|
||||||
for feed in feeds:
|
for feed in feeds:
|
||||||
|
|
@ -621,19 +624,19 @@ async def add_feed(db_file, url):
|
||||||
else:
|
else:
|
||||||
return await database.add_feed(db_file, feed, url, res)
|
return await database.add_feed(db_file, feed, url, res)
|
||||||
else:
|
else:
|
||||||
return "Failed to get URL <{}>. HTTP Error {}".format(url, res[1])
|
return "Failed to get URL <{}>. Reason: {}".format(url, res[1])
|
||||||
else:
|
else:
|
||||||
return "News source <{}> is already listed in the subscription list".format(url)
|
ix = exist[0]
|
||||||
|
return "News source <{}> is already listed in the subscription list at index {}".format(url, ix)
|
||||||
|
|
||||||
|
|
||||||
def toggle_state(jid, state):
|
def toggle_state(jid, state):
|
||||||
# print("toggle_state")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Set status of update
|
Set status of update.
|
||||||
:param jid: jid of the user
|
|
||||||
:param state: boolean
|
:param jid: JID (Jabber ID).
|
||||||
:return:
|
:param state: True or False.
|
||||||
|
:return: Status message.
|
||||||
"""
|
"""
|
||||||
db_dir = get_default_dbdir()
|
db_dir = get_default_dbdir()
|
||||||
db_file = os.path.join(db_dir, r"{}.db".format(jid))
|
db_file = os.path.join(db_dir, r"{}.db".format(jid))
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,6 @@ from sqlite3 import Error
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from datetime import date
|
from datetime import date
|
||||||
import feedparser
|
|
||||||
|
|
||||||
# from eliot import start_action, to_file
|
# from eliot import start_action, to_file
|
||||||
# # with start_action(action_type="list_subscriptions()", db=db_file):
|
# # with start_action(action_type="list_subscriptions()", db=db_file):
|
||||||
|
|
@ -23,15 +22,12 @@ DBLOCK = asyncio.Lock()
|
||||||
CURSORS = {}
|
CURSORS = {}
|
||||||
|
|
||||||
def create_connection(db_file):
|
def create_connection(db_file):
|
||||||
# print("create_connection")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Create a database connection to the SQLite database
|
Create a database connection to the SQLite database
|
||||||
specified by db_file
|
specified by db_file.
|
||||||
:param db_file: database file
|
|
||||||
:return: Connection object or None
|
:param db_file: Database filename.
|
||||||
|
:return: Connection object or None.
|
||||||
"""
|
"""
|
||||||
conn = None
|
conn = None
|
||||||
try:
|
try:
|
||||||
|
|
@ -43,10 +39,11 @@ def create_connection(db_file):
|
||||||
|
|
||||||
|
|
||||||
def create_tables(db_file):
|
def create_tables(db_file):
|
||||||
# print("create_tables")
|
"""
|
||||||
# print("db_file")
|
Create SQLite tables.
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
:param db_file: Database filename.
|
||||||
|
"""
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
feeds_table_sql = """
|
feeds_table_sql = """
|
||||||
CREATE TABLE IF NOT EXISTS feeds (
|
CREATE TABLE IF NOT EXISTS feeds (
|
||||||
|
|
@ -68,18 +65,26 @@ def create_tables(db_file):
|
||||||
source text,
|
source text,
|
||||||
read integer
|
read integer
|
||||||
); """
|
); """
|
||||||
|
# statistics_table_sql = """
|
||||||
|
# CREATE TABLE IF NOT EXISTS statistics (
|
||||||
|
# id integer PRIMARY KEY,
|
||||||
|
# title text NOT NULL,
|
||||||
|
# number integer
|
||||||
|
# ); """
|
||||||
|
|
||||||
c = conn.cursor()
|
c = conn.cursor()
|
||||||
# c = get_cursor(db_file)
|
# c = get_cursor(db_file)
|
||||||
c.execute(feeds_table_sql)
|
c.execute(feeds_table_sql)
|
||||||
c.execute(entries_table_sql)
|
c.execute(entries_table_sql)
|
||||||
|
# c.execute(statistics_table_sql)
|
||||||
|
|
||||||
|
|
||||||
def get_cursor(db_file):
|
def get_cursor(db_file):
|
||||||
"""
|
"""
|
||||||
Allocate a cursor to connection per database.
|
Allocate a cursor to connection per database.
|
||||||
:param db_file: database file
|
|
||||||
:return: Cursor
|
:param db_file: Database filename.
|
||||||
|
:return: Cursor.
|
||||||
"""
|
"""
|
||||||
if db_file in CURSORS:
|
if db_file in CURSORS:
|
||||||
return CURSORS[db_file]
|
return CURSORS[db_file]
|
||||||
|
|
@ -91,15 +96,14 @@ def get_cursor(db_file):
|
||||||
|
|
||||||
|
|
||||||
async def add_feed(db_file, feed, url, res):
|
async def add_feed(db_file, feed, url, res):
|
||||||
# print("add_feed")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Add a new feed into the feeds table
|
Add a new feed into the feeds table.
|
||||||
:param conn:
|
|
||||||
:param feed:
|
:param db_file: Database filename.
|
||||||
:return: string
|
:param feed: Parsed XML document.
|
||||||
|
:param url: URL.
|
||||||
|
:param res: XML document.
|
||||||
|
:return: Message.
|
||||||
"""
|
"""
|
||||||
#TODO consider async with DBLOCK
|
#TODO consider async with DBLOCK
|
||||||
#conn = create_connection(db_file)
|
#conn = create_connection(db_file)
|
||||||
|
|
@ -128,15 +132,12 @@ async def add_feed(db_file, feed, url, res):
|
||||||
|
|
||||||
|
|
||||||
async def remove_feed(db_file, ix):
|
async def remove_feed(db_file, ix):
|
||||||
# print("remove_feed")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Delete a feed by feed id
|
Delete a feed by feed id.
|
||||||
:param conn:
|
|
||||||
:param id: id of the feed
|
:param db_file: Database filename.
|
||||||
:return: string
|
:param ix: Index of feed.
|
||||||
|
:return: Message.
|
||||||
"""
|
"""
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
|
|
@ -158,16 +159,13 @@ async def remove_feed(db_file, ix):
|
||||||
|
|
||||||
|
|
||||||
async def check_feed_exist(db_file, url):
|
async def check_feed_exist(db_file, url):
|
||||||
# print("is_feed_exist")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Check whether a feed exists
|
Check whether a feed exists.
|
||||||
Query for feeds by url
|
Query for feeds by given url.
|
||||||
:param conn:
|
|
||||||
:param url:
|
:param db_file: Database filename.
|
||||||
:return: row
|
:param url: URL.
|
||||||
|
:return: SQL row or None.
|
||||||
"""
|
"""
|
||||||
cur = get_cursor(db_file)
|
cur = get_cursor(db_file)
|
||||||
sql = "SELECT id FROM feeds WHERE address = ?"
|
sql = "SELECT id FROM feeds WHERE address = ?"
|
||||||
|
|
@ -175,11 +173,29 @@ async def check_feed_exist(db_file, url):
|
||||||
return cur.fetchone()
|
return cur.fetchone()
|
||||||
|
|
||||||
|
|
||||||
async def get_unread_entries_number(db_file):
|
async def get_number_of_items(db_file, str):
|
||||||
"""
|
"""
|
||||||
Check number of unread items
|
Return number of entries or feeds.
|
||||||
:param db_file
|
|
||||||
:return: string
|
:param cur: Cursor object.
|
||||||
|
:param str: "entries" or "feeds".
|
||||||
|
:return: Number of rows.
|
||||||
|
"""
|
||||||
|
with create_connection(db_file) as conn:
|
||||||
|
cur = conn.cursor()
|
||||||
|
sql = "SELECT count(id) FROM {}".format(str)
|
||||||
|
count = cur.execute(sql)
|
||||||
|
count = cur.fetchone()[0]
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
async def get_number_of_entries_unread(db_file):
|
||||||
|
"""
|
||||||
|
Return number of unread items.
|
||||||
|
|
||||||
|
:param db_file: Database filename.
|
||||||
|
:param cur: Cursor object.
|
||||||
|
:return: Number of rows.
|
||||||
"""
|
"""
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
@ -189,22 +205,16 @@ async def get_unread_entries_number(db_file):
|
||||||
return count
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
async def get_entry_unread(db_file):
|
||||||
async def get_unread(db_file):
|
|
||||||
# print("get_unread")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Check read status of entry
|
Check read status of entry.
|
||||||
:param conn:
|
|
||||||
:param id: id of the entry
|
:param db_file: Database filename.
|
||||||
:return: string
|
:return: News item as message.
|
||||||
"""
|
"""
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
entry = []
|
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
# cur = get_cursor(db_file)
|
entry = []
|
||||||
sql = "SELECT id FROM entries WHERE read = 0"
|
sql = "SELECT id FROM entries WHERE read = 0"
|
||||||
ix = cur.execute(sql).fetchone()
|
ix = cur.execute(sql).fetchone()
|
||||||
if ix is None:
|
if ix is None:
|
||||||
|
|
@ -222,36 +232,72 @@ async def get_unread(db_file):
|
||||||
cur.execute(sql, (ix,))
|
cur.execute(sql, (ix,))
|
||||||
link = cur.fetchone()[0]
|
link = cur.fetchone()[0]
|
||||||
entry.append(link)
|
entry.append(link)
|
||||||
entry = "{}\n\n{}\n\nLink to article:\n{}".format(entry[0], entry[1], entry[2])
|
entry = "{}\n\n{}\n\n{}".format(entry[0], entry[1], entry[2])
|
||||||
# print(entry)
|
# print(entry)
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
await mark_as_read(cur, ix)
|
await mark_as_read(cur, ix)
|
||||||
|
# async with DBLOCK:
|
||||||
|
# await update_statistics(db_file)
|
||||||
return entry
|
return entry
|
||||||
|
|
||||||
|
|
||||||
async def mark_as_read(cur, ix):
|
async def mark_as_read(cur, ix):
|
||||||
# print("mark_as_read", ix)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Set read status of entry
|
Set read status of entry.
|
||||||
:param cur:
|
|
||||||
:param ix: index of the entry
|
:param cur: Cursor object.
|
||||||
|
:param ix: Index of entry.
|
||||||
"""
|
"""
|
||||||
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
|
sql = "UPDATE entries SET summary = '', read = 1 WHERE id = ?"
|
||||||
cur.execute(sql, (ix,))
|
cur.execute(sql, (ix,))
|
||||||
|
|
||||||
|
|
||||||
|
async def statistics(db_file):
|
||||||
|
"""
|
||||||
|
Return table statistics.
|
||||||
|
|
||||||
|
:param db_file: Database filename.
|
||||||
|
:return: News item as message.
|
||||||
|
"""
|
||||||
|
feeds = await get_number_of_items(db_file, 'feeds')
|
||||||
|
entries = await get_number_of_items(db_file, 'entries')
|
||||||
|
unread_entries = await get_number_of_entries_unread(db_file)
|
||||||
|
return "You have {} unread news items out of {} from {} news sources.".format(unread_entries, entries, feeds)
|
||||||
|
|
||||||
|
|
||||||
|
async def update_statistics(cur):
|
||||||
|
"""
|
||||||
|
Update table statistics.
|
||||||
|
|
||||||
|
:param cur: Cursor object.
|
||||||
|
"""
|
||||||
|
stat_dict = {}
|
||||||
|
stat_dict["feeds"] = await get_number_of_items(cur, 'feeds')
|
||||||
|
stat_dict["entries"] = await get_number_of_items(cur, 'entries')
|
||||||
|
stat_dict["unread"] = await get_number_of_entries_unread(cur=cur)
|
||||||
|
for i in stat_dict:
|
||||||
|
sql = "SELECT id FROM statistics WHERE title = ?"
|
||||||
|
cur.execute(sql, (i,))
|
||||||
|
if cur.fetchone():
|
||||||
|
sql = "UPDATE statistics SET number = :num WHERE title = :title"
|
||||||
|
cur.execute(sql, {"title": i, "num": stat_dict[i]})
|
||||||
|
else:
|
||||||
|
sql = "SELECT count(id) FROM statistics"
|
||||||
|
count = cur.execute(sql)
|
||||||
|
count = cur.fetchone()[0]
|
||||||
|
ix = count + 1
|
||||||
|
sql = "INSERT INTO statistics VALUES(?,?,?)"
|
||||||
|
cur.execute(sql, (ix, i, stat_dict[i]))
|
||||||
|
|
||||||
|
|
||||||
# TODO mark_all_read for entries of feed
|
# TODO mark_all_read for entries of feed
|
||||||
async def toggle_status(db_file, ix):
|
async def toggle_status(db_file, ix):
|
||||||
# print("toggle_status")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Set status of feed
|
Toggle status of feed.
|
||||||
:param conn:
|
|
||||||
:param id: id of the feed
|
:param db_file: Database filename.
|
||||||
:return: string
|
:param ix: Index of entry.
|
||||||
|
:return: Message
|
||||||
"""
|
"""
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
|
|
@ -279,12 +325,11 @@ async def toggle_status(db_file, ix):
|
||||||
|
|
||||||
|
|
||||||
async def set_date(cur, url):
|
async def set_date(cur, url):
|
||||||
# print("set_date")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Set last update date of feed
|
Set last update date of feed.
|
||||||
:param url: url of the feed
|
|
||||||
:return:
|
:param cur: Cursor object.
|
||||||
|
:param url: URL.
|
||||||
"""
|
"""
|
||||||
today = date.today()
|
today = date.today()
|
||||||
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
|
sql = "UPDATE feeds SET updated = :today WHERE address = :url"
|
||||||
|
|
@ -293,6 +338,9 @@ async def set_date(cur, url):
|
||||||
|
|
||||||
|
|
||||||
async def add_entry_and_set_date(db_file, source, entry):
|
async def add_entry_and_set_date(db_file, source, entry):
|
||||||
|
"""
|
||||||
|
TODO
|
||||||
|
"""
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
@ -301,6 +349,9 @@ async def add_entry_and_set_date(db_file, source, entry):
|
||||||
|
|
||||||
|
|
||||||
async def update_source_status(db_file, status, source):
|
async def update_source_status(db_file, status, source):
|
||||||
|
"""
|
||||||
|
TODO
|
||||||
|
"""
|
||||||
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
|
sql = "UPDATE feeds SET status = :status, scanned = :scanned WHERE address = :url"
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
|
|
@ -309,6 +360,9 @@ async def update_source_status(db_file, status, source):
|
||||||
|
|
||||||
|
|
||||||
async def update_source_validity(db_file, source, valid):
|
async def update_source_validity(db_file, source, valid):
|
||||||
|
"""
|
||||||
|
TODO
|
||||||
|
"""
|
||||||
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
|
sql = "UPDATE feeds SET valid = :validity WHERE address = :url"
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
|
|
@ -317,29 +371,25 @@ async def update_source_validity(db_file, source, valid):
|
||||||
|
|
||||||
|
|
||||||
async def add_entry(cur, entry):
|
async def add_entry(cur, entry):
|
||||||
# print("add_entry")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Add a new entry into the entries table
|
Add a new entry into the entries table.
|
||||||
:param conn:
|
|
||||||
|
:param cur: Cursor object.
|
||||||
:param entry:
|
:param entry:
|
||||||
:return:
|
|
||||||
"""
|
"""
|
||||||
sql = """ INSERT INTO entries(title,summary,link,source,read)
|
sql = """ INSERT INTO entries(title,summary,link,source,read)
|
||||||
VALUES(?,?,?,?,?) """
|
VALUES(?,?,?,?,?) """
|
||||||
# cur = conn.cursor()
|
|
||||||
cur.execute(sql, entry)
|
cur.execute(sql, entry)
|
||||||
|
|
||||||
|
|
||||||
# This function doesn't work as expected with bbs and wiki feeds
|
# This function doesn't work as expected with bbs and wiki feeds
|
||||||
async def remove_entry(db_file, source, length):
|
async def remove_entry(db_file, source, length):
|
||||||
# print("remove_entry")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Maintain list of entries
|
Maintain list of entries equal to feed.
|
||||||
Check the number returned by feed and delete
|
Check the number returned by feed and delete
|
||||||
existing entries up to the same returned amount
|
existing entries up to the same returned amount.
|
||||||
:param conn:
|
|
||||||
|
:param db_file: Database filename.
|
||||||
:param source:
|
:param source:
|
||||||
:param length:
|
:param length:
|
||||||
:return:
|
:return:
|
||||||
|
|
@ -364,18 +414,17 @@ async def remove_entry(db_file, source, length):
|
||||||
ORDER BY id
|
ORDER BY id
|
||||||
ASC LIMIT :limit)"""
|
ASC LIMIT :limit)"""
|
||||||
cur.execute(sql, {"source": source, "limit": limit})
|
cur.execute(sql, {"source": source, "limit": limit})
|
||||||
print('### removed', limit, 'from', source)
|
|
||||||
|
|
||||||
|
|
||||||
async def remove_nonexistent_entries(db_file, feed, source):
|
async def remove_nonexistent_entries(db_file, feed, source):
|
||||||
"""
|
"""
|
||||||
Remove entries that don't exist in feed'
|
Remove entries that don't exist in a given parsed feed.
|
||||||
Check the entries returned from feed and delete
|
Check the entries returned from feed and delete non
|
||||||
non existing entries
|
existing entries
|
||||||
:param conn:
|
|
||||||
:param source:
|
:param db_file: Database filename.
|
||||||
:param length:
|
:param feed: URL of parsed feed.
|
||||||
:return:
|
:param source: URL of associated feed.
|
||||||
"""
|
"""
|
||||||
async with DBLOCK:
|
async with DBLOCK:
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
|
|
@ -420,12 +469,11 @@ async def remove_nonexistent_entries(db_file, feed, source):
|
||||||
|
|
||||||
|
|
||||||
async def get_subscriptions(db_file):
|
async def get_subscriptions(db_file):
|
||||||
# print("get_subscriptions")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Query feeds
|
Query table feeds.
|
||||||
:param conn:
|
|
||||||
:return: rows (tuple)
|
:param db_file: Database filename.
|
||||||
|
:return: List of feeds.
|
||||||
"""
|
"""
|
||||||
with create_connection(db_file) as conn:
|
with create_connection(db_file) as conn:
|
||||||
cur = conn.cursor()
|
cur = conn.cursor()
|
||||||
|
|
@ -435,20 +483,15 @@ async def get_subscriptions(db_file):
|
||||||
|
|
||||||
|
|
||||||
async def list_subscriptions(db_file):
|
async def list_subscriptions(db_file):
|
||||||
# print("list_subscriptions")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Query feeds
|
Query table feeds and list items.
|
||||||
:param conn:
|
|
||||||
:return: rows (string)
|
:param db_file: Database filename.
|
||||||
|
:return: List of feeds.
|
||||||
"""
|
"""
|
||||||
with create_connection(db_file) as conn:
|
cur = get_cursor(db_file)
|
||||||
# cur = conn.cursor()
|
sql = "SELECT name, address, updated, id, enabled FROM feeds"
|
||||||
cur = get_cursor(db_file)
|
results = cur.execute(sql)
|
||||||
sql = "SELECT name, address, updated, id, enabled FROM feeds"
|
|
||||||
results = cur.execute(sql)
|
|
||||||
|
|
||||||
feeds_list = "List of subscriptions: \n"
|
feeds_list = "List of subscriptions: \n"
|
||||||
counter = 0
|
counter = 0
|
||||||
|
|
@ -464,31 +507,26 @@ async def list_subscriptions(db_file):
|
||||||
"To add feed, send a message as follows: \n"
|
"To add feed, send a message as follows: \n"
|
||||||
"feed add URL \n"
|
"feed add URL \n"
|
||||||
"Example: \n"
|
"Example: \n"
|
||||||
"feed add https://reclaimthenet.org/feed/")
|
"add https://reclaimthenet.org/feed/")
|
||||||
return msg
|
return msg
|
||||||
|
|
||||||
|
|
||||||
async def last_entries(db_file, num):
|
async def last_entries(db_file, num):
|
||||||
# print("last_entries")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Query feeds
|
Query entries
|
||||||
:param conn:
|
|
||||||
:param num: integer
|
:param db_file: Database filename.
|
||||||
:return: rows (string)
|
:param num: Number
|
||||||
|
:return: List of recent N entries
|
||||||
"""
|
"""
|
||||||
num = int(num)
|
num = int(num)
|
||||||
if num > 50:
|
if num > 50:
|
||||||
num = 50
|
num = 50
|
||||||
elif num < 1:
|
elif num < 1:
|
||||||
num = 1
|
num = 1
|
||||||
with create_connection(db_file) as conn:
|
cur = get_cursor(db_file)
|
||||||
# cur = conn.cursor()
|
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
|
||||||
cur = get_cursor(db_file)
|
results = cur.execute(sql, (num,))
|
||||||
sql = "SELECT title, link FROM entries ORDER BY ROWID DESC LIMIT :num"
|
|
||||||
results = cur.execute(sql, (num,))
|
|
||||||
|
|
||||||
|
|
||||||
titles_list = "Recent {} titles: \n".format(num)
|
titles_list = "Recent {} titles: \n".format(num)
|
||||||
|
|
@ -498,24 +536,19 @@ async def last_entries(db_file, num):
|
||||||
|
|
||||||
|
|
||||||
async def search_entries(db_file, query):
|
async def search_entries(db_file, query):
|
||||||
# print("search_entries")
|
|
||||||
# print("db_file")
|
|
||||||
# print(db_file)
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Query feeds
|
Query entries
|
||||||
:param conn:
|
|
||||||
:param query: string
|
:param db_file: Database filename.
|
||||||
:return: rows (string)
|
:param query: Search query
|
||||||
|
:return: Entries with specified keywords
|
||||||
"""
|
"""
|
||||||
if len(query) < 2:
|
if len(query) < 2:
|
||||||
return "Please enter at least 2 characters to search"
|
return "Please enter at least 2 characters to search"
|
||||||
|
|
||||||
with create_connection(db_file) as conn:
|
cur = get_cursor(db_file)
|
||||||
# cur = conn.cursor()
|
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
|
||||||
cur = get_cursor(db_file)
|
results = cur.execute(sql, [f'%{query}%'])
|
||||||
sql = "SELECT title, link FROM entries WHERE title LIKE ? LIMIT 50"
|
|
||||||
results = cur.execute(sql, [f'%{query}%'])
|
|
||||||
|
|
||||||
results_list = "Search results for '{}': \n".format(query)
|
results_list = "Search results for '{}': \n".format(query)
|
||||||
counter = 0
|
counter = 0
|
||||||
|
|
@ -530,15 +563,14 @@ async def search_entries(db_file, query):
|
||||||
|
|
||||||
|
|
||||||
async def check_entry_exist(db_file, title, link):
|
async def check_entry_exist(db_file, title, link):
|
||||||
# print("check_entry")
|
|
||||||
# time.sleep(1)
|
|
||||||
"""
|
"""
|
||||||
Check whether an entry exists
|
Check whether an entry exists.
|
||||||
Query entries by title and link
|
Query entries by title and link.
|
||||||
:param conn:
|
|
||||||
:param link:
|
:param db_file: Database filename.
|
||||||
:param title:
|
:param link: Entry URL.
|
||||||
:return: row
|
:param title: Entry title.
|
||||||
|
:return: SQL row or None.
|
||||||
"""
|
"""
|
||||||
cur = get_cursor(db_file)
|
cur = get_cursor(db_file)
|
||||||
sql = "SELECT id FROM entries WHERE title = :title and link = :link"
|
sql = "SELECT id FROM entries WHERE title = :title and link = :link"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue