sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

Make toggle feed status operational

Browse source
This commit is contained in:
Schimon Jehudah 2022-06-24 08:44:36 +00:00
parent df8bfcf00a
commit 10d49c51af
1 changed files with 165 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

314
slixfeed.py
View file

@ -25,7 +25,7 @@ from sqlite3 import Error
import sys import sys
#import xdg #import xdg
class EchoBot(slixmpp.ClientXMPP): class Slixfeed(slixmpp.ClientXMPP):
""" """
Slixmpp bot that will send updates of feeds it Slixmpp bot that will send updates of feeds it
receives. receives.
@ -40,11 +40,13 @@ class EchoBot(slixmpp.ClientXMPP):
# listen for this event so that we we can initialize # listen for this event so that we we can initialize
# our roster. # our roster.
self.add_event_handler("session_start", self.start) self.add_event_handler("session_start", self.start)
self.add_event_handler("session_start", self.send_updates)
# The message event is triggered whenever a message # The message event is triggered whenever a message
# stanza is received. Be aware that that includes # stanza is received. Be aware that that includes
# MUC messages and error messages. # MUC messages and error messages.
self.add_event_handler("message", self.message) self.add_event_handler("message", self.message)
self.add_event_handler("disconnected", self.disconnected)
async def start(self, event): async def start(self, event):
""" """
@ -62,6 +64,10 @@ class EchoBot(slixmpp.ClientXMPP):
self.send_presence() self.send_presence()
await self.get_roster() await self.get_roster()
def disconnected(self):
print("disconnected")
return True
def message(self, msg): def message(self, msg):
""" """
Process incoming message stanzas. Be aware that this also Process incoming message stanzas. Be aware that this also
@ -79,136 +85,138 @@ class EchoBot(slixmpp.ClientXMPP):
message = " ".join(msg['body'].split()) message = " ".join(msg['body'].split())
if message.startswith('help'): if message.startswith('help'):
action = print_help() action = print_help()
elif message.startswith('update'): # NOTE: Might not need it
elif message.startswith('feed update'):
action = "/me is scanning feeds for updates..." action = "/me is scanning feeds for updates..."
msg.reply(action).send()
initdb(msg['from'].bare, initdb(msg['from'].bare,
False, False,
download_updates) download_updates)
elif message.startswith('recent updates '): elif message.startswith('feed recent '):
action = initdb(msg['from'].bare, action = initdb(msg['from'].bare,
message[14:], message[12:],
last_entries) last_entries)
elif message.startswith('list feeds'): elif message.startswith('feed search '):
action = initdb(msg['from'].bare,
message[12:],
search_entries)
elif message.startswith('feed list'):
action = initdb(msg['from'].bare, action = initdb(msg['from'].bare,
False, False,
list_subscriptions) list_subscriptions)
elif message.startswith('add feed '): elif message.startswith('feed add '):
action = initdb(msg['from'].bare, action = initdb(msg['from'].bare,
message[9:], message[9:],
add_feed) add_feed)
elif message.startswith('remove feed '): elif message.startswith('feed remove '):
action = initdb(msg['from'].bare, action = initdb(msg['from'].bare,
message[12:], message[12:],
remove_feed) remove_feed)
elif message.startswith('status '): elif message.startswith('feed status '):
action = initdb(msg['from'].bare, action = initdb(msg['from'].bare,
message[7:], message[12:],
toggle_status) toggle_status)
else:
action = "Unknown command. Press \"help\" for list of commands"
msg.reply(action).send() msg.reply(action).send()
async def check_updates():
while True:
db_dir = get_default_dbdir()
if not os.path.isdir(db_dir):
# NOTE: Impossible scenario
msg = """
No database directory was found. \n
To create News database,send these messages to bot: \n
add feed https://reclaimthenet.org/feed/
update
"""
print(msg)
else:
os.chdir(db_dir)
files = os.listdir()
for file in files:
jid = file[:-3]
initdb(jid,
False,
download_updates)
await asyncio.sleep(30)
#await asyncio.sleep(60 * 30)
#await asyncio.sleep(180 * 60)
async def check_updates(self): async def send_updates(self, event):
while True:
while True: db_dir = get_default_dbdir()
db_dir = get_default_dbdir() if not os.path.isdir(db_dir):
if not os.path.isdir(db_dir): # NOTE: Impossible scenario
msg = """ msg = """
No database directory was found. \n No database directory was found. \n
To create News database,send these messages to bot: \n To create News database,send these messages to bot: \n
add feed https://reclaimthenet.org/feed/ add feed https://reclaimthenet.org/feed/
update update
""" """
print(msg) print(msg)
else: else:
os.chdir(db_dir) os.chdir(db_dir)
files = os.listdir() files = os.listdir()
for file in files: for file in files:
jid = file[:-3] jid = file[:-3]
initdb(jid, new = initdb(jid, False, get_unread)
False, if new:
download_updates) msg = self.make_message(mto=jid, mbody=new,
await asyncio.sleep(60 * 30) mtype='chat')
#await asyncio.sleep(180 * 60) msg.send()
# today = str(date.today())
# news.insert = [0, 'News fetched on: ' + today]
async def send_updates(self): #news.append('End of News update')
#for new in news:
while True: #print("sending to: jid")
db_dir = get_default_dbdir() #print("sending to: " + jid)
if not os.path.isdir(db_dir): # self.send_message(mto=jid,
msg = """ # mbody=new,
No database directory was found. \n # mtype='normal').send()
To create News database,send these messages to bot: \n #msg = self.make_message(mto=jid,
add feed https://reclaimthenet.org/feed/ # mbody=new,
update # mtype='chat')
""" #print(msg)
print(msg) #msg.send()
else: await asyncio.sleep(10)
os.chdir(db_dir)
files = os.listdir()
for file in files:
jid = file[:-3]
new = initdb(jid,
False,
get_unread)
if new:
msg = self.make_message(mto=jid,
mbody=new,
mtype='chat')
msg.send()
# today = str(date.today())
# news.insert = [0, 'News fetched on: ' + today]
#news.append('End of News update')
#for new in news:
#print("sending to: jid")
#print("sending to: " + jid)
# self.send_message(mto=jid,
# mbody=new,
# mtype='normal').send()
#msg = self.make_message(mto=jid,
# mbody=new,
# mtype='chat')
#print(msg)
#msg.send()
await asyncio.sleep(10)
asyncio.ensure_future(check_updates(self))
asyncio.ensure_future(send_updates(self))
#asyncio.ensure_future(check_updates())
#asyncio.ensure_future(send_updates())
def print_help(): def print_help():
msg = """ msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
Slixfeed - News syndication bot for Jabber/XMPP "\n"
"DESCRIPTION: \n"
" Slixfeed is an aggregator bot for online news feeds. \n"
"\n"
"BASIC USAGE: \n"
" feed update \n"
" Update subscriptions. \n"
" feed list \n"
" List subscriptions list. \n"
"\n"
"EDIT OPTIONS: \n"
" feed add URL \n"
" Add URL to the subscriptions list. \n"
" feed remove ID \n"
" Remove feed from subscription list. \n"
" feed status ID \n"
" Toggle update status of feed. \n"
"\n"
"SEARCH OPTIONS: \n"
" feed search TEXT \n"
" Search news items by given keywords. \n"
" feed recent N \n"
" List recent N news items. \n"
"\n"
"DOCUMENTATION: \n"
" feedparser \n"
" https://pythonhosted.org/feedparser \n"
" Slixmpp \n"
" https://slixmpp.readthedocs.io/")
DESCRIPTION:
Slixfeed is an aggregator bot for online news feeds.
BASIC USAGE:
update
Update subscriptions.
list feeds
List subscriptions list.
EDIT OPTIONS:
feed add URL
Add URL to the subscriptions list.
feed remove ID
Remove feed from subscription list.
status ID
Toggle update status of feed.
SEARCH OPTIONS:
search TEXT
Search news items by given keywords.
recent updates N
List recent N news items.
DOCUMENTATION:
feedparser
https://pythonhosted.org/feedparser
Slixmpp
https://slixmpp.readthedocs.io/
"""
return msg return msg
# Function from buku # Function from buku
@ -244,11 +252,9 @@ def get_default_dbdir():
return os.path.join(data_home, 'slixfeed') return os.path.join(data_home, 'slixfeed')
# TODO Perhaps this needs to be executed # TODO Perhaps this needs to be executed
# just once per program execution # just once per program execution
def initdb(jid, message, callback): def initdb(jid, message, callback):
db_dir = get_default_dbdir() db_dir = get_default_dbdir()
if not os.path.isdir(db_dir): if not os.path.isdir(db_dir):
os.mkdir(db_dir) os.mkdir(db_dir)
@ -290,7 +296,6 @@ def initdb(jid, message, callback):
else: else:
return callback(conn) return callback(conn)
def create_connection(db_file): def create_connection(db_file):
""" """
Create a database connection to the SQLite database Create a database connection to the SQLite database
@ -307,7 +312,6 @@ def create_connection(db_file):
return conn return conn
def create_table(conn, create_table_sql): def create_table(conn, create_table_sql):
""" """
Create a table from the create_table_sql statement Create a table from the create_table_sql statement
@ -321,7 +325,6 @@ def create_table(conn, create_table_sql):
except Error as e: except Error as e:
print(e) print(e)
# def setup_info(jid): # def setup_info(jid):
# def start_process(jid): # def start_process(jid):
def download_updates(conn): def download_updates(conn):
@ -335,12 +338,15 @@ def download_updates(conn):
source = url[0] source = url[0]
try: try:
feed = feedparser.parse(source) feed = feedparser.parse(source)
if feed.bozo:
bozo = ("WARNING: Bozo detected for feed <{}>. "
"For more information, visit "
"https://pythonhosted.org/feedparser/bozo.html"
.format(source))
print(bozo)
except (IncompleteReadError, IncompleteRead, error.URLError) as e: except (IncompleteReadError, IncompleteRead, error.URLError) as e:
print(e) print(e)
continue continue
if feed.bozo:
bozo = 'WARNING: Bozo detected for feed <{}>. For more information visit https://pythonhosted.org/feedparser/bozo.html'.format(source)
print(bozo)
# TODO Place these couple of lines back down # TODO Place these couple of lines back down
# NOTE Need to correct the SQL statement to do so # NOTE Need to correct the SQL statement to do so
entries = feed.entries entries = feed.entries
@ -351,11 +357,12 @@ def download_updates(conn):
link = source if not entry.link else entry.link link = source if not entry.link else entry.link
exist = check_entry(conn, title, link) exist = check_entry(conn, title, link)
if not exist: if not exist:
if entry.has_key('summary'): if entry.has_key("summary"):
summary = entry.summary summary = entry.summary
# Remove HTML tags # Remove HTML tags
summary = BeautifulSoup(summary, "lxml").text summary = BeautifulSoup(summary, "lxml").text
# TODO Limit text length # TODO Limit text length
summary = summary.replace("\n\n", "\n")
else: else:
summary = '*** No summary ***' summary = '*** No summary ***'
#print('~~~~~~summary not in entry') #print('~~~~~~summary not in entry')
@ -369,7 +376,6 @@ def download_updates(conn):
# print(len(news)) # print(len(news))
# return news # return news
def check_feed(conn, url): def check_feed(conn, url):
""" """
Check whether a feed exists Check whether a feed exists
@ -384,7 +390,6 @@ def check_feed(conn, url):
print(cur.fetchone()) print(cur.fetchone())
return cur.fetchone() return cur.fetchone()
def add_feed(conn, url): def add_feed(conn, url):
""" """
Add a new feed into the feeds table Add a new feed into the feeds table
@ -395,17 +400,22 @@ def add_feed(conn, url):
#conn = create_connection(db_file) #conn = create_connection(db_file)
exist = check_feed(conn, url) exist = check_feed(conn, url)
if not exist: if not exist:
title = feedparser.parse(url)['feed']['title'] feed = feedparser.parse(url)
if feed.bozo:
bozo = ("WARNING: Bozo detected. Failed to load URL.")
print(bozo)
return "Failed to parse URL as feed"
title = feedparser.parse(url)["feed"]["title"]
feed = (title, url, 1) feed = (title, url, 1)
cur = conn.cursor() cur = conn.cursor()
sql = """ INSERT INTO feeds(name,address,status) sql = """INSERT INTO feeds(name,address,status)
VALUES(?,?,?) """ VALUES(?,?,?) """
cur.execute(sql, feed) cur.execute(sql, feed)
conn.commit() conn.commit()
# source = title if not '' else url # source = title if not '' else url
source = title if title else url source = title if title else url
return 'News source "{}" has been added to subscriptions list'.format(source) return """News source "{}" has been added to subscriptions list
""".format(source)
def remove_feed(conn, id): def remove_feed(conn, id):
""" """
@ -417,18 +427,18 @@ def remove_feed(conn, id):
# Enter "delete" to confirm removal. # Enter "delete" to confirm removal.
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
sql = 'SELECT address FROM feeds WHERE id = ?' sql = "SELECT address FROM feeds WHERE id = ?"
# NOTE [0][1][2] # NOTE [0][1][2]
url = cur.execute(sql, (id,)) url = cur.execute(sql, (id,))
for i in url: for i in url:
url = i[0] url = i[0]
sql = 'DELETE FROM entries WHERE source = ? ' sql = "DELETE FROM entries WHERE source = ?"
cur.execute(sql, (url,)) cur.execute(sql, (url,))
sql = 'DELETE FROM feeds WHERE id = ?' sql = "DELETE FROM feeds WHERE id = ?"
cur.execute(sql, (id,)) cur.execute(sql, (id,))
conn.commit() conn.commit()
return 'News source "{}" has been removed from subscriptions list'.format(url) return """News source "{}" has been removed from subscriptions list
""".format(url)
def get_unread(conn): def get_unread(conn):
""" """
@ -462,12 +472,11 @@ def get_unread(conn):
# cur.execute(sql, {"column": column, "id": id}) # cur.execute(sql, {"column": column, "id": id})
# str = cur.fetchone()[0] # str = cur.fetchone()[0]
# entry.append(str) # entry.append(str)
entry = '{}\n\n{}\n\nLink: {}'.format(entry[0], entry[1], entry[2]) entry = "{}\n\n{}\n\nMore information at:\n{}".format(entry[0], entry[1], entry[2])
mark_as_read(conn, id) mark_as_read(conn, id)
conn.commit() conn.commit()
return entry return entry
def mark_as_read(conn, id): def mark_as_read(conn, id):
""" """
Set read status of entry Set read status of entry
@ -480,7 +489,6 @@ def mark_as_read(conn, id):
conn.commit() conn.commit()
return return
# TODO test # TODO test
def toggle_status(conn, id): def toggle_status(conn, id):
""" """
@ -490,18 +498,26 @@ def toggle_status(conn, id):
""" """
#conn = create_connection(db_file) #conn = create_connection(db_file)
cur = conn.cursor() cur = conn.cursor()
sql = "SELECT name FROM feeds WHERE id = :id"
cur.execute(sql, (id,))
title = cur.fetchone()[0]
sql = "SELECT status FROM feeds WHERE id = ?" sql = "SELECT status FROM feeds WHERE id = ?"
# NOTE [0][1][2] # NOTE [0][1][2]
status = cur.execute(sql, (id,)) cur.execute(sql, (id,))
status = cur.fetchone()[0]
# FIXME always set to 1 # FIXME always set to 1
# NOTE Maybe because is not integer # NOTE Maybe because is not integer
# TODO Reset feed table before further testing # TODO Reset feed table before further testing
status = 0 if status == 1 else 1 if status == 1:
status = 0
notice = "News updates for '{}' are now disabled".format(title)
else:
status = 1
notice = "News updates for '{}' are now enabled".format(title)
sql = "UPDATE feeds SET status = :status WHERE id = :id" sql = "UPDATE feeds SET status = :status WHERE id = :id"
cur.execute(sql, {"status": status, "id": id}) cur.execute(sql, {"status": status, "id": id})
conn.commit() conn.commit()
return 'News source status has changed to {}'.format(status) return notice
def set_date(conn, url): def set_date(conn, url):
""" """
@ -515,7 +531,6 @@ def set_date(conn, url):
cur.execute(sql, {"today": today, "url": url}) cur.execute(sql, {"today": today, "url": url})
conn.commit() conn.commit()
def get_subscriptions(conn): def get_subscriptions(conn):
""" """
Query feeds Query feeds
@ -523,11 +538,10 @@ def get_subscriptions(conn):
:return: rows (tuple) :return: rows (tuple)
""" """
cur = conn.cursor() cur = conn.cursor()
sql = "SELECT address FROM feeds" sql = "SELECT address FROM feeds WHERE status = 1"
result = cur.execute(sql) result = cur.execute(sql)
return result return result
def list_subscriptions(conn): def list_subscriptions(conn):
""" """
Query feeds Query feeds
@ -536,15 +550,16 @@ def list_subscriptions(conn):
""" """
cur = conn.cursor() cur = conn.cursor()
#sql = "SELECT id, address FROM feeds" #sql = "SELECT id, address FROM feeds"
sql = "SELECT name, address, updated, id FROM feeds" sql = "SELECT name, address, updated, id, status FROM feeds"
results = cur.execute(sql) results = cur.execute(sql)
feeds_list = 'List of subscriptions: \n' feeds_list = "List of subscriptions: \n"
for result in results: for result in results:
#feeds_list = feeds_list + '\n {}. {}'.format(str(result[0]), str(result[1])) #feeds_list = feeds_list + '\n {}. {}'.format(str(result[0]), str(result[1]))
feeds_list = feeds_list + '\n{} \n{} \nLast updated: {} \nID: {} \n'.format(str(result[0]), str(result[1]), str(result[2]), str(result[3])) feeds_list += """\n{} \n{} \nLast updated: {} \nID: {} [{}]
""".format(str(result[0]), str(result[1]), str(result[2]),
str(result[3]), str(result[4]))
return feeds_list return feeds_list
def check_entry(conn, title, link): def check_entry(conn, title, link):
""" """
Check whether an entry exists Check whether an entry exists
@ -559,7 +574,6 @@ def check_entry(conn, title, link):
cur.execute(sql, {"title": title, "link": link}) cur.execute(sql, {"title": title, "link": link})
return cur.fetchone() return cur.fetchone()
def add_entry(conn, entry): def add_entry(conn, entry):
""" """
Add a new entry into the entries table Add a new entry into the entries table
@ -573,7 +587,6 @@ def add_entry(conn, entry):
cur.execute(sql, entry) cur.execute(sql, entry)
conn.commit() conn.commit()
def remove_entry(conn, source, length): def remove_entry(conn, source, length):
""" """
Maintain list of entries Maintain list of entries
@ -598,14 +611,17 @@ def remove_entry(conn, source, length):
if limit: if limit:
#if limit > 0: #if limit > 0:
limit = limit; limit = limit;
sql = "DELETE FROM entries WHERE id IN (SELECT id FROM entries WHERE source = :source ORDER BY id ASC LIMIT :limit)" sql = """DELETE FROM entries WHERE id IN (
SELECT id FROM entries
WHERE source = :source
ORDER BY id
ASC LIMIT :limit)"""
cur.execute(sql, {"source": source, "limit": limit}) cur.execute(sql, {"source": source, "limit": limit})
conn.commit() conn.commit()
if __name__ == '__main__': if __name__ == '__main__':
# Setup the command line arguments. # Setup the command line arguments.
parser = ArgumentParser(description=EchoBot.__doc__) parser = ArgumentParser(description=Slixfeed.__doc__)
# Output verbosity options. # Output verbosity options.
parser.add_argument("-q", "--quiet", help="set logging to ERROR", parser.add_argument("-q", "--quiet", help="set logging to ERROR",
@ -632,10 +648,10 @@ if __name__ == '__main__':
if args.password is None: if args.password is None:
args.password = getpass("Password: ") args.password = getpass("Password: ")
# Setup the EchoBot and register plugins. Note that while plugins may # Setup the Slixfeed and register plugins. Note that while plugins may
# have interdependencies, the order in which you register them does # have interdependencies, the order in which you register them does
# not matter. # not matter.
xmpp = EchoBot(args.jid, args.password) xmpp = Slixfeed(args.jid, args.password)
xmpp.register_plugin('xep_0030') # Service Discovery xmpp.register_plugin('xep_0030') # Service Discovery
xmpp.register_plugin('xep_0004') # Data Forms xmpp.register_plugin('xep_0004') # Data Forms
xmpp.register_plugin('xep_0060') # PubSub xmpp.register_plugin('xep_0060') # PubSub