forked from sch/Slixfeed
Add for Add Subscription.
Segregating more code as one task per function, particularly adding of new subscription. Fix allow/deny keys.
This commit is contained in:
Schimon Jehudah
parent
6c0c571c1d
commit
4e193a2b02
7 changed files with 284 additions and 91 deletions
|
|
@ -737,22 +737,26 @@ async def add_feed(db_file, url):
|
|||
updated = ''
|
||||
version = feed["version"]
|
||||
entries = len(feed["entries"])
|
||||
await sqlite.insert_feed(
|
||||
db_file, url,
|
||||
title=title,
|
||||
entries=entries,
|
||||
version=version,
|
||||
encoding=encoding,
|
||||
language=language,
|
||||
status_code=status_code,
|
||||
updated=updated
|
||||
)
|
||||
await sqlite.insert_feed(db_file, url,
|
||||
title=title,
|
||||
entries=entries,
|
||||
version=version,
|
||||
encoding=encoding,
|
||||
language=language,
|
||||
status_code=status_code,
|
||||
updated=updated)
|
||||
await scan(db_file, url)
|
||||
old = config.get_setting_value(db_file, "old")
|
||||
if not old:
|
||||
feed_id = await sqlite.get_feed_id(db_file, url)
|
||||
feed_id = feed_id[0]
|
||||
await sqlite.mark_feed_as_read(db_file, feed_id)
|
||||
result_final = {'url' : url,
|
||||
'index' : feed_id,
|
||||
'name' : title,
|
||||
'code' : status_code,
|
||||
'error' : False,
|
||||
'exist' : False}
|
||||
response = ('> {}\nNews source "{}" has been '
|
||||
'added to subscription list.'
|
||||
.format(url, title))
|
||||
|
|
@ -783,47 +787,64 @@ async def add_feed(db_file, url):
|
|||
updated = ''
|
||||
version = 'json' + feed["version"].split('/').pop()
|
||||
entries = len(feed["items"])
|
||||
await sqlite.insert_feed(
|
||||
db_file, url,
|
||||
title=title,
|
||||
entries=entries,
|
||||
version=version,
|
||||
encoding=encoding,
|
||||
language=language,
|
||||
status_code=status_code,
|
||||
updated=updated
|
||||
)
|
||||
await scan_json(
|
||||
db_file, url)
|
||||
await sqlite.insert_feed(db_file, url,
|
||||
title=title,
|
||||
entries=entries,
|
||||
version=version,
|
||||
encoding=encoding,
|
||||
language=language,
|
||||
status_code=status_code,
|
||||
updated=updated)
|
||||
await scan_json(db_file, url)
|
||||
old = config.get_setting_value(db_file, "old")
|
||||
if not old:
|
||||
feed_id = await sqlite.get_feed_id(db_file, url)
|
||||
feed_id = feed_id[0]
|
||||
await sqlite.mark_feed_as_read(db_file, feed_id)
|
||||
result_final = {'url' : url,
|
||||
'index' : feed_id,
|
||||
'name' : title,
|
||||
'code' : status_code,
|
||||
'error' : False,
|
||||
'exist' : False}
|
||||
response = ('> {}\nNews source "{}" has been '
|
||||
'added to subscription list.'
|
||||
.format(url, title))
|
||||
break
|
||||
else:
|
||||
result = await crawl.probe_page(
|
||||
url, document)
|
||||
if isinstance(result, str):
|
||||
response = result
|
||||
# NOTE Do not be tempted to return a compact dictionary.
|
||||
# That is, dictionary within dictionary
|
||||
# Return multimple dictionaries.
|
||||
result = await crawl.probe_page(url, document)
|
||||
if isinstance(result, list):
|
||||
result_final = result
|
||||
break
|
||||
else:
|
||||
url = result[0]
|
||||
url = result['url']
|
||||
else:
|
||||
result_final = {'url' : url,
|
||||
'index' : None,
|
||||
'name' : None,
|
||||
'code' : status_code,
|
||||
'error' : True,
|
||||
'exist' : False}
|
||||
response = ('> {}\nFailed to load URL. Reason: {}'
|
||||
.format(url, status_code))
|
||||
break
|
||||
else:
|
||||
ix = exist[0]
|
||||
name = exist[1]
|
||||
result_final = {'url' : url,
|
||||
'index' : ix,
|
||||
'name' : name,
|
||||
'code' : None,
|
||||
'error' : False,
|
||||
'exist' : True}
|
||||
response = ('> {}\nNews source "{}" is already '
|
||||
'listed in the subscription list at '
|
||||
'index {}'.format(url, name, ix))
|
||||
break
|
||||
return response
|
||||
return result_final
|
||||
|
||||
|
||||
async def scan_json(db_file, url):
|
||||
|
|
|
|||
|
|
@ -154,9 +154,9 @@ thanks = [
|
|||
"Strix from Loqi",
|
||||
"Thibaud Guerin (SalixOS)",
|
||||
"Thorsten Fröhlich (France)",
|
||||
"Thorsten Mühlfelder (SalixOS, Germany)",
|
||||
"Thorsten Mühlfelder (SalixOS, Germany)",
|
||||
"Tim Beech (SalixOS, Brazil)",
|
||||
"Yann Leboulanger (Gajim, France)"
|
||||
"Yann Leboulanger (Gajim, France)"
|
||||
]
|
||||
|
||||
xmpp = """
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ pathnames = [
|
|||
# would get the best out of this news application.
|
||||
|
||||
# Entries with the following keywords will not be filtered
|
||||
filter-allow = [
|
||||
allow = [
|
||||
"akkoma",
|
||||
"censorship",
|
||||
"earthing",
|
||||
|
|
@ -187,7 +187,7 @@ filter-allow = [
|
|||
]
|
||||
|
||||
# Entries with the following keywords will be filtered
|
||||
filter-deny = [
|
||||
deny = [
|
||||
# brands
|
||||
# Almost every time you see a brand name in title or content, it is because
|
||||
# someone, usually a marketing agency or a venture capital firm, has paid for
|
||||
|
|
@ -264,6 +264,7 @@ filter-deny = [
|
|||
"homosex",
|
||||
"lesbian",
|
||||
"lgbt",
|
||||
"nonbinary",
|
||||
"nude",
|
||||
"nudity",
|
||||
"onlyfans",
|
||||
|
|
|
|||
|
|
@ -300,7 +300,8 @@ async def process_feed_selection(url, urls):
|
|||
feeds = {}
|
||||
for i in urls:
|
||||
res = await fetch.http(i)
|
||||
if res[1] == 200:
|
||||
status_code = res[1]
|
||||
if status_code == 200:
|
||||
try:
|
||||
feeds[i] = [parse(res[0])]
|
||||
except:
|
||||
|
|
@ -308,8 +309,7 @@ async def process_feed_selection(url, urls):
|
|||
message = (
|
||||
"Web feeds found for {}\n\n```\n"
|
||||
).format(url)
|
||||
counter = 0
|
||||
feed_url_mark = 0
|
||||
urls = []
|
||||
for feed_url in feeds:
|
||||
# try:
|
||||
# res = await fetch.http(feed)
|
||||
|
|
@ -327,24 +327,28 @@ async def process_feed_selection(url, urls):
|
|||
continue
|
||||
if feed_amnt:
|
||||
# NOTE Because there could be many false positives
|
||||
# which are revealed in second phase of scan, we
|
||||
# could end with a single feed, which would be
|
||||
# listed instead of fetched, so feed_url_mark is
|
||||
# utilized in order to make fetch possible.
|
||||
feed_url_mark = [feed_url]
|
||||
counter += 1
|
||||
message += (
|
||||
"Title : {}\n"
|
||||
"Link : {}\n"
|
||||
"\n"
|
||||
).format(feed_name, feed_url)
|
||||
if counter > 1:
|
||||
message += (
|
||||
"```\nTotal of {} feeds."
|
||||
).format(counter)
|
||||
result = message
|
||||
elif feed_url_mark:
|
||||
result = feed_url_mark
|
||||
# which are revealed in second phase of scan, we
|
||||
# could end with a single feed, which would be
|
||||
# listed instead of fetched, so feed_url_mark is
|
||||
# utilized in order to make fetch possible.
|
||||
# NOTE feed_url_mark was a variable which stored
|
||||
# single URL (probably first accepted as valid)
|
||||
# in order to get an indication whether a single
|
||||
# URL has been fetched, so that the receiving
|
||||
# function will scan that single URL instead of
|
||||
# listing it as a message.
|
||||
url = {'url' : feed_url,
|
||||
'index' : None,
|
||||
'name' : feed_name,
|
||||
'code' : status_code,
|
||||
'error' : False,
|
||||
'exist' : None}
|
||||
urls.extend([url])
|
||||
count = len(urls)
|
||||
if count > 1:
|
||||
result = urls
|
||||
elif count:
|
||||
result = urls[0]
|
||||
else:
|
||||
result = None
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -469,8 +469,11 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
# )
|
||||
|
||||
# if jid == config.get_value('accounts', 'XMPP', 'operator'):
|
||||
self['xep_0050'].add_command(node='subscription',
|
||||
name='➕️ Add Subscription',
|
||||
handler=self._handle_subscription_add)
|
||||
self['xep_0050'].add_command(node='subscriptions',
|
||||
name='📰️ Subscriptions',
|
||||
name='📰️ Browse Subscriptions',
|
||||
handler=self._handle_subscriptions)
|
||||
# self['xep_0050'].add_command(node='subscriptions_cat',
|
||||
# name='🔖️ Categories',
|
||||
|
|
@ -481,18 +484,20 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
# self['xep_0050'].add_command(node='subscriptions_index',
|
||||
# name='📑️ Index (A - Z)',
|
||||
# handler=self._handle_subscription)
|
||||
self['xep_0050'].add_command(node='settings',
|
||||
name='📮️ Settings',
|
||||
handler=self._handle_settings)
|
||||
# TODO Join Filters and Settings into Preferences
|
||||
self['xep_0050'].add_command(node='filters',
|
||||
name='🛡️ Filters',
|
||||
handler=self._handle_filters)
|
||||
self['xep_0050'].add_command(node='bookmarks',
|
||||
name='📕 Bookmarks',
|
||||
handler=self._handle_bookmarks)
|
||||
# self['xep_0050'].add_command(node='roster',
|
||||
# name='📓 Roster', # 📋
|
||||
# handler=self._handle_roster)
|
||||
self['xep_0050'].add_command(node='settings',
|
||||
name='📮️ Settings',
|
||||
handler=self._handle_settings)
|
||||
if not self.is_component: # This will be changed with XEP-0222 XEP-0223
|
||||
self['xep_0050'].add_command(node='bookmarks',
|
||||
name='📕 Bookmarks',
|
||||
handler=self._handle_bookmarks)
|
||||
self['xep_0050'].add_command(node='roster',
|
||||
name='📓 Roster', # 📋
|
||||
handler=self._handle_roster)
|
||||
self['xep_0050'].add_command(node='help',
|
||||
name='📔️ Manual',
|
||||
handler=self._handle_help)
|
||||
|
|
@ -604,6 +609,102 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
return session
|
||||
|
||||
|
||||
async def _handle_subscription_add(self, iq, session):
|
||||
jid = session['from'].bare
|
||||
form = self['xep_0004'].make_form('form', 'Add Subscriptions')
|
||||
form['instructions'] = '📰️ Add a new subscription'
|
||||
options = form.add_field(var='subscription',
|
||||
# TODO Make it possible to add several subscriptions at once;
|
||||
# Similarly to BitTorrent trackers list
|
||||
# ftype='text-multi',
|
||||
# label='Subscription URLs',
|
||||
# desc=('Add subscriptions one time per '
|
||||
# 'subscription.'),
|
||||
ftype='text-single',
|
||||
label='Subscription URL',
|
||||
desc='Enter subscription URL.',
|
||||
required=True)
|
||||
form.add_field(var='scan',
|
||||
ftype='boolean',
|
||||
label='Scan',
|
||||
desc='Scan URL for validity.',
|
||||
value=True)
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_subscription_new
|
||||
session['has_next'] = True
|
||||
return session
|
||||
|
||||
|
||||
async def _handle_subscription_new(self, payload, session):
|
||||
jid = session['from'].bare
|
||||
jid_file = jid
|
||||
db_file = config.get_pathname_to_database(jid_file)
|
||||
url = payload['values']['subscription']
|
||||
result = await action.add_feed(db_file, url)
|
||||
if isinstance(result, list):
|
||||
results = result
|
||||
form = self['xep_0004'].make_form('form', 'Subscriptions')
|
||||
form['instructions'] = ('🔍️ Discovered {} subscriptions for {}'
|
||||
.format(len(results), url))
|
||||
options = form.add_field(var='subscriptions',
|
||||
ftype='list-single',
|
||||
label='Subscriptions',
|
||||
desc=('Select a subscription to add.'),
|
||||
required=True)
|
||||
for result in results:
|
||||
options.addOption(result['name'], result['url'])
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_subscription_editor
|
||||
session['has_next'] = True
|
||||
elif result['exist']:
|
||||
# response = ('News source "{}" is already listed '
|
||||
# 'in the subscription list at index '
|
||||
# '{}.\n{}'.format(result['name'], result['index'],
|
||||
# result['url']))
|
||||
# session['notes'] = [['warning', response]] # Not supported by Gajim
|
||||
# session['notes'] = [['info', response]]
|
||||
form = self['xep_0004'].make_form('result', 'Subscriptions')
|
||||
form['instructions'] = ('⚠️ Feed "{}" already exist as index {}'
|
||||
.format(result['name'], result['index']))
|
||||
options = form.add_field(var='subscriptions',
|
||||
ftype='text-single',
|
||||
label=result['url'],
|
||||
desc='Choose next to edit subscription.',
|
||||
value=result['url'])
|
||||
# FIXME payload value does not pass, only [].
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_subscription_editor
|
||||
session['has_next'] = True
|
||||
elif result['error']:
|
||||
response = ('Failed to load URL.'
|
||||
'\n\n'
|
||||
'Reason: {}'
|
||||
'\n\n'
|
||||
'URL: {}'
|
||||
.format(result['code'], url))
|
||||
session['notes'] = [['error', response]]
|
||||
session['next'] = None
|
||||
else:
|
||||
# response = ('News source "{}" has been '
|
||||
# 'added to subscription list.\n{}'
|
||||
# .format(result['name'], result['url']))
|
||||
# session['notes'] = [['info', response]]
|
||||
form = self['xep_0004'].make_form('result', 'Subscriptions')
|
||||
form['instructions'] = ('✅️ News source "{}" has been added to '
|
||||
'subscription list as index {}'
|
||||
.format(result['name'], result['index']))
|
||||
options = form.add_field(var='subscriptions',
|
||||
ftype='text-single',
|
||||
label=result['url'],
|
||||
desc='Choose next to edit subscription.',
|
||||
value=result['url'])
|
||||
# FIXME payload value does not pass, only [].
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_subscription_editor
|
||||
session['has_next'] = True
|
||||
return session
|
||||
|
||||
|
||||
async def _handle_subscriptions(self, iq, session):
|
||||
jid = session['from'].bare
|
||||
form = self['xep_0004'].make_form('form',
|
||||
|
|
@ -702,12 +803,13 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
|
||||
|
||||
async def _handle_subscription_editor(self, payload, session):
|
||||
urls = payload['values']['subscriptions']
|
||||
jid = session['from'].bare
|
||||
jid_file = jid
|
||||
db_file = config.get_pathname_to_database(jid_file)
|
||||
if 'subscriptions' in payload['values']:
|
||||
urls = payload['values']['subscriptions']
|
||||
url_count = len(urls)
|
||||
if url_count > 1:
|
||||
if isinstance(urls, list) and url_count > 1:
|
||||
form = self['xep_0004'].make_form('form', 'Subscription editor')
|
||||
form['instructions'] = '📂️ Editing {} subscriptions'.format(url_count)
|
||||
form.add_field(var='options',
|
||||
|
|
@ -743,10 +845,9 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
label='Name',
|
||||
value=title)
|
||||
# NOTE This does not look good in Gajim
|
||||
#url = form.add_field(ftype='fixed',
|
||||
# value=url)
|
||||
# url = form.add_field(ftype='fixed',
|
||||
# value=url)
|
||||
#url['validate']['datatype'] = 'xs:anyURI'
|
||||
|
||||
form.add_field(var='url',
|
||||
ftype='text-single',
|
||||
label='URL',
|
||||
|
|
@ -810,6 +911,45 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
return session
|
||||
|
||||
|
||||
async def _handle_subscription_selector(self, payload, session):
|
||||
jid = session['from'].bare
|
||||
form = self['xep_0004'].make_form('form',
|
||||
'Discovered ubscriptions for {}'.format(jid))
|
||||
form['instructions'] = ('📰️ Select a subscriptions to add\n'
|
||||
'Subsciptions discovered for {}'
|
||||
.format(url))
|
||||
# form.addField(var='interval',
|
||||
# ftype='text-single',
|
||||
# label='Interval period')
|
||||
options = form.add_field(var='subscriptions',
|
||||
ftype='list-multi',
|
||||
label='Subscriptions',
|
||||
desc=('Select subscriptions to perform '
|
||||
'actions upon.'),
|
||||
required=True)
|
||||
jid_file = jid
|
||||
db_file = config.get_pathname_to_database(jid_file)
|
||||
subscriptions = await sqlite.get_feeds(db_file)
|
||||
subscriptions = sorted(subscriptions, key=lambda x: x[0])
|
||||
for subscription in subscriptions:
|
||||
title = subscription[0]
|
||||
url = subscription[1]
|
||||
options.addOption(title, url)
|
||||
# options = form.add_field(var='action',
|
||||
# ftype='list-single',
|
||||
# label='Action',
|
||||
# value='none')
|
||||
# options.addOption('None', 'none')
|
||||
# options.addOption('Reset', 'reset')
|
||||
# options.addOption('Enable', 'enable')
|
||||
# options.addOption('Disable', 'disable')
|
||||
# options.addOption('Delete', 'delete')
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_subscription_editor
|
||||
session['has_next'] = True
|
||||
return session
|
||||
|
||||
|
||||
async def _handle_subscription_complete(self, payload, session):
|
||||
form = self['xep_0004'].make_form('form', 'Subscription editor')
|
||||
form['instructions'] = ('📁️ Subscription #{} has been {}'
|
||||
|
|
@ -863,10 +1003,10 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
'Import data for {}'.format(jid))
|
||||
form['instructions'] = '🗞️ Import feeds from OPML'
|
||||
url = form.add_field(var='url',
|
||||
ftype='text-single',
|
||||
label='URL',
|
||||
desc='Enter URL to OPML file.',
|
||||
required=True)
|
||||
ftype='text-single',
|
||||
label='URL',
|
||||
desc='Enter URL to OPML file.',
|
||||
required=True)
|
||||
url['validate']['datatype'] = 'xs:anyURI'
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_import_complete
|
||||
|
|
@ -1194,7 +1334,7 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
options = form.add_field(var='interval',
|
||||
ftype='list-single',
|
||||
label='Interval',
|
||||
desc='Set interval update (in hours).',
|
||||
desc='Interval update (in hours).',
|
||||
value=value)
|
||||
options['validate']['datatype'] = 'xs:integer'
|
||||
options['validate']['range'] = { 'minimum': 1, 'maximum': 48 }
|
||||
|
|
@ -1207,6 +1347,21 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
else:
|
||||
i += 1
|
||||
|
||||
value = config.get_setting_value(db_file, 'quantum')
|
||||
value = str(value)
|
||||
options = form.add_field(var='quantum',
|
||||
ftype='list-single',
|
||||
label='Amount',
|
||||
desc='Amount of items per update.',
|
||||
value=value)
|
||||
options['validate']['datatype'] = 'xs:integer'
|
||||
options['validate']['range'] = { 'minimum': 1, 'maximum': 5 }
|
||||
i = 1
|
||||
while i <= 5:
|
||||
x = str(i)
|
||||
options.addOption(x, x)
|
||||
i += 1
|
||||
|
||||
value = config.get_setting_value(db_file, 'archive')
|
||||
value = str(value)
|
||||
options = form.add_field(var='archive',
|
||||
|
|
@ -1222,21 +1377,6 @@ class Slixfeed(slixmpp.ClientXMPP):
|
|||
options.addOption(x, x)
|
||||
i += 50
|
||||
|
||||
value = config.get_setting_value(db_file, 'quantum')
|
||||
value = str(value)
|
||||
options = form.add_field(var='quantum',
|
||||
ftype='list-single',
|
||||
label='Amount',
|
||||
desc='Set amount of items per update.',
|
||||
value=value)
|
||||
options['validate']['datatype'] = 'xs:integer'
|
||||
options['validate']['range'] = { 'minimum': 1, 'maximum': 5 }
|
||||
i = 1
|
||||
while i <= 5:
|
||||
x = str(i)
|
||||
options.addOption(x, x)
|
||||
i += 1
|
||||
|
||||
session['payload'] = form
|
||||
session['next'] = self._handle_settings_complete
|
||||
session['has_next'] = False
|
||||
|
|
|
|||
|
|
@ -8,6 +8,10 @@ TODO
|
|||
1) Look into self.set_jid in order to be able to join to groupchats
|
||||
https://slixmpp.readthedocs.io/en/latest/api/basexmpp.html#slixmpp.basexmpp.BaseXMPP.set_jid
|
||||
|
||||
2) czar
|
||||
https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0223.html
|
||||
https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0222.html#module-slixmpp.plugins.xep_0222
|
||||
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
|
|
|||
|
|
@ -512,8 +512,7 @@ async def message(self, message):
|
|||
url = message_text
|
||||
# task.clean_tasks_xmpp(self, jid, ['status'])
|
||||
status_type = 'dnd'
|
||||
status_message = ('📫️ Processing request '
|
||||
'to fetch data from {}'
|
||||
status_message = ('📫️ Processing request to fetch data from {}'
|
||||
.format(url))
|
||||
XmppPresence.send(self, jid, status_message,
|
||||
status_type=status_type)
|
||||
|
|
@ -522,7 +521,31 @@ async def message(self, message):
|
|||
url = (uri.replace_hostname(url, 'feed')) or url
|
||||
db_file = config.get_pathname_to_database(jid_file)
|
||||
# try:
|
||||
response = await action.add_feed(db_file, url)
|
||||
result = await action.add_feed(db_file, url)
|
||||
if isinstance(result, list):
|
||||
results = result
|
||||
response = ("Web feeds found for {}\n\n```\n"
|
||||
.format(url))
|
||||
for result in results:
|
||||
response += ("Title : {}\n"
|
||||
"Link : {}\n"
|
||||
"\n"
|
||||
.format(result['name'], result['url']))
|
||||
response += ('```\nTotal of {} feeds.'
|
||||
.format(len(results)))
|
||||
elif result['exist']:
|
||||
response = ('> {}\nNews source "{}" is already '
|
||||
'listed in the subscription list at '
|
||||
'index {}'.format(result['url'],
|
||||
result['name'],
|
||||
result['index']))
|
||||
elif result['error']:
|
||||
response = ('> {}\nFailed to load URL. Reason: {}'
|
||||
.format(url, result['code']))
|
||||
else:
|
||||
response = ('> {}\nNews source "{}" has been '
|
||||
'added to subscription list.'
|
||||
.format(result['url'], result['name']))
|
||||
# task.clean_tasks_xmpp(self, jid, ['status'])
|
||||
await task.start_tasks_xmpp(self, jid, ['status'])
|
||||
# except:
|
||||
|
|
|
|||
Loading…
Reference in a new issue