sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

Add for Add Subscription.

Browse source 
Segregating more code as one task per function, particularly adding of new subscription.
Fix allow/deny keys.
This commit is contained in:
Schimon Jehudah 2024-02-16 01:46:04 +00:00
parent 6c0c571c1d
commit 4e193a2b02
7 changed files with 284 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
slixfeed/action.py
View file

@ -737,22 +737,26 @@ async def add_feed(db_file, url):
updated = ''
version = feed["version"]
entries = len(feed["entries"])
await sqlite.insert_feed(
db_file, url,
title=title,
entries=entries,
version=version,
encoding=encoding,
language=language,
status_code=status_code,
updated=updated
)
await sqlite.insert_feed(db_file, url,
title=title,
entries=entries,
version=version,
encoding=encoding,
language=language,
status_code=status_code,
updated=updated)
await scan(db_file, url)
old = config.get_setting_value(db_file, "old")
if not old:
feed_id = await sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
await sqlite.mark_feed_as_read(db_file, feed_id)
result_final = {'url' : url,
'index' : feed_id,
'name' : title,
'code' : status_code,
'error' : False,
'exist' : False}
response = ('> {}\nNews source "{}" has been '
'added to subscription list.'
.format(url, title))
@ -783,47 +787,64 @@ async def add_feed(db_file, url):
updated = ''
version = 'json' + feed["version"].split('/').pop()
entries = len(feed["items"])
await sqlite.insert_feed(
db_file, url,
title=title,
entries=entries,
version=version,
encoding=encoding,
language=language,
status_code=status_code,
updated=updated
)
await scan_json(
db_file, url)
await sqlite.insert_feed(db_file, url,
title=title,
entries=entries,
version=version,
encoding=encoding,
language=language,
status_code=status_code,
updated=updated)
await scan_json(db_file, url)
old = config.get_setting_value(db_file, "old")
if not old:
feed_id = await sqlite.get_feed_id(db_file, url)
feed_id = feed_id[0]
await sqlite.mark_feed_as_read(db_file, feed_id)
result_final = {'url' : url,
'index' : feed_id,
'name' : title,
'code' : status_code,
'error' : False,
'exist' : False}
response = ('> {}\nNews source "{}" has been '
'added to subscription list.'
.format(url, title))
break
else:
result = await crawl.probe_page(
url, document)
if isinstance(result, str):
response = result
# NOTE Do not be tempted to return a compact dictionary.
# That is, dictionary within dictionary
# Return multimple dictionaries.
result = await crawl.probe_page(url, document)
if isinstance(result, list):
result_final = result
break
else:
url = result[0]
url = result['url']
else:
result_final = {'url' : url,
'index' : None,
'name' : None,
'code' : status_code,
'error' : True,
'exist' : False}
response = ('> {}\nFailed to load URL. Reason: {}'
.format(url, status_code))
break
else:
ix = exist[0]
name = exist[1]
result_final = {'url' : url,
'index' : ix,
'name' : name,
'code' : None,
'error' : False,
'exist' : True}
response = ('> {}\nNews source "{}" is already '
'listed in the subscription list at '
'index {}'.format(url, name, ix))
break
return response
return result_final
async def scan_json(db_file, url):

4
slixfeed/assets/information.toml
View file

@ -154,9 +154,9 @@ thanks = [
"Strix from Loqi",
"Thibaud Guerin (SalixOS)",
"Thorsten Fröhlich (France)",
"Thorsten Mühlfelder (SalixOS, Germany)",
"Thorsten Mühlfelder (SalixOS, Germany)",
"Tim Beech (SalixOS, Brazil)",
"Yann Leboulanger (Gajim, France)"
"Yann Leboulanger (Gajim, France)"
]
xmpp = """

5
slixfeed/assets/lists.toml
View file

@ -147,7 +147,7 @@ pathnames = [
# would get the best out of this news application.
# Entries with the following keywords will not be filtered
filter-allow = [
allow = [
"akkoma",
"censorship",
"earthing",
@ -187,7 +187,7 @@ filter-allow = [
]
# Entries with the following keywords will be filtered
filter-deny = [
deny = [
# brands
# Almost every time you see a brand name in title or content, it is because
# someone, usually a marketing agency or a venture capital firm, has paid for
@ -264,6 +264,7 @@ filter-deny = [
"homosex",
"lesbian",
"lgbt",
"nonbinary",
"nude",
"nudity",
"onlyfans",

46
slixfeed/crawl.py
View file

@ -300,7 +300,8 @@ async def process_feed_selection(url, urls):
feeds = {}
for i in urls:
res = await fetch.http(i)
if res[1] == 200:
status_code = res[1]
if status_code == 200:
try:
feeds[i] = [parse(res[0])]
except:
@ -308,8 +309,7 @@ async def process_feed_selection(url, urls):
message = (
"Web feeds found for {}\n\n```\n"
).format(url)
counter = 0
feed_url_mark = 0
urls = []
for feed_url in feeds:
# try:
# res = await fetch.http(feed)
@ -327,24 +327,28 @@ async def process_feed_selection(url, urls):
continue
if feed_amnt:
# NOTE Because there could be many false positives
# which are revealed in second phase of scan, we
# could end with a single feed, which would be
# listed instead of fetched, so feed_url_mark is
# utilized in order to make fetch possible.
feed_url_mark = [feed_url]
counter += 1
message += (
"Title : {}\n"
"Link : {}\n"
"\n"
).format(feed_name, feed_url)
if counter > 1:
message += (
"```\nTotal of {} feeds."
).format(counter)
result = message
elif feed_url_mark:
result = feed_url_mark
# which are revealed in second phase of scan, we
# could end with a single feed, which would be
# listed instead of fetched, so feed_url_mark is
# utilized in order to make fetch possible.
# NOTE feed_url_mark was a variable which stored
# single URL (probably first accepted as valid)
# in order to get an indication whether a single
# URL has been fetched, so that the receiving
# function will scan that single URL instead of
# listing it as a message.
url = {'url' : feed_url,
'index' : None,
'name' : feed_name,
'code' : status_code,
'error' : False,
'exist' : None}
urls.extend([url])
count = len(urls)
if count > 1:
result = urls
elif count:
result = urls[0]
else:
result = None
return result

210
slixfeed/xmpp/client.py
View file

@ -469,8 +469,11 @@ class Slixfeed(slixmpp.ClientXMPP):
# )
# if jid == config.get_value('accounts', 'XMPP', 'operator'):
self['xep_0050'].add_command(node='subscription',
name='➕️ Add Subscription',
handler=self._handle_subscription_add)
self['xep_0050'].add_command(node='subscriptions',
name='📰️ Subscriptions',
name='📰️ Browse Subscriptions',
handler=self._handle_subscriptions)
# self['xep_0050'].add_command(node='subscriptions_cat',
# name='🔖️ Categories',
@ -481,18 +484,20 @@ class Slixfeed(slixmpp.ClientXMPP):
# self['xep_0050'].add_command(node='subscriptions_index',
# name='📑️ Index (A - Z)',
# handler=self._handle_subscription)
self['xep_0050'].add_command(node='settings',
name='📮️ Settings',
handler=self._handle_settings)
# TODO Join Filters and Settings into Preferences
self['xep_0050'].add_command(node='filters',
name='🛡️ Filters',
handler=self._handle_filters)
self['xep_0050'].add_command(node='bookmarks',
name='📕 Bookmarks',
handler=self._handle_bookmarks)
# self['xep_0050'].add_command(node='roster',
# name='📓 Roster', # 📋
# handler=self._handle_roster)
self['xep_0050'].add_command(node='settings',
name='📮️ Settings',
handler=self._handle_settings)
if not self.is_component: # This will be changed with XEP-0222 XEP-0223
self['xep_0050'].add_command(node='bookmarks',
name='📕 Bookmarks',
handler=self._handle_bookmarks)
self['xep_0050'].add_command(node='roster',
name='📓 Roster', # 📋
handler=self._handle_roster)
self['xep_0050'].add_command(node='help',
name='📔️ Manual',
handler=self._handle_help)
@ -604,6 +609,102 @@ class Slixfeed(slixmpp.ClientXMPP):
return session
async def _handle_subscription_add(self, iq, session):
jid = session['from'].bare
form = self['xep_0004'].make_form('form', 'Add Subscriptions')
form['instructions'] = '📰️ Add a new subscription'
options = form.add_field(var='subscription',
# TODO Make it possible to add several subscriptions at once;
# Similarly to BitTorrent trackers list
# ftype='text-multi',
# label='Subscription URLs',
# desc=('Add subscriptions one time per '
# 'subscription.'),
ftype='text-single',
label='Subscription URL',
desc='Enter subscription URL.',
required=True)
form.add_field(var='scan',
ftype='boolean',
label='Scan',
desc='Scan URL for validity.',
value=True)
session['payload'] = form
session['next'] = self._handle_subscription_new
session['has_next'] = True
return session
async def _handle_subscription_new(self, payload, session):
jid = session['from'].bare
jid_file = jid
db_file = config.get_pathname_to_database(jid_file)
url = payload['values']['subscription']
result = await action.add_feed(db_file, url)
if isinstance(result, list):
results = result
form = self['xep_0004'].make_form('form', 'Subscriptions')
form['instructions'] = ('🔍️ Discovered {} subscriptions for {}'
.format(len(results), url))
options = form.add_field(var='subscriptions',
ftype='list-single',
label='Subscriptions',
desc=('Select a subscription to add.'),
required=True)
for result in results:
options.addOption(result['name'], result['url'])
session['payload'] = form
session['next'] = self._handle_subscription_editor
session['has_next'] = True
elif result['exist']:
# response = ('News source "{}" is already listed '
# 'in the subscription list at index '
# '{}.\n{}'.format(result['name'], result['index'],
# result['url']))
# session['notes'] = [['warning', response]] # Not supported by Gajim
# session['notes'] = [['info', response]]
form = self['xep_0004'].make_form('result', 'Subscriptions')
form['instructions'] = ('⚠️ Feed "{}" already exist as index {}'
.format(result['name'], result['index']))
options = form.add_field(var='subscriptions',
ftype='text-single',
label=result['url'],
desc='Choose next to edit subscription.',
value=result['url'])
# FIXME payload value does not pass, only [].
session['payload'] = form
session['next'] = self._handle_subscription_editor
session['has_next'] = True
elif result['error']:
response = ('Failed to load URL.'
'\n\n'
'Reason: {}'
'\n\n'
'URL: {}'
.format(result['code'], url))
session['notes'] = [['error', response]]
session['next'] = None
else:
# response = ('News source "{}" has been '
# 'added to subscription list.\n{}'
# .format(result['name'], result['url']))
# session['notes'] = [['info', response]]
form = self['xep_0004'].make_form('result', 'Subscriptions')
form['instructions'] = ('✅️ News source "{}" has been added to '
'subscription list as index {}'
.format(result['name'], result['index']))
options = form.add_field(var='subscriptions',
ftype='text-single',
label=result['url'],
desc='Choose next to edit subscription.',
value=result['url'])
# FIXME payload value does not pass, only [].
session['payload'] = form
session['next'] = self._handle_subscription_editor
session['has_next'] = True
return session
async def _handle_subscriptions(self, iq, session):
jid = session['from'].bare
form = self['xep_0004'].make_form('form',
@ -702,12 +803,13 @@ class Slixfeed(slixmpp.ClientXMPP):
async def _handle_subscription_editor(self, payload, session):
urls = payload['values']['subscriptions']
jid = session['from'].bare
jid_file = jid
db_file = config.get_pathname_to_database(jid_file)
if 'subscriptions' in payload['values']:
urls = payload['values']['subscriptions']
url_count = len(urls)
if url_count > 1:
if isinstance(urls, list) and url_count > 1:
form = self['xep_0004'].make_form('form', 'Subscription editor')
form['instructions'] = '📂️ Editing {} subscriptions'.format(url_count)
form.add_field(var='options',
@ -743,10 +845,9 @@ class Slixfeed(slixmpp.ClientXMPP):
label='Name',
value=title)
# NOTE This does not look good in Gajim
#url = form.add_field(ftype='fixed',
# value=url)
# url = form.add_field(ftype='fixed',
# value=url)
#url['validate']['datatype'] = 'xs:anyURI'
form.add_field(var='url',
ftype='text-single',
label='URL',
@ -810,6 +911,45 @@ class Slixfeed(slixmpp.ClientXMPP):
return session
async def _handle_subscription_selector(self, payload, session):
jid = session['from'].bare
form = self['xep_0004'].make_form('form',
'Discovered ubscriptions for {}'.format(jid))
form['instructions'] = ('📰️ Select a subscriptions to add\n'
'Subsciptions discovered for {}'
.format(url))
# form.addField(var='interval',
# ftype='text-single',
# label='Interval period')
options = form.add_field(var='subscriptions',
ftype='list-multi',
label='Subscriptions',
desc=('Select subscriptions to perform '
'actions upon.'),
required=True)
jid_file = jid
db_file = config.get_pathname_to_database(jid_file)
subscriptions = await sqlite.get_feeds(db_file)
subscriptions = sorted(subscriptions, key=lambda x: x[0])
for subscription in subscriptions:
title = subscription[0]
url = subscription[1]
options.addOption(title, url)
# options = form.add_field(var='action',
# ftype='list-single',
# label='Action',
# value='none')
# options.addOption('None', 'none')
# options.addOption('Reset', 'reset')
# options.addOption('Enable', 'enable')
# options.addOption('Disable', 'disable')
# options.addOption('Delete', 'delete')
session['payload'] = form
session['next'] = self._handle_subscription_editor
session['has_next'] = True
return session
async def _handle_subscription_complete(self, payload, session):
form = self['xep_0004'].make_form('form', 'Subscription editor')
form['instructions'] = ('📁️ Subscription #{} has been {}'
@ -863,10 +1003,10 @@ class Slixfeed(slixmpp.ClientXMPP):
'Import data for {}'.format(jid))
form['instructions'] = '🗞️ Import feeds from OPML'
url = form.add_field(var='url',
ftype='text-single',
label='URL',
desc='Enter URL to OPML file.',
required=True)
ftype='text-single',
label='URL',
desc='Enter URL to OPML file.',
required=True)
url['validate']['datatype'] = 'xs:anyURI'
session['payload'] = form
session['next'] = self._handle_import_complete
@ -1194,7 +1334,7 @@ class Slixfeed(slixmpp.ClientXMPP):
options = form.add_field(var='interval',
ftype='list-single',
label='Interval',
desc='Set interval update (in hours).',
desc='Interval update (in hours).',
value=value)
options['validate']['datatype'] = 'xs:integer'
options['validate']['range'] = { 'minimum': 1, 'maximum': 48 }
@ -1207,6 +1347,21 @@ class Slixfeed(slixmpp.ClientXMPP):
else:
i += 1
value = config.get_setting_value(db_file, 'quantum')
value = str(value)
options = form.add_field(var='quantum',
ftype='list-single',
label='Amount',
desc='Amount of items per update.',
value=value)
options['validate']['datatype'] = 'xs:integer'
options['validate']['range'] = { 'minimum': 1, 'maximum': 5 }
i = 1
while i <= 5:
x = str(i)
options.addOption(x, x)
i += 1
value = config.get_setting_value(db_file, 'archive')
value = str(value)
options = form.add_field(var='archive',
@ -1222,21 +1377,6 @@ class Slixfeed(slixmpp.ClientXMPP):
options.addOption(x, x)
i += 50
value = config.get_setting_value(db_file, 'quantum')
value = str(value)
options = form.add_field(var='quantum',
ftype='list-single',
label='Amount',
desc='Set amount of items per update.',
value=value)
options['validate']['datatype'] = 'xs:integer'
options['validate']['range'] = { 'minimum': 1, 'maximum': 5 }
i = 1
while i <= 5:
x = str(i)
options.addOption(x, x)
i += 1
session['payload'] = form
session['next'] = self._handle_settings_complete
session['has_next'] = False

4
slixfeed/xmpp/component.py
View file

@ -8,6 +8,10 @@ TODO
1) Look into self.set_jid in order to be able to join to groupchats
https://slixmpp.readthedocs.io/en/latest/api/basexmpp.html#slixmpp.basexmpp.BaseXMPP.set_jid
2) czar
https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0223.html
https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0222.html#module-slixmpp.plugins.xep_0222
"""
import asyncio

29
slixfeed/xmpp/process.py
View file

@ -512,8 +512,7 @@ async def message(self, message):
url = message_text
# task.clean_tasks_xmpp(self, jid, ['status'])
status_type = 'dnd'
status_message = ('📫️ Processing request '
'to fetch data from {}'
status_message = ('📫️ Processing request to fetch data from {}'
.format(url))
XmppPresence.send(self, jid, status_message,
status_type=status_type)
@ -522,7 +521,31 @@ async def message(self, message):
url = (uri.replace_hostname(url, 'feed')) or url
db_file = config.get_pathname_to_database(jid_file)
# try:
response = await action.add_feed(db_file, url)
result = await action.add_feed(db_file, url)
if isinstance(result, list):
results = result
response = ("Web feeds found for {}\n\n```\n"
.format(url))
for result in results:
response += ("Title : {}\n"
"Link : {}\n"
"\n"
.format(result['name'], result['url']))
response += ('```\nTotal of {} feeds.'
.format(len(results)))
elif result['exist']:
response = ('> {}\nNews source "{}" is already '
'listed in the subscription list at '
'index {}'.format(result['url'],
result['name'],
result['index']))
elif result['error']:
response = ('> {}\nFailed to load URL. Reason: {}'
.format(url, result['code']))
else:
response = ('> {}\nNews source "{}" has been '
'added to subscription list.'
.format(result['url'], result['name']))
# task.clean_tasks_xmpp(self, jid, ['status'])
await task.start_tasks_xmpp(self, jid, ['status'])
# except: