diff --git a/slixfeed/action.py b/slixfeed/action.py index 061d98f..3fb262a 100644 --- a/slixfeed/action.py +++ b/slixfeed/action.py @@ -737,22 +737,26 @@ async def add_feed(db_file, url): updated = '' version = feed["version"] entries = len(feed["entries"]) - await sqlite.insert_feed( - db_file, url, - title=title, - entries=entries, - version=version, - encoding=encoding, - language=language, - status_code=status_code, - updated=updated - ) + await sqlite.insert_feed(db_file, url, + title=title, + entries=entries, + version=version, + encoding=encoding, + language=language, + status_code=status_code, + updated=updated) await scan(db_file, url) old = config.get_setting_value(db_file, "old") if not old: feed_id = await sqlite.get_feed_id(db_file, url) feed_id = feed_id[0] await sqlite.mark_feed_as_read(db_file, feed_id) + result_final = {'url' : url, + 'index' : feed_id, + 'name' : title, + 'code' : status_code, + 'error' : False, + 'exist' : False} response = ('> {}\nNews source "{}" has been ' 'added to subscription list.' .format(url, title)) @@ -783,47 +787,64 @@ async def add_feed(db_file, url): updated = '' version = 'json' + feed["version"].split('/').pop() entries = len(feed["items"]) - await sqlite.insert_feed( - db_file, url, - title=title, - entries=entries, - version=version, - encoding=encoding, - language=language, - status_code=status_code, - updated=updated - ) - await scan_json( - db_file, url) + await sqlite.insert_feed(db_file, url, + title=title, + entries=entries, + version=version, + encoding=encoding, + language=language, + status_code=status_code, + updated=updated) + await scan_json(db_file, url) old = config.get_setting_value(db_file, "old") if not old: feed_id = await sqlite.get_feed_id(db_file, url) feed_id = feed_id[0] await sqlite.mark_feed_as_read(db_file, feed_id) + result_final = {'url' : url, + 'index' : feed_id, + 'name' : title, + 'code' : status_code, + 'error' : False, + 'exist' : False} response = ('> {}\nNews source "{}" has been ' 'added to subscription list.' .format(url, title)) break else: - result = await crawl.probe_page( - url, document) - if isinstance(result, str): - response = result + # NOTE Do not be tempted to return a compact dictionary. + # That is, dictionary within dictionary + # Return multimple dictionaries. + result = await crawl.probe_page(url, document) + if isinstance(result, list): + result_final = result break else: - url = result[0] + url = result['url'] else: + result_final = {'url' : url, + 'index' : None, + 'name' : None, + 'code' : status_code, + 'error' : True, + 'exist' : False} response = ('> {}\nFailed to load URL. Reason: {}' .format(url, status_code)) break else: ix = exist[0] name = exist[1] + result_final = {'url' : url, + 'index' : ix, + 'name' : name, + 'code' : None, + 'error' : False, + 'exist' : True} response = ('> {}\nNews source "{}" is already ' 'listed in the subscription list at ' 'index {}'.format(url, name, ix)) break - return response + return result_final async def scan_json(db_file, url): diff --git a/slixfeed/assets/information.toml b/slixfeed/assets/information.toml index 8f7ee21..cd8e1ad 100644 --- a/slixfeed/assets/information.toml +++ b/slixfeed/assets/information.toml @@ -154,9 +154,9 @@ thanks = [ "Strix from Loqi", "Thibaud Guerin (SalixOS)", "Thorsten Fröhlich (France)", - "Thorsten Mühlfelder (SalixOS, Germany)", + "Thorsten Mühlfelder (SalixOS, Germany)", "Tim Beech (SalixOS, Brazil)", - "Yann Leboulanger (Gajim, France)" + "Yann Leboulanger (Gajim, France)" ] xmpp = """ diff --git a/slixfeed/assets/lists.toml b/slixfeed/assets/lists.toml index 25185e9..8450b72 100644 --- a/slixfeed/assets/lists.toml +++ b/slixfeed/assets/lists.toml @@ -147,7 +147,7 @@ pathnames = [ # would get the best out of this news application. # Entries with the following keywords will not be filtered -filter-allow = [ +allow = [ "akkoma", "censorship", "earthing", @@ -187,7 +187,7 @@ filter-allow = [ ] # Entries with the following keywords will be filtered -filter-deny = [ +deny = [ # brands # Almost every time you see a brand name in title or content, it is because # someone, usually a marketing agency or a venture capital firm, has paid for @@ -264,6 +264,7 @@ filter-deny = [ "homosex", "lesbian", "lgbt", +"nonbinary", "nude", "nudity", "onlyfans", diff --git a/slixfeed/crawl.py b/slixfeed/crawl.py index ce3df7f..79691b0 100644 --- a/slixfeed/crawl.py +++ b/slixfeed/crawl.py @@ -300,7 +300,8 @@ async def process_feed_selection(url, urls): feeds = {} for i in urls: res = await fetch.http(i) - if res[1] == 200: + status_code = res[1] + if status_code == 200: try: feeds[i] = [parse(res[0])] except: @@ -308,8 +309,7 @@ async def process_feed_selection(url, urls): message = ( "Web feeds found for {}\n\n```\n" ).format(url) - counter = 0 - feed_url_mark = 0 + urls = [] for feed_url in feeds: # try: # res = await fetch.http(feed) @@ -327,24 +327,28 @@ async def process_feed_selection(url, urls): continue if feed_amnt: # NOTE Because there could be many false positives - # which are revealed in second phase of scan, we - # could end with a single feed, which would be - # listed instead of fetched, so feed_url_mark is - # utilized in order to make fetch possible. - feed_url_mark = [feed_url] - counter += 1 - message += ( - "Title : {}\n" - "Link : {}\n" - "\n" - ).format(feed_name, feed_url) - if counter > 1: - message += ( - "```\nTotal of {} feeds." - ).format(counter) - result = message - elif feed_url_mark: - result = feed_url_mark + # which are revealed in second phase of scan, we + # could end with a single feed, which would be + # listed instead of fetched, so feed_url_mark is + # utilized in order to make fetch possible. + # NOTE feed_url_mark was a variable which stored + # single URL (probably first accepted as valid) + # in order to get an indication whether a single + # URL has been fetched, so that the receiving + # function will scan that single URL instead of + # listing it as a message. + url = {'url' : feed_url, + 'index' : None, + 'name' : feed_name, + 'code' : status_code, + 'error' : False, + 'exist' : None} + urls.extend([url]) + count = len(urls) + if count > 1: + result = urls + elif count: + result = urls[0] else: result = None return result diff --git a/slixfeed/xmpp/client.py b/slixfeed/xmpp/client.py index 797e9bc..288d291 100644 --- a/slixfeed/xmpp/client.py +++ b/slixfeed/xmpp/client.py @@ -469,8 +469,11 @@ class Slixfeed(slixmpp.ClientXMPP): # ) # if jid == config.get_value('accounts', 'XMPP', 'operator'): + self['xep_0050'].add_command(node='subscription', + name='➕️ Add Subscription', + handler=self._handle_subscription_add) self['xep_0050'].add_command(node='subscriptions', - name='📰️ Subscriptions', + name='📰️ Browse Subscriptions', handler=self._handle_subscriptions) # self['xep_0050'].add_command(node='subscriptions_cat', # name='🔖️ Categories', @@ -481,18 +484,20 @@ class Slixfeed(slixmpp.ClientXMPP): # self['xep_0050'].add_command(node='subscriptions_index', # name='📑️ Index (A - Z)', # handler=self._handle_subscription) - self['xep_0050'].add_command(node='settings', - name='📮️ Settings', - handler=self._handle_settings) + # TODO Join Filters and Settings into Preferences self['xep_0050'].add_command(node='filters', name='🛡️ Filters', handler=self._handle_filters) - self['xep_0050'].add_command(node='bookmarks', - name='📕 Bookmarks', - handler=self._handle_bookmarks) - # self['xep_0050'].add_command(node='roster', - # name='📓 Roster', # 📋 - # handler=self._handle_roster) + self['xep_0050'].add_command(node='settings', + name='📮️ Settings', + handler=self._handle_settings) + if not self.is_component: # This will be changed with XEP-0222 XEP-0223 + self['xep_0050'].add_command(node='bookmarks', + name='📕 Bookmarks', + handler=self._handle_bookmarks) + self['xep_0050'].add_command(node='roster', + name='📓 Roster', # 📋 + handler=self._handle_roster) self['xep_0050'].add_command(node='help', name='📔️ Manual', handler=self._handle_help) @@ -604,6 +609,102 @@ class Slixfeed(slixmpp.ClientXMPP): return session + async def _handle_subscription_add(self, iq, session): + jid = session['from'].bare + form = self['xep_0004'].make_form('form', 'Add Subscriptions') + form['instructions'] = '📰️ Add a new subscription' + options = form.add_field(var='subscription', + # TODO Make it possible to add several subscriptions at once; + # Similarly to BitTorrent trackers list + # ftype='text-multi', + # label='Subscription URLs', + # desc=('Add subscriptions one time per ' + # 'subscription.'), + ftype='text-single', + label='Subscription URL', + desc='Enter subscription URL.', + required=True) + form.add_field(var='scan', + ftype='boolean', + label='Scan', + desc='Scan URL for validity.', + value=True) + session['payload'] = form + session['next'] = self._handle_subscription_new + session['has_next'] = True + return session + + + async def _handle_subscription_new(self, payload, session): + jid = session['from'].bare + jid_file = jid + db_file = config.get_pathname_to_database(jid_file) + url = payload['values']['subscription'] + result = await action.add_feed(db_file, url) + if isinstance(result, list): + results = result + form = self['xep_0004'].make_form('form', 'Subscriptions') + form['instructions'] = ('🔍️ Discovered {} subscriptions for {}' + .format(len(results), url)) + options = form.add_field(var='subscriptions', + ftype='list-single', + label='Subscriptions', + desc=('Select a subscription to add.'), + required=True) + for result in results: + options.addOption(result['name'], result['url']) + session['payload'] = form + session['next'] = self._handle_subscription_editor + session['has_next'] = True + elif result['exist']: + # response = ('News source "{}" is already listed ' + # 'in the subscription list at index ' + # '{}.\n{}'.format(result['name'], result['index'], + # result['url'])) + # session['notes'] = [['warning', response]] # Not supported by Gajim + # session['notes'] = [['info', response]] + form = self['xep_0004'].make_form('result', 'Subscriptions') + form['instructions'] = ('⚠️ Feed "{}" already exist as index {}' + .format(result['name'], result['index'])) + options = form.add_field(var='subscriptions', + ftype='text-single', + label=result['url'], + desc='Choose next to edit subscription.', + value=result['url']) + # FIXME payload value does not pass, only []. + session['payload'] = form + session['next'] = self._handle_subscription_editor + session['has_next'] = True + elif result['error']: + response = ('Failed to load URL.' + '\n\n' + 'Reason: {}' + '\n\n' + 'URL: {}' + .format(result['code'], url)) + session['notes'] = [['error', response]] + session['next'] = None + else: + # response = ('News source "{}" has been ' + # 'added to subscription list.\n{}' + # .format(result['name'], result['url'])) + # session['notes'] = [['info', response]] + form = self['xep_0004'].make_form('result', 'Subscriptions') + form['instructions'] = ('✅️ News source "{}" has been added to ' + 'subscription list as index {}' + .format(result['name'], result['index'])) + options = form.add_field(var='subscriptions', + ftype='text-single', + label=result['url'], + desc='Choose next to edit subscription.', + value=result['url']) + # FIXME payload value does not pass, only []. + session['payload'] = form + session['next'] = self._handle_subscription_editor + session['has_next'] = True + return session + + async def _handle_subscriptions(self, iq, session): jid = session['from'].bare form = self['xep_0004'].make_form('form', @@ -702,12 +803,13 @@ class Slixfeed(slixmpp.ClientXMPP): async def _handle_subscription_editor(self, payload, session): - urls = payload['values']['subscriptions'] jid = session['from'].bare jid_file = jid db_file = config.get_pathname_to_database(jid_file) + if 'subscriptions' in payload['values']: + urls = payload['values']['subscriptions'] url_count = len(urls) - if url_count > 1: + if isinstance(urls, list) and url_count > 1: form = self['xep_0004'].make_form('form', 'Subscription editor') form['instructions'] = '📂️ Editing {} subscriptions'.format(url_count) form.add_field(var='options', @@ -743,10 +845,9 @@ class Slixfeed(slixmpp.ClientXMPP): label='Name', value=title) # NOTE This does not look good in Gajim - #url = form.add_field(ftype='fixed', - # value=url) + # url = form.add_field(ftype='fixed', + # value=url) #url['validate']['datatype'] = 'xs:anyURI' - form.add_field(var='url', ftype='text-single', label='URL', @@ -810,6 +911,45 @@ class Slixfeed(slixmpp.ClientXMPP): return session + async def _handle_subscription_selector(self, payload, session): + jid = session['from'].bare + form = self['xep_0004'].make_form('form', + 'Discovered ubscriptions for {}'.format(jid)) + form['instructions'] = ('📰️ Select a subscriptions to add\n' + 'Subsciptions discovered for {}' + .format(url)) + # form.addField(var='interval', + # ftype='text-single', + # label='Interval period') + options = form.add_field(var='subscriptions', + ftype='list-multi', + label='Subscriptions', + desc=('Select subscriptions to perform ' + 'actions upon.'), + required=True) + jid_file = jid + db_file = config.get_pathname_to_database(jid_file) + subscriptions = await sqlite.get_feeds(db_file) + subscriptions = sorted(subscriptions, key=lambda x: x[0]) + for subscription in subscriptions: + title = subscription[0] + url = subscription[1] + options.addOption(title, url) + # options = form.add_field(var='action', + # ftype='list-single', + # label='Action', + # value='none') + # options.addOption('None', 'none') + # options.addOption('Reset', 'reset') + # options.addOption('Enable', 'enable') + # options.addOption('Disable', 'disable') + # options.addOption('Delete', 'delete') + session['payload'] = form + session['next'] = self._handle_subscription_editor + session['has_next'] = True + return session + + async def _handle_subscription_complete(self, payload, session): form = self['xep_0004'].make_form('form', 'Subscription editor') form['instructions'] = ('📁️ Subscription #{} has been {}' @@ -863,10 +1003,10 @@ class Slixfeed(slixmpp.ClientXMPP): 'Import data for {}'.format(jid)) form['instructions'] = '🗞️ Import feeds from OPML' url = form.add_field(var='url', - ftype='text-single', - label='URL', - desc='Enter URL to OPML file.', - required=True) + ftype='text-single', + label='URL', + desc='Enter URL to OPML file.', + required=True) url['validate']['datatype'] = 'xs:anyURI' session['payload'] = form session['next'] = self._handle_import_complete @@ -1194,7 +1334,7 @@ class Slixfeed(slixmpp.ClientXMPP): options = form.add_field(var='interval', ftype='list-single', label='Interval', - desc='Set interval update (in hours).', + desc='Interval update (in hours).', value=value) options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 48 } @@ -1207,6 +1347,21 @@ class Slixfeed(slixmpp.ClientXMPP): else: i += 1 + value = config.get_setting_value(db_file, 'quantum') + value = str(value) + options = form.add_field(var='quantum', + ftype='list-single', + label='Amount', + desc='Amount of items per update.', + value=value) + options['validate']['datatype'] = 'xs:integer' + options['validate']['range'] = { 'minimum': 1, 'maximum': 5 } + i = 1 + while i <= 5: + x = str(i) + options.addOption(x, x) + i += 1 + value = config.get_setting_value(db_file, 'archive') value = str(value) options = form.add_field(var='archive', @@ -1222,21 +1377,6 @@ class Slixfeed(slixmpp.ClientXMPP): options.addOption(x, x) i += 50 - value = config.get_setting_value(db_file, 'quantum') - value = str(value) - options = form.add_field(var='quantum', - ftype='list-single', - label='Amount', - desc='Set amount of items per update.', - value=value) - options['validate']['datatype'] = 'xs:integer' - options['validate']['range'] = { 'minimum': 1, 'maximum': 5 } - i = 1 - while i <= 5: - x = str(i) - options.addOption(x, x) - i += 1 - session['payload'] = form session['next'] = self._handle_settings_complete session['has_next'] = False diff --git a/slixfeed/xmpp/component.py b/slixfeed/xmpp/component.py index 3d4e688..ff1e389 100644 --- a/slixfeed/xmpp/component.py +++ b/slixfeed/xmpp/component.py @@ -8,6 +8,10 @@ TODO 1) Look into self.set_jid in order to be able to join to groupchats https://slixmpp.readthedocs.io/en/latest/api/basexmpp.html#slixmpp.basexmpp.BaseXMPP.set_jid +2) czar + https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0223.html + https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0222.html#module-slixmpp.plugins.xep_0222 + """ import asyncio diff --git a/slixfeed/xmpp/process.py b/slixfeed/xmpp/process.py index 0868884..b91fe6c 100644 --- a/slixfeed/xmpp/process.py +++ b/slixfeed/xmpp/process.py @@ -512,8 +512,7 @@ async def message(self, message): url = message_text # task.clean_tasks_xmpp(self, jid, ['status']) status_type = 'dnd' - status_message = ('📫️ Processing request ' - 'to fetch data from {}' + status_message = ('📫️ Processing request to fetch data from {}' .format(url)) XmppPresence.send(self, jid, status_message, status_type=status_type) @@ -522,7 +521,31 @@ async def message(self, message): url = (uri.replace_hostname(url, 'feed')) or url db_file = config.get_pathname_to_database(jid_file) # try: - response = await action.add_feed(db_file, url) + result = await action.add_feed(db_file, url) + if isinstance(result, list): + results = result + response = ("Web feeds found for {}\n\n```\n" + .format(url)) + for result in results: + response += ("Title : {}\n" + "Link : {}\n" + "\n" + .format(result['name'], result['url'])) + response += ('```\nTotal of {} feeds.' + .format(len(results))) + elif result['exist']: + response = ('> {}\nNews source "{}" is already ' + 'listed in the subscription list at ' + 'index {}'.format(result['url'], + result['name'], + result['index'])) + elif result['error']: + response = ('> {}\nFailed to load URL. Reason: {}' + .format(url, result['code'])) + else: + response = ('> {}\nNews source "{}" has been ' + 'added to subscription list.' + .format(result['url'], result['name'])) # task.clean_tasks_xmpp(self, jid, ['status']) await task.start_tasks_xmpp(self, jid, ['status']) # except: