Add for Add Subscription.

Segregating more code as one task per function, particularly adding of new subscription.
Fix allow/deny keys.

slixfeed/action.py

@@ -737,22 +737,26 @@ async def add_feed(db_file, url):
                         updated = ''
                     version = feed["version"]
                     entries = len(feed["entries"])
-                    await sqlite.insert_feed(
+                    await sqlite.insert_feed(db_file, url,
-                        db_file, url,
+                                             title=title,
-                        title=title,
+                                             entries=entries,
-                        entries=entries,
+                                             version=version,
-                        version=version,
+                                             encoding=encoding,
-                        encoding=encoding,
+                                             language=language,
-                        language=language,
+                                             status_code=status_code,
-                        status_code=status_code,
+                                             updated=updated)
-                        updated=updated
-                        )
                     await scan(db_file, url)
                     old = config.get_setting_value(db_file, "old")
                     if not old:
                         feed_id = await sqlite.get_feed_id(db_file, url)
                         feed_id = feed_id[0]
                         await sqlite.mark_feed_as_read(db_file, feed_id)
+                    result_final = {'url' : url,
+                                    'index' : feed_id,
+                                    'name' : title,
+                                    'code' : status_code,
+                                    'error' : False,
+                                    'exist' : False}
                     response = ('> {}\nNews source "{}" has been '
                                 'added to subscription list.'
                                 .format(url, title))
@@ -783,47 +787,64 @@ async def add_feed(db_file, url):
                         updated = ''
                     version = 'json' + feed["version"].split('/').pop()
                     entries = len(feed["items"])
-                    await sqlite.insert_feed(
+                    await sqlite.insert_feed(db_file, url,
-                        db_file, url,
+                                             title=title,
-                        title=title,
+                                             entries=entries,
-                        entries=entries,
+                                             version=version,
-                        version=version,
+                                             encoding=encoding,
-                        encoding=encoding,
+                                             language=language,
-                        language=language,
+                                             status_code=status_code,
-                        status_code=status_code,
+                                             updated=updated)
-                        updated=updated
+                    await scan_json(db_file, url)
-                        )
-                    await scan_json(
-                        db_file, url)
                     old = config.get_setting_value(db_file, "old")
                     if not old:
                         feed_id = await sqlite.get_feed_id(db_file, url)
                         feed_id = feed_id[0]
                         await sqlite.mark_feed_as_read(db_file, feed_id)
+                    result_final = {'url' : url,
+                                    'index' : feed_id,
+                                    'name' : title,
+                                    'code' : status_code,
+                                    'error' : False,
+                                    'exist' : False}
                     response = ('> {}\nNews source "{}" has been '
                                 'added to subscription list.'
                                 .format(url, title))
                     break
                 else:
-                    result = await crawl.probe_page(
+                    # NOTE Do not be tempted to return a compact dictionary.
-                        url, document)
+                    #      That is, dictionary within dictionary
-                    if isinstance(result, str):
+                    #      Return multimple dictionaries.
-                        response = result
+                    result = await crawl.probe_page(url, document)
+                    if isinstance(result, list):
+                        result_final = result
                         break
                     else:
-                        url = result[0]
+                        url = result['url']
             else:
+                result_final = {'url' : url,
+                                'index' : None,
+                                'name' : None,
+                                'code' : status_code,
+                                'error' : True,
+                                'exist' : False}
                 response = ('> {}\nFailed to load URL.  Reason: {}'
                             .format(url, status_code))
                 break
         else:
             ix = exist[0]
             name = exist[1]
+            result_final = {'url' : url,
+                            'index' : ix,
+                            'name' : name,
+                            'code' : None,
+                            'error' : False,
+                            'exist' : True}
             response = ('> {}\nNews source "{}" is already '
                         'listed in the subscription list at '
                         'index {}'.format(url, name, ix))
             break
-    return response
+    return result_final
 
 
 async def scan_json(db_file, url):

slixfeed/assets/information.toml

@@ -154,9 +154,9 @@ thanks = [
     "Strix from Loqi",
     "Thibaud Guerin (SalixOS)",
     "Thorsten Fröhlich (France)",
-	 "Thorsten Mühlfelder (SalixOS, Germany)",
+    "Thorsten Mühlfelder (SalixOS, Germany)",
     "Tim Beech (SalixOS, Brazil)",
-	 "Yann Leboulanger (Gajim, France)"
+    "Yann Leboulanger (Gajim, France)"
 ]
 
 xmpp = """

slixfeed/assets/lists.toml

@@ -147,7 +147,7 @@ pathnames = [
 # would get the best out of this news application.
 
 # Entries with the following keywords will not be filtered
-filter-allow = [
+allow = [
 "akkoma",
 "censorship",
 "earthing",
@@ -187,7 +187,7 @@ filter-allow = [
 ]
 
 # Entries with the following keywords will be filtered
-filter-deny = [
+deny = [
 # brands
 # Almost every time you see a brand name in title or content, it is because
 # someone, usually a marketing agency or a venture capital firm, has paid for
@@ -264,6 +264,7 @@ filter-deny = [
 "homosex",
 "lesbian",
 "lgbt",
+"nonbinary",
 "nude",
 "nudity",
 "onlyfans",

slixfeed/crawl.py

@@ -300,7 +300,8 @@ async def process_feed_selection(url, urls):
     feeds = {}
     for i in urls:
         res = await fetch.http(i)
-        if res[1] == 200:
+        status_code = res[1]
+        if status_code == 200:
             try:
                 feeds[i] = [parse(res[0])]
             except:
@@ -308,8 +309,7 @@ async def process_feed_selection(url, urls):
     message = (
         "Web feeds found for {}\n\n```\n"
         ).format(url)
-    counter = 0
+    urls = []
-    feed_url_mark = 0
     for feed_url in feeds:
         # try:
         #     res = await fetch.http(feed)
@@ -327,24 +327,28 @@ async def process_feed_selection(url, urls):
             continue
         if feed_amnt:
             # NOTE Because there could be many false positives
-            # which are revealed in second phase of scan, we
+            #      which are revealed in second phase of scan, we
-            # could end with a single feed, which would be
+            #      could end with a single feed, which would be
-            # listed instead of fetched, so feed_url_mark is
+            #      listed instead of fetched, so feed_url_mark is
-            # utilized in order to make fetch possible.
+            #      utilized in order to make fetch possible.
-            feed_url_mark = [feed_url]
+            # NOTE feed_url_mark was a variable which stored
-            counter += 1
+            #      single URL (probably first accepted as valid)
-            message += (
+            #      in order to get an indication whether a single
-                "Title : {}\n"
+            #      URL has been fetched, so that the receiving
-                "Link  : {}\n"
+            #      function will scan that single URL instead of
-                "\n"
+            #      listing it as a message.
-                ).format(feed_name, feed_url)
+            url = {'url' : feed_url,
-    if counter > 1:
+                   'index' : None,
-        message += (
+                   'name' : feed_name,
-            "```\nTotal of {} feeds."
+                   'code' : status_code,
-            ).format(counter)
+                   'error' : False,
-        result = message
+                   'exist' : None}
-    elif feed_url_mark:
+            urls.extend([url])
-        result = feed_url_mark
+    count = len(urls)
+    if count > 1:
+        result = urls
+    elif count:
+        result = urls[0]
     else:
         result = None
     return result

slixfeed/xmpp/client.py

@@ -469,8 +469,11 @@ class Slixfeed(slixmpp.ClientXMPP):
         #     )
 
         # if jid == config.get_value('accounts', 'XMPP', 'operator'):
+        self['xep_0050'].add_command(node='subscription',
+                                     name='➕️ Add Subscription',
+                                     handler=self._handle_subscription_add)
         self['xep_0050'].add_command(node='subscriptions',
-                                     name='📰️  Subscriptions',
+                                     name='📰️ Browse Subscriptions',
                                      handler=self._handle_subscriptions)
         # self['xep_0050'].add_command(node='subscriptions_cat',
         #                              name='🔖️ Categories',
@@ -481,18 +484,20 @@ class Slixfeed(slixmpp.ClientXMPP):
         # self['xep_0050'].add_command(node='subscriptions_index',
         #                              name='📑️ Index (A - Z)',
         #                              handler=self._handle_subscription)
-        self['xep_0050'].add_command(node='settings',
+        # TODO Join Filters and Settings into  Preferences
-                                     name='📮️ Settings',
-                                     handler=self._handle_settings)
         self['xep_0050'].add_command(node='filters',
                                      name='🛡️ Filters',
                                      handler=self._handle_filters)
-        self['xep_0050'].add_command(node='bookmarks',
+        self['xep_0050'].add_command(node='settings',
-                                      name='📕 Bookmarks',
+                                     name='📮️ Settings',
-                                      handler=self._handle_bookmarks)
+                                     handler=self._handle_settings)
-        # self['xep_0050'].add_command(node='roster',
+        if not self.is_component: # This will be changed with XEP-0222 XEP-0223
-        #                               name='📓 Roster', # 📋
+            self['xep_0050'].add_command(node='bookmarks',
-        #                               handler=self._handle_roster)
+                                          name='📕 Bookmarks',
+                                          handler=self._handle_bookmarks)
+            self['xep_0050'].add_command(node='roster',
+                                          name='📓 Roster', # 📋
+                                          handler=self._handle_roster)
         self['xep_0050'].add_command(node='help',
                                      name='📔️ Manual',
                                      handler=self._handle_help)
@@ -604,6 +609,102 @@ class Slixfeed(slixmpp.ClientXMPP):
         return session
 
 
+    async def _handle_subscription_add(self, iq, session):
+        jid = session['from'].bare
+        form = self['xep_0004'].make_form('form', 'Add Subscriptions')
+        form['instructions'] = '📰️ Add a new subscription'
+        options = form.add_field(var='subscription',
+                                 # TODO Make it possible to add several subscriptions at once;
+                                 #      Similarly to BitTorrent trackers list
+                                 # ftype='text-multi',
+                                 # label='Subscription URLs',
+                                 # desc=('Add subscriptions one time per '
+                                 #       'subscription.'),
+                                 ftype='text-single',
+                                 label='Subscription URL',
+                                 desc='Enter subscription URL.',
+                                 required=True)
+        form.add_field(var='scan',
+                       ftype='boolean',
+                       label='Scan',
+                       desc='Scan URL for validity.',
+                       value=True)
+        session['payload'] = form
+        session['next'] = self._handle_subscription_new
+        session['has_next'] = True
+        return session
+
+
+    async def _handle_subscription_new(self, payload, session):
+        jid = session['from'].bare
+        jid_file = jid
+        db_file = config.get_pathname_to_database(jid_file)
+        url = payload['values']['subscription']
+        result = await action.add_feed(db_file, url)
+        if isinstance(result, list):
+            results = result
+            form = self['xep_0004'].make_form('form', 'Subscriptions')
+            form['instructions'] = ('🔍️ Discovered {} subscriptions for {}'
+                                    .format(len(results), url))
+            options = form.add_field(var='subscriptions',
+                                     ftype='list-single',
+                                     label='Subscriptions',
+                                     desc=('Select a subscription to add.'),
+                                     required=True)
+            for result in results:
+                options.addOption(result['name'], result['url'])
+            session['payload'] = form
+            session['next'] = self._handle_subscription_editor
+            session['has_next'] = True
+        elif result['exist']:
+            # response = ('News source "{}" is already listed '
+            #             'in the subscription list at index '
+            #             '{}.\n{}'.format(result['name'], result['index'],
+            #                              result['url']))
+            # session['notes'] = [['warning', response]] # Not supported by Gajim
+            # session['notes'] = [['info', response]]
+            form = self['xep_0004'].make_form('result', 'Subscriptions')
+            form['instructions'] = ('⚠️ Feed "{}" already exist as index {}'
+                                    .format(result['name'], result['index']))
+            options = form.add_field(var='subscriptions',
+                                     ftype='text-single',
+                                     label=result['url'],
+                                     desc='Choose next to edit subscription.',
+                                     value=result['url'])
+            # FIXME payload value does not pass, only [].
+            session['payload'] = form
+            session['next'] = self._handle_subscription_editor
+            session['has_next'] = True
+        elif result['error']:
+            response = ('Failed to load URL.'
+                        '\n\n'
+                        'Reason: {}'
+                        '\n\n'
+                        'URL: {}'
+                        .format(result['code'], url))
+            session['notes'] = [['error', response]]
+            session['next'] = None
+        else:
+            # response = ('News source "{}" has been '
+            #             'added to subscription list.\n{}'
+            #             .format(result['name'], result['url']))
+            # session['notes'] = [['info', response]]
+            form = self['xep_0004'].make_form('result', 'Subscriptions')
+            form['instructions'] = ('✅️ News source "{}" has been added to '
+                                    'subscription list as index {}'
+                                    .format(result['name'], result['index']))
+            options = form.add_field(var='subscriptions',
+                                     ftype='text-single',
+                                     label=result['url'],
+                                     desc='Choose next to edit subscription.',
+                                     value=result['url'])
+            # FIXME payload value does not pass, only [].
+            session['payload'] = form
+            session['next'] = self._handle_subscription_editor
+            session['has_next'] = True
+        return session
+
+
     async def _handle_subscriptions(self, iq, session):
         jid = session['from'].bare
         form = self['xep_0004'].make_form('form',
@@ -702,12 +803,13 @@ class Slixfeed(slixmpp.ClientXMPP):
 
 
     async def _handle_subscription_editor(self, payload, session):
-        urls = payload['values']['subscriptions']
         jid = session['from'].bare
         jid_file = jid
         db_file = config.get_pathname_to_database(jid_file)
+        if 'subscriptions' in payload['values']:
+            urls = payload['values']['subscriptions']
         url_count = len(urls)
-        if url_count > 1:
+        if isinstance(urls, list) and url_count > 1:
             form = self['xep_0004'].make_form('form', 'Subscription editor')
             form['instructions'] = '📂️ Editing {} subscriptions'.format(url_count)
             form.add_field(var='options',
@@ -743,10 +845,9 @@ class Slixfeed(slixmpp.ClientXMPP):
                            label='Name',
                            value=title)
             # NOTE This does not look good in Gajim
-            #url = form.add_field(ftype='fixed',
+            # url = form.add_field(ftype='fixed',
-            #                         value=url)
+            #                      value=url)
             #url['validate']['datatype'] = 'xs:anyURI'
-
             form.add_field(var='url',
                            ftype='text-single',
                            label='URL',
@@ -810,6 +911,45 @@ class Slixfeed(slixmpp.ClientXMPP):
         return session
 
 
+    async def _handle_subscription_selector(self, payload, session):
+        jid = session['from'].bare
+        form = self['xep_0004'].make_form('form',
+                                          'Discovered ubscriptions for {}'.format(jid))
+        form['instructions'] = ('📰️ Select a subscriptions to add\n'
+                                'Subsciptions discovered for {}'
+                                .format(url))
+        # form.addField(var='interval',
+        #               ftype='text-single',
+        #               label='Interval period')
+        options = form.add_field(var='subscriptions',
+                                 ftype='list-multi',
+                                 label='Subscriptions',
+                                 desc=('Select subscriptions to perform '
+                                       'actions upon.'),
+                                 required=True)
+        jid_file = jid
+        db_file = config.get_pathname_to_database(jid_file)
+        subscriptions = await sqlite.get_feeds(db_file)
+        subscriptions = sorted(subscriptions, key=lambda x: x[0])
+        for subscription in subscriptions:
+            title = subscription[0]
+            url = subscription[1]
+            options.addOption(title, url)
+        # options = form.add_field(var='action',
+        #                          ftype='list-single',
+        #                          label='Action',
+        #                          value='none')
+        # options.addOption('None', 'none')
+        # options.addOption('Reset', 'reset')
+        # options.addOption('Enable', 'enable')
+        # options.addOption('Disable', 'disable')
+        # options.addOption('Delete', 'delete')
+        session['payload'] = form
+        session['next'] = self._handle_subscription_editor
+        session['has_next'] = True
+        return session
+
+
     async def _handle_subscription_complete(self, payload, session):
         form = self['xep_0004'].make_form('form', 'Subscription editor')
         form['instructions'] = ('📁️ Subscription #{} has been {}'
@@ -863,10 +1003,10 @@ class Slixfeed(slixmpp.ClientXMPP):
                                           'Import data for {}'.format(jid))
         form['instructions'] = '🗞️ Import feeds from OPML'
         url = form.add_field(var='url',
-                       ftype='text-single',
+                             ftype='text-single',
-                       label='URL',
+                             label='URL',
-                       desc='Enter URL to OPML file.',
+                             desc='Enter URL to OPML file.',
-                       required=True)
+                             required=True)
         url['validate']['datatype'] = 'xs:anyURI'
         session['payload'] = form
         session['next'] = self._handle_import_complete
@@ -1194,7 +1334,7 @@ class Slixfeed(slixmpp.ClientXMPP):
         options = form.add_field(var='interval',
                                  ftype='list-single',
                                  label='Interval',
-                                 desc='Set interval update (in hours).',
+                                 desc='Interval update (in hours).',
                                  value=value)
         options['validate']['datatype'] = 'xs:integer'
         options['validate']['range'] = { 'minimum': 1, 'maximum': 48 }
@@ -1207,6 +1347,21 @@ class Slixfeed(slixmpp.ClientXMPP):
             else:
                 i += 1
 
+        value = config.get_setting_value(db_file, 'quantum')
+        value = str(value)
+        options = form.add_field(var='quantum',
+                                 ftype='list-single',
+                                 label='Amount',
+                                 desc='Amount of items per update.',
+                                 value=value)
+        options['validate']['datatype'] = 'xs:integer'
+        options['validate']['range'] = { 'minimum': 1, 'maximum': 5 }
+        i = 1
+        while i <= 5:
+            x = str(i)
+            options.addOption(x, x)
+            i += 1
+
         value = config.get_setting_value(db_file, 'archive')
         value = str(value)
         options = form.add_field(var='archive',
@@ -1222,21 +1377,6 @@ class Slixfeed(slixmpp.ClientXMPP):
             options.addOption(x, x)
             i += 50
 
-        value = config.get_setting_value(db_file, 'quantum')
-        value = str(value)
-        options = form.add_field(var='quantum',
-                                 ftype='list-single',
-                                 label='Amount',
-                                 desc='Set amount of items per update.',
-                                 value=value)
-        options['validate']['datatype'] = 'xs:integer'
-        options['validate']['range'] = { 'minimum': 1, 'maximum': 5 }
-        i = 1
-        while i <= 5:
-            x = str(i)
-            options.addOption(x, x)
-            i += 1
-
         session['payload'] = form
         session['next'] = self._handle_settings_complete
         session['has_next'] = False

slixfeed/xmpp/component.py

@@ -8,6 +8,10 @@ TODO
 1) Look into self.set_jid in order to be able to join to groupchats
    https://slixmpp.readthedocs.io/en/latest/api/basexmpp.html#slixmpp.basexmpp.BaseXMPP.set_jid
 
+2) czar
+   https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0223.html
+   https://slixmpp.readthedocs.io/en/latest/api/plugins/xep_0222.html#module-slixmpp.plugins.xep_0222
+
 """
 
 import asyncio

slixfeed/xmpp/process.py

@@ -512,8 +512,7 @@ async def message(self, message):
                 url = message_text
                 # task.clean_tasks_xmpp(self, jid, ['status'])
                 status_type = 'dnd'
-                status_message = ('📫️ Processing request '
+                status_message = ('📫️ Processing request to fetch data from {}'
-                                  'to fetch data from {}'
                                   .format(url))
                 XmppPresence.send(self, jid, status_message,
                                   status_type=status_type)
@@ -522,7 +521,31 @@ async def message(self, message):
                 url = (uri.replace_hostname(url, 'feed')) or url
                 db_file = config.get_pathname_to_database(jid_file)
                 # try:
-                response = await action.add_feed(db_file, url)
+                result = await action.add_feed(db_file, url)
+                if isinstance(result, list):
+                    results = result
+                    response = ("Web feeds found for {}\n\n```\n"
+                                .format(url))
+                    for result in results:
+                        response += ("Title : {}\n"
+                                     "Link  : {}\n"
+                                     "\n"
+                                     .format(result['name'], result['url']))
+                    response += ('```\nTotal of {} feeds.'
+                                .format(len(results)))
+                elif result['exist']:
+                    response = ('> {}\nNews source "{}" is already '
+                                'listed in the subscription list at '
+                                'index {}'.format(result['url'],
+                                                  result['name'],
+                                                  result['index']))
+                elif result['error']:
+                    response = ('> {}\nFailed to load URL.  Reason: {}'
+                                .format(url, result['code']))
+                else:
+                    response = ('> {}\nNews source "{}" has been '
+                                'added to subscription list.'
+                                .format(result['url'], result['name']))
                 # task.clean_tasks_xmpp(self, jid, ['status'])
                 await task.start_tasks_xmpp(self, jid, ['status'])
                 # except: