From 60756dbdd2cc8b415b5de6a0e9718f206a0737b7 Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Fri, 5 Apr 2024 15:25:04 +0000 Subject: [PATCH] Ad-Hoc: Add more operator options; Ad-Hoc: Add menus (list-single) for selection from a fixed list of bookmarks ans contacts; Database: Identifier (node name) includes hyphens instead of dots. Database: SQLite database now stores more items. Bookmarks: Improve code; MUC: Improve code; SQLite: Manjor code changes to adapt to new table; URL: Fix redirection (hostname switcher). --- slixfeed/action.py | 567 ++++++++++++------- slixfeed/sqlite.py | 1063 ++++++++++++++++++++++-------------- slixfeed/task.py | 60 +- slixfeed/url.py | 24 +- slixfeed/version.py | 4 +- slixfeed/xmpp/bookmark.py | 2 +- slixfeed/xmpp/client.py | 562 ++++++++++--------- slixfeed/xmpp/component.py | 549 ++++++++++--------- slixfeed/xmpp/muc.py | 14 - slixfeed/xmpp/process.py | 102 ++-- slixfeed/xmpp/publish.py | 16 +- slixfeed/xmpp/utility.py | 11 + 12 files changed, 1786 insertions(+), 1188 deletions(-) diff --git a/slixfeed/action.py b/slixfeed/action.py index e972c2c..adabd04 100644 --- a/slixfeed/action.py +++ b/slixfeed/action.py @@ -183,7 +183,7 @@ async def xmpp_send_status_message(self, jid): status_mode = 'dnd' status_text = jid_task[list(jid_task.keys())[0]] else: - feeds = sqlite.get_number_of_items(db_file, 'feeds') + feeds = sqlite.get_number_of_items(db_file, 'feeds_properties') # print(await current_time(), jid, "has", feeds, "feeds") if not feeds: status_mode = 'available' @@ -227,21 +227,22 @@ async def xmpp_send_pubsub(self, jid_bare, num=None): subscriptions = sqlite.get_active_feeds_url(db_file) for url in subscriptions: url = url[0] - feed_id = sqlite.get_feed_id(db_file, url) - feed_id = feed_id[0] - feed_title = None - feed_summary = None if jid_bare == self.boundjid.bare: node = 'urn:xmpp:microblog:0' + feed_title = None + feed_subtitle = None else: + feed_id = sqlite.get_feed_id(db_file, url) + feed_id = feed_id[0] feed_title = sqlite.get_feed_title(db_file, feed_id) feed_title = feed_title[0] - feed_summary = None - node = sqlite.get_node_name(db_file, feed_id) + feed_subtitle = sqlite.get_feed_subtitle(db_file, feed_id) + feed_subtitle = feed_subtitle[0] + node = sqlite.get_feed_identifier(db_file, feed_id) node = node[0] xep = None iq_create_node = XmppPubsub.create_node( - self, jid_bare, node, xep, feed_title, feed_summary) + self, jid_bare, node, xep, feed_title, feed_subtitle) await XmppIQ.send(self, iq_create_node) entries = sqlite.get_unread_entries_of_feed(db_file, feed_id) feed_properties = sqlite.get_feed_properties(db_file, feed_id) @@ -251,17 +252,21 @@ async def xmpp_send_pubsub(self, jid_bare, num=None): # if num and counter < num: report[url] = len(entries) for entry in entries: - feed_entry = {'author' : None, - 'authors' : None, - 'category' : None, - 'content' : None, - 'description' : entry[3], - 'link' : entry[2], - 'links' : entry[4], - 'tags' : None, - 'title' : entry[1], - 'type' : None, - 'updated' : entry[7]} + feed_entry = {'authors' : entry[3], + 'content' : entry[6], + 'content_type' : entry[7], + 'contact' : entry[4], + 'contributors' : entry[5], + 'summary' : entry[8], + 'summary_type' : entry[9], + 'enclosures' : entry[13], + 'language' : entry[10], + 'link' : entry[2], + 'links' : entry[11], + 'published' : entry[15], + 'tags' : entry[12], + 'title' : entry[1], + 'updated' : entry[16]} iq_create_entry = XmppPubsub.create_entry( self, jid_bare, node, feed_entry, feed_version) await XmppIQ.send(self, iq_create_entry) @@ -303,12 +308,11 @@ async def xmpp_send_message(self, jid, num=None): title_e = result[1] url = result[2] summary = result[3] - enclosure = result[4] - feed_id = result[5] - date = result[6] + feed_id = result[4] + date = result[5] title_f = sqlite.get_feed_title(db_file, feed_id) title_f = title_f[0] - news_digest += list_unread_entries(self, result, title_f, jid) + news_digest += await list_unread_entries(self, result, title_f, jid) # print(db_file) # print(result[0]) # breakpoint() @@ -533,7 +537,7 @@ def is_feed(feed): return value -def list_unread_entries(self, result, feed_title, jid): +async def list_unread_entries(self, result, feed_title, jid): function_name = sys._getframe().f_code.co_name logger.debug('{}: feed_title: {} jid: {}' .format(function_name, feed_title, jid)) @@ -581,7 +585,7 @@ def list_unread_entries(self, result, feed_title, jid): # summary = "\n".join(summary) link = result[2] link = remove_tracking_parameters(link) - link = (replace_hostname(link, "link")) or link + link = await replace_hostname(link, "link") or link # news_item = ("\n{}\n{}\n{} [{}]\n").format(str(title), str(link), # str(feed_title), str(ix)) formatting = Config.get_setting_value(self.settings, jid, 'formatting') @@ -691,11 +695,9 @@ async def list_statistics(db_file): logger.debug('{}: db_file: {}' .format(function_name, db_file)) entries_unread = sqlite.get_number_of_entries_unread(db_file) - entries = sqlite.get_number_of_items(db_file, 'entries') - archive = sqlite.get_number_of_items(db_file, 'archive') - entries_all = entries + archive + entries = sqlite.get_number_of_items(db_file, 'entries_properties') feeds_active = sqlite.get_number_of_feeds_active(db_file) - feeds_all = sqlite.get_number_of_items(db_file, 'feeds') + feeds_all = sqlite.get_number_of_items(db_file, 'feeds_properties') # msg = """You have {} unread news items out of {} from {} news sources. # """.format(unread_entries, entries, feeds) @@ -714,7 +716,7 @@ async def list_statistics(db_file): "News items : {}/{}\n" "News sources : {}/{}\n" "```").format(entries_unread, - entries_all, + entries, feeds_active, feeds_all) return message @@ -762,19 +764,16 @@ def list_feeds(results): .format(len(results))) else: url = pick_a_feed() - message = ('List of subscriptions is empty.' + message = ('List of subscriptions is empty. To add a feed, send a URL.' '\n' - 'To add a feed, send a URL.' - '\n' - 'Featured news:\n*{}*\n{}' + 'Featured news: *{}*\n{}' .format(url['name'], url['link'])) return message -async def list_bookmarks(self): +def list_bookmarks(self, conferences): function_name = sys._getframe().f_code.co_name logger.debug('{}'.format(function_name)) - conferences = await XmppBookmark.get(self) message = '\nList of groupchats:\n\n```\n' for conference in conferences: message += ('Name: {}\n' @@ -835,36 +834,42 @@ async def import_opml(db_file, result): if not result['error']: document = result['content'] root = ET.fromstring(document) - before = sqlite.get_number_of_items(db_file, 'feeds') + before = sqlite.get_number_of_items(db_file, 'feeds_properties') feeds = [] for child in root.findall(".//outline"): url = child.get("xmlUrl") title = child.get("text") # feed = (url, title) # feeds.extend([feed]) - feeds.extend([(url, title)]) + feed = { + 'title' : title, + 'url' : url, + } + feeds.extend([feed]) await sqlite.import_feeds(db_file, feeds) await sqlite.add_metadata(db_file) - after = sqlite.get_number_of_items(db_file, 'feeds') + after = sqlite.get_number_of_items(db_file, 'feeds_properties') difference = int(after) - int(before) return difference -async def add_feed(self, jid_bare, db_file, url, node): +async def add_feed(self, jid_bare, db_file, url, identifier): function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} url: {}' .format(function_name, db_file, url)) while True: - exist_feed = sqlite.get_feed_id_and_name(db_file, url) - if not exist_feed: - exist_node = sqlite.check_node_exist(db_file, node) - if not exist_node: + feed_id = sqlite.get_feed_id(db_file, url) + if not feed_id: + exist_identifier = sqlite.check_identifier_exist(db_file, identifier) + if not exist_identifier: result = await fetch.http(url) message = result['message'] status_code = result['status_code'] if not result['error']: + await sqlite.update_feed_status(db_file, feed_id, status_code) document = result['content'] feed = parse(document) + # if document and status_code == 200: # if is_feed(url, feed): if is_feed(feed): if "title" in feed["feed"].keys(): @@ -887,21 +892,41 @@ async def add_feed(self, jid_bare, db_file, url, node): updated = '' else: updated = '' - version = feed["version"] - entries = len(feed["entries"]) - await sqlite.insert_feed(db_file, url, title, node, - entries=entries, + version = feed.version + entries_count = len(feed.entries) + await sqlite.insert_feed(db_file, + url, + title, + identifier, + entries=entries_count, version=version, encoding=encoding, language=language, status_code=status_code, updated=updated) - await scan(self, jid_bare, db_file, url) - old = Config.get_setting_value(self.settings, jid_bare, 'old') + feed_valid = 0 if feed.bozo else 1 + await sqlite.update_feed_validity(db_file, feed_id, feed_valid) + if feed.has_key('updated_parsed'): + feed_updated = feed.updated_parsed + try: + feed_updated = dt.convert_struct_time_to_iso8601(feed_updated) + except: + feed_updated = None + else: + feed_updated = None + entries_count = len(feed.entries) + await sqlite.update_feed_properties(db_file, feed_id, + entries_count, + feed_updated) feed_id = sqlite.get_feed_id(db_file, url) feed_id = feed_id[0] - if not old: - await sqlite.mark_feed_as_read(db_file, feed_id) + new_entries = get_properties_of_entries( + self, jid_bare, db_file, url, feed_id, feed) + if new_entries: + await sqlite.add_entries_and_update_feed_state( + db_file, feed_id, new_entries) + old = Config.get_setting_value(self.settings, jid_bare, 'old') + if not old: await sqlite.mark_feed_as_read(db_file, feed_id) result_final = {'link' : url, 'index' : feed_id, 'name' : title, @@ -909,7 +934,7 @@ async def add_feed(self, jid_bare, db_file, url, node): 'error' : False, 'message': message, 'exist' : False, - 'node' : None} + 'identifier' : None} break # NOTE This elif statement be unnecessary # when feedparser be supporting json feed. @@ -936,9 +961,12 @@ async def add_feed(self, jid_bare, db_file, url, node): else: updated = '' version = 'json' + feed["version"].split('/').pop() - entries = len(feed["items"]) - await sqlite.insert_feed(db_file, url, title, node, - entries=entries, + entries_count = len(feed["items"]) + await sqlite.insert_feed(db_file, + url, + title, + identifier, + entries=entries_count, version=version, encoding=encoding, language=language, @@ -957,7 +985,7 @@ async def add_feed(self, jid_bare, db_file, url, node): 'error' : False, 'message': message, 'exist' : False, - 'node' : None} + 'identifier' : None} break else: # NOTE Do not be tempted to return a compact dictionary. @@ -973,7 +1001,7 @@ async def add_feed(self, jid_bare, db_file, url, node): 'error' : True, 'message': message, 'exist' : False, - 'node' : None} + 'identifier' : None} break elif isinstance(result, list): # Get out of the loop and deliver a list of dicts. @@ -983,6 +1011,7 @@ async def add_feed(self, jid_bare, db_file, url, node): # Go back up to the while loop and try again. url = result['link'] else: + await sqlite.update_feed_status(db_file, feed_id, status_code) result_final = {'link' : url, 'index' : None, 'name' : None, @@ -990,12 +1019,13 @@ async def add_feed(self, jid_bare, db_file, url, node): 'error' : True, 'message': message, 'exist' : False, - 'node' : None} + 'identifier' : None} break else: - ix = exist_node[1] - node = exist_node[2] - message = 'Node is already allocated.' + ix = exist_identifier[1] + identifier = exist_identifier[2] + message = ('Identifier "{}" is already allocated.' + .format(identifier)) result_final = {'link' : url, 'index' : ix, 'name' : None, @@ -1003,20 +1033,21 @@ async def add_feed(self, jid_bare, db_file, url, node): 'error' : False, 'message': message, 'exist' : False, - 'node' : node} + 'identifier' : identifier} break else: - ix = exist_feed[0] - name = exist_feed[1] + feed_id = feed_id[0] + title = sqlite.get_feed_title(db_file, feed_id) + title = title[0] message = 'URL already exist.' result_final = {'link' : url, - 'index' : ix, - 'name' : name, + 'index' : feed_id, + 'name' : title, 'code' : None, 'error' : False, 'message': message, 'exist' : True, - 'node' : None} + 'identifier' : None} break return result_final @@ -1168,8 +1199,8 @@ async def scan_json(self, jid_bare, db_file, url): if len(new_entries): feed_id = sqlite.get_feed_id(db_file, url) feed_id = feed_id[0] - await sqlite.add_entries_and_update_timestamp(db_file, feed_id, - new_entries) + await sqlite.add_entries_and_update_feed_state(db_file, feed_id, + new_entries) def view_feed(url, feed): @@ -1266,162 +1297,274 @@ def view_entry(url, feed, num): return response -# TODO get all active feeds of active accounts and scan the feed with the earliest scanned time -# TODO Rename function name (idea: scan_and_populate) -async def scan(self, jid_bare, db_file, url): +async def download_feed(self, db_file, feed_url): """ - Check feeds for new entries. + Get feed content. Parameters ---------- db_file : str Path to database file. url : str, optional - URL. The default is None. + URL. """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} url: {}' - .format(function_name, db_file, url)) - if isinstance(url, tuple): url = url[0] - result = await fetch.http(url) - feed_id = sqlite.get_feed_id(db_file, url) + .format(function_name, db_file, feed_url)) + if isinstance(feed_url, tuple): feed_url = feed_url[0] + result = await fetch.http(feed_url) + feed_id = sqlite.get_feed_id(db_file, feed_url) feed_id = feed_id[0] status_code = result['status_code'] await sqlite.update_feed_status(db_file, feed_id, status_code) - if not result['error']: - document = result['content'] - status = result['status_code'] - new_entries = [] - if document and status == 200: - feed = parse(document) - entries = feed.entries - # length = len(entries) - await remove_nonexistent_entries(self, jid_bare, db_file, url, feed) - try: - if feed.bozo: - # bozo = ( - # "WARNING: Bozo detected for feed: {}\n" - # "For more information, visit " - # "https://pythonhosted.org/feedparser/bozo.html" - # ).format(url) - # print(bozo) - valid = 0 - else: - valid = 1 - feed_id = sqlite.get_feed_id(db_file, url) - feed_id = feed_id[0] - await sqlite.update_feed_validity( - db_file, feed_id, valid) - if "updated_parsed" in feed["feed"].keys(): - updated = feed["feed"]["updated_parsed"] - try: - updated = dt.convert_struct_time_to_iso8601(updated) - except: - updated = '' - else: - updated = '' - feed_id = sqlite.get_feed_id(db_file, url) - feed_id = feed_id[0] - await sqlite.update_feed_properties(db_file, feed_id, - len(feed["entries"]), updated) - # await update_feed_status - except (IncompleteReadError, IncompleteRead, error.URLError) as e: - logger.error(e) - return - # new_entry = 0 - for entry in entries: - logger.debug('{}: entry: {}'.format(function_name, entry.link)) - if entry.has_key("published"): - date = entry.published - date = dt.rfc2822_to_iso8601(date) - elif entry.has_key("updated"): - date = entry.updated - date = dt.rfc2822_to_iso8601(date) - else: - date = dt.now() - if entry.has_key("link"): - # link = complete_url(source, entry.link) - link = join_url(url, entry.link) - link = trim_url(link) - else: - link = url - # title = feed["feed"]["title"] - # title = "{}: *{}*".format(feed["feed"]["title"], entry.title) - title = entry.title if entry.has_key("title") else date - entry_id = entry.id if entry.has_key("id") else link - feed_id = sqlite.get_feed_id(db_file, url) - feed_id = feed_id[0] - exist = sqlite.check_entry_exist(db_file, feed_id, - entry_id=entry_id, - title=title, link=link, - date=date) - if not exist: - summary = entry.summary if entry.has_key("summary") else '' - read_status = 0 - pathname = urlsplit(link).path - string = ( - "{} {} {}" - ).format( - title, summary, pathname) - if self.settings['default']['filter']: - print('Filter is now processing data.') - allow_list = config.is_include_keyword(db_file, - "allow", string) - if not allow_list: - reject_list = config.is_include_keyword(db_file, - "deny", - string) - if reject_list: - read_status = 1 - logger.debug('Rejected : {}' - '\n' - 'Keyword : {}' - .format(link, reject_list)) - if isinstance(date, int): - logger.error('Variable "date" is int: {}'.format(date)) - media_link = '' - if entry.has_key("links"): - for e_link in entry.links: - try: - # if (link.rel == "enclosure" and - # (link.type.startswith("audio/") or - # link.type.startswith("image/") or - # link.type.startswith("video/")) - # ): - media_type = e_link.type[:e_link.type.index("/")] - if e_link.has_key("rel"): - if (e_link.rel == "enclosure" and - media_type in ("audio", "image", "video")): - media_link = e_link.href - media_link = join_url(url, e_link.href) - media_link = trim_url(media_link) - break - except: - logger.error('KeyError: "href"\n' - 'Missing "href" attribute for {}' - .format(url)) - logger.error('Continue scanning for next ' - 'potential enclosure of {}' - .format(link)) - entry = { - "title": title, - "link": link, - "summary": summary, - "enclosure": media_link, - "entry_id": entry_id, - "date": date, - "read_status": read_status + + +# TODO get all active feeds of active accounts and scan the feed with the earliest scanned time +# TODO Rename function name (idea: scan_and_populate) +def get_properties_of_entries(self, jid_bare, db_file, feed_url, feed_id, feed): + """ + Get new entries. + + Parameters + ---------- + db_file : str + Path to database file. + url : str, optional + URL. + """ + print('GET', feed_url, jid_bare) + function_name = sys._getframe().f_code.co_name + logger.debug('{}: feed_id: {} url: {}' + .format(function_name, feed_id, feed_url)) + + new_entries = [] + for entry in feed.entries: + logger.debug('{}: entry: {}'.format(function_name, entry.link)) + if entry.has_key("published"): + entry_published = entry.published + entry_published = dt.rfc2822_to_iso8601(entry_published) + else: + entry_published = '' + if entry.has_key("updated"): + entry_updated = entry.updated + entry_updated = dt.rfc2822_to_iso8601(entry_updated) + else: + entry_updated = dt.now() + if entry.has_key("link"): + # link = complete_url(source, entry.link) + entry_link = join_url(feed_url, entry.link) + entry_link = trim_url(entry_link) + else: + entry_link = feed_url + # title = feed["feed"]["title"] + # title = "{}: *{}*".format(feed["feed"]["title"], entry.title) + entry_title = entry.title if entry.has_key("title") else entry_published + entry_id = entry.id if entry.has_key("id") else entry_link + exist = sqlite.check_entry_exist(db_file, feed_id, + identifier=entry_id, + title=entry_title, + link=entry_link, + published=entry_published) + if not exist: + read_status = 0 + # # Filter + # pathname = urlsplit(link).path + # string = ( + # "{} {} {}" + # ).format( + # title, summary, pathname) + # if self.settings['default']['filter']: + # print('Filter is now processing data.') + # allow_list = config.is_include_keyword(db_file, + # "allow", string) + # if not allow_list: + # reject_list = config.is_include_keyword(db_file, + # "deny", + # string) + # if reject_list: + # read_status = 1 + # logger.debug('Rejected : {}' + # '\n' + # 'Keyword : {}' + # .format(link, reject_list)) + if isinstance(entry_published, int): + logger.error('Variable "published" is int: {}'.format(entry_published)) + if isinstance(entry_updated, int): + logger.error('Variable "updated" is int: {}'.format(entry_updated)) + + # Authors + entry_authors =[] + if entry.has_key('authors'): + for author in entry.authors: + author_properties = { + 'name' : author.name if author.has_key('name') else '', + 'url' : author.href if author.has_key('href') else '', + 'email' : author.email if author.has_key('email') else '', } - new_entries.extend([entry]) - # await sqlite.add_entry( - # db_file, title, link, entry_id, - # url, date, read_status) - # await sqlite.set_date(db_file, url) - if len(new_entries): - feed_id = sqlite.get_feed_id(db_file, url) - feed_id = feed_id[0] - await sqlite.add_entries_and_update_timestamp(db_file, feed_id, - new_entries) + entry_authors.extend([author_properties]) + elif entry.has_key('author_detail'): + author_properties = { + 'name' : entry.author_detail.name if entry.author_detail.has_key('name') else '', + 'url' : entry.author_detail.href if entry.author_detail.has_key('href') else '', + 'email' : entry.author_detail.email if entry.author_detail.has_key('email') else '', + } + entry_authors.extend([author_properties]) + elif entry.has_key('author'): + author_properties = { + 'name' : entry.author, + 'url' : '', + 'email' : '', + } + entry_authors.extend([author_properties]) + + # Contributors + entry_contributors = [] + if entry.has_key('contributors'): + for contributor in entry.contributors: + contributor_properties = { + 'name' : contributor.name if contributor.has_key('name') else '', + 'url' : contributor.href if contributor.has_key('href') else '', + 'email' : contributor.email if contributor.has_key('email') else '', + } + entry_contributors.extend([contributor_properties]) + + # Tags + entry_tags = [] + if entry.has_key('tags'): + for tag in entry.tags: + tag_properties = { + 'term' : tag.term if tag.has_key('term') else '', + 'scheme' : tag.scheme if tag.has_key('scheme') else '', + 'label' : tag.label if tag.has_key('label') else '', + } + entry_tags.extend([tag_properties]) + + # Content + entry_contents = [] + if entry.has_key('content'): + for content in entry.content: + text = content.value if content.has_key('value') else '' + type = content.type if content.has_key('type') else '' + lang = content.lang if content.has_key('lang') else '' + base = content.base if content.has_key('base') else '' + entry_content = { + 'text' : text, + 'lang' : lang, + 'type' : type, + 'base' : base, + } + entry_contents.extend([entry_content]) + + # Links and Enclosures + entry_links = [] + if entry.has_key('links'): + for link in entry.links: + link_properties = { + 'url' : link.href if link.has_key('href') else '', + 'rel' : link.rel if link.has_key('rel') else '', + 'type' : link.type if link.has_key('type') else '', + 'length' : '', + } + entry_links.extend([link_properties]) + # Element media:content is utilized by Mastodon + if entry.has_key('media_content'): + for link in entry.media_content: + link_properties = { + 'url' : link['url'] if 'url' in link else '', + 'rel' : 'enclosure', + 'type' : link['type'] if 'type' in link else '', + # 'medium' : link['medium'] if 'medium' in link else '', + 'length' : link['filesize'] if 'filesize' in link else '', + } + entry_links.extend([link_properties]) + if entry.has_key('media_thumbnail'): + for link in entry.media_thumbnail: + link_properties = { + 'url' : link['url'] if 'url' in link else '', + 'rel' : 'enclosure', + 'type' : '', + # 'medium' : 'image', + 'length' : '', + } + entry_links.extend([link_properties]) + + # Category + entry_category = entry.category if entry.has_key('category') else '' + + # Comments + entry_comments = entry.comments if entry.has_key('comments') else '' + + # href + entry_href = entry.href if entry.has_key('href') else '' + + # Link: Same as entry.links[0].href in most if not all cases + entry_link = entry.link if entry.has_key('link') else '' + + # Rating + entry_rating = entry.rating if entry.has_key('rating') else '' + + # Summary + entry_summary_text = entry.summary if entry.has_key('summary') else '' + if entry.has_key('summary_detail'): + entry_summary_type = entry.summary_detail.type if entry.summary_detail.has_key('type') else '' + entry_summary_lang = entry.summary_detail.lang if entry.summary_detail.has_key('lang') else '' + entry_summary_base = entry.summary_detail.base if entry.summary_detail.has_key('base') else '' + else: + entry_summary_type = '' + entry_summary_lang = '' + entry_summary_base = '' + + # Title + entry_title = entry.title if entry.has_key('title') else '' + if entry.has_key('title_detail'): + entry_title_type = entry.title_detail.type if entry.title_detail.has_key('type') else '' + else: + entry_title_type = '' + + ########################################################### + + # media_type = e_link.type[:e_link.type.index("/")] + # if (e_link.rel == "enclosure" and + # media_type in ("audio", "image", "video")): + # media_link = e_link.href + # media_link = join_url(url, e_link.href) + # media_link = trim_url(media_link) + + ########################################################### + + entry_properties = { + "identifier": entry_id, + "link": entry_link, + "href": entry_href, + "title": entry_title, + "title_type": entry_title_type, + 'summary_text' : entry_summary_text, + 'summary_lang' : entry_summary_lang, + 'summary_type' : entry_summary_type, + 'summary_base' : entry_summary_base, + 'category' : entry_category, + "comments": entry_comments, + "rating": entry_rating, + "published": entry_published, + "updated": entry_updated, + "read_status": read_status + } + print('entry_properties') + print(entry_properties) + + new_entries.extend([{ + "entry_properties" : entry_properties, + "entry_authors" : entry_authors, + "entry_contributors" : entry_contributors, + "entry_contents" : entry_contents, + "entry_links" : entry_links, + "entry_tags" : entry_tags + }]) + # await sqlite.add_entry( + # db_file, title, link, entry_id, + # url, date, read_status) + # await sqlite.set_date(db_file, url) + return new_entries def get_document_title(data): diff --git a/slixfeed/sqlite.py b/slixfeed/sqlite.py index e316a8a..54206e5 100644 --- a/slixfeed/sqlite.py +++ b/slixfeed/sqlite.py @@ -5,6 +5,8 @@ TODO +0) Function "mark_feed_as_read": see function "maintain_archive" + 1) Function to open connection (receive db_file). Function to close connection. All other functions to receive cursor. @@ -85,52 +87,120 @@ def create_tables(db_file): logger.debug('{}: db_file: {}' .format(function_name, db_file)) with create_connection(db_file) as conn: - archive_table_sql = ( + entries_properties_authors_table_sql = ( """ - CREATE TABLE IF NOT EXISTS archive ( - id INTEGER NOT NULL, - title TEXT NOT NULL, - link TEXT NOT NULL, - summary TEXT, - enclosure TEXT, - entry_id TEXT NOT NULL, - feed_id INTEGER NOT NULL, - timestamp TEXT, - read INTEGER NOT NULL DEFAULT 0, - reject INTEGER NOT NULL DEFAULT 0, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY ("id") - ); - """ - ) - entries_table_sql = ( - """ - CREATE TABLE IF NOT EXISTS entries ( - id INTEGER NOT NULL, - title TEXT NOT NULL, - link TEXT NOT NULL, - summary TEXT, - enclosure TEXT, - entry_id TEXT NOT NULL, - feed_id INTEGER NOT NULL, - timestamp TEXT, - read INTEGER NOT NULL DEFAULT 0, - reject INTEGER NOT NULL DEFAULT 0, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY ("id") - ); - """ - ) - feeds_table_sql = ( - """ - CREATE TABLE IF NOT EXISTS feeds ( + CREATE TABLE IF NOT EXISTS entries_properties_authors ( id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, name TEXT, - url TEXT NOT NULL UNIQUE, + url TEXT, + email TEXT, + FOREIGN KEY ("entry_id") REFERENCES "entries_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + entries_properties_contributors_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries_properties_contributors ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + name TEXT, + url TEXT, + email TEXT, + FOREIGN KEY ("entry_id") REFERENCES "entries_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + entries_properties_contents_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries_properties_contents ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + text TEXT, + type TEXT, + base TEXT, + lang TEXT, + FOREIGN KEY ("entry_id") REFERENCES "entries_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + entries_properties_links_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries_properties_links ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + url TEXT, + type TEXT, + rel TEXT, + size INTEGER, + FOREIGN KEY ("entry_id") REFERENCES "entries_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + entries_properties_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries_properties ( + id INTEGER NOT NULL, + feed_id INTEGER NOT NULL, + identifier TEXT, + link TEXT, + title TEXT, + title_type TEXT, + summary_text TEXT, + summary_lang TEXT, + summary_type TEXT, + summary_base TEXT, + category TEXT, + href TEXT, + comments TEXT, + rating TEXT, + published TEXT, + updated TEXT, + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + entries_state_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries_state ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + rejected INTEGER NOT NULL DEFAULT 0, + read INTEGER NOT NULL DEFAULT 0, + archived INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY ("entry_id") REFERENCES "entries_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + entries_properties_tags_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS entries_properties_tags ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + term TEXT, + scheme TEXT, + label TEXT, + FOREIGN KEY ("entry_id") REFERENCES "entries_properties" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, PRIMARY KEY ("id") ); """ @@ -138,16 +208,19 @@ def create_tables(db_file): # TODO Rethink! # Albeit, probably, more expensive, we might want to have feed_id # as foreign key, as it is with feeds_properties and feeds_state - feeds_categories_table_sql = ( + feeds_preferences_table_sql = ( """ - CREATE TABLE IF NOT EXISTS feeds_categories ( + CREATE TABLE IF NOT EXISTS feeds_preferences ( id INTEGER NOT NULL, - category_id INTEGER NOT NULL UNIQUE, - feed_id INTEGER, - FOREIGN KEY ("category_id") REFERENCES "categories" ("id") + feed_id INTEGER NOT NULL UNIQUE, + enabled INTEGER NOT NULL DEFAULT 1, + mutable INTEGER NOT NULL DEFAULT 0, + filter INTEGER NOT NULL DEFAULT 1, + priority INTEGER NOT NULL DEFAULT 0, + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") ON UPDATE CASCADE ON DELETE CASCADE, - PRIMARY KEY (id) + PRIMARY KEY ("id") ); """ ) @@ -155,28 +228,39 @@ def create_tables(db_file): """ CREATE TABLE IF NOT EXISTS feeds_properties ( id INTEGER NOT NULL, - feed_id INTEGER NOT NULL UNIQUE, - type TEXT, + url TEXT NOT NULL UNIQUE, + identifier TEXT, + title TEXT, + title_type TEXT, + subtitle TEXT, + subtitle_type TEXT, + version TEXT, encoding TEXT, language TEXT, + rating TEXT, entries INTEGER, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY (id) + icon TEXT, + image TEXT, + logo TEXT, + ttl TEXT, + updated TEXT, + PRIMARY KEY ("id") ); """ ) - feeds_pubsub_table_sql = ( + feeds_properties_links_table_sql = ( """ - CREATE TABLE IF NOT EXISTS feeds_pubsub ( + CREATE TABLE IF NOT EXISTS feeds_properties_links ( id INTEGER NOT NULL, - feed_id INTEGER NOT NULL UNIQUE, - node TEXT NOT NULL UNIQUE, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + feed_id INTEGER NOT NULL, + url TEXT, + type TEXT, + rel TEXT, + size INTEGER, + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") ON UPDATE CASCADE ON DELETE CASCADE, - PRIMARY KEY (id) + PRIMARY KEY ("id") ); """ ) @@ -186,15 +270,11 @@ def create_tables(db_file): id INTEGER NOT NULL, feed_id INTEGER NOT NULL UNIQUE, type TEXT NOT NULL, - base TEXT NOT NULL, - title TEXT NOT NULL, - link TEXT NOT NULL, - enclosure TEXT, - summary TEXT, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + keywords TEXT, + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") ON UPDATE CASCADE ON DELETE CASCADE, - PRIMARY KEY (id) + PRIMARY KEY ("id") ); """ ) @@ -203,15 +283,11 @@ def create_tables(db_file): CREATE TABLE IF NOT EXISTS feeds_state ( id INTEGER NOT NULL, feed_id INTEGER NOT NULL UNIQUE, - enabled INTEGER NOT NULL DEFAULT 1, - updated TEXT, - scanned TEXT, renewed TEXT, + scanned TEXT, status_code INTEGER, valid INTEGER, - filter INTEGER NOT NULL DEFAULT 1, - priority INTEGER, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY ("id") @@ -226,23 +302,22 @@ def create_tables(db_file): offline INTEGER, entries INTEGER, entries INTEGER, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY ("id") ); """ ) - feeds_tags_table_sql = ( + feeds_properties_tags_table_sql = ( """ - CREATE TABLE IF NOT EXISTS feeds_tags ( + CREATE TABLE IF NOT EXISTS feeds_properties_tags ( id INTEGER NOT NULL, feed_id INTEGER NOT NULL, - tag_id INTEGER NOT NULL, - FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY ("tag_id") REFERENCES "tags" ("id") + term TEXT, + scheme TEXT, + label TEXT, + FOREIGN KEY ("feed_id") REFERENCES "feeds_properties" ("id") ON UPDATE CASCADE ON DELETE CASCADE, PRIMARY KEY ("id") @@ -284,6 +359,22 @@ def create_tables(db_file): ); """ ) + tagged_feeds_table_sql = ( + """ + CREATE TABLE IF NOT EXISTS tagged_feeds ( + id INTEGER NOT NULL, + feed_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + FOREIGN KEY ("feed_id") REFERENCES "feeds" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY ("tag_id") REFERENCES "tags" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) tags_table_sql = ( """ CREATE TABLE IF NOT EXISTS tags ( @@ -295,18 +386,24 @@ def create_tables(db_file): ) cur = conn.cursor() # cur = get_cursor(db_file) - cur.execute(archive_table_sql) - cur.execute(entries_table_sql) - cur.execute(feeds_table_sql) - cur.execute(feeds_state_table_sql) + cur.execute(entries_properties_table_sql) + cur.execute(entries_properties_authors_table_sql) + cur.execute(entries_properties_contents_table_sql) + cur.execute(entries_properties_contributors_table_sql) + cur.execute(entries_properties_links_table_sql) + cur.execute(entries_properties_tags_table_sql) + cur.execute(entries_state_table_sql) cur.execute(feeds_properties_table_sql) - cur.execute(feeds_pubsub_table_sql) + cur.execute(feeds_properties_links_table_sql) + cur.execute(feeds_properties_tags_table_sql) + cur.execute(feeds_preferences_table_sql) cur.execute(feeds_rules_table_sql) - cur.execute(feeds_tags_table_sql) + cur.execute(feeds_state_table_sql) cur.execute(filters_table_sql) # cur.execute(statistics_table_sql) cur.execute(settings_table_sql) cur.execute(status_table_sql) + cur.execute(tagged_feeds_table_sql) cur.execute(tags_table_sql) @@ -356,20 +453,18 @@ async def import_feeds(db_file, feeds): for feed in feeds: logger.debug('{}: feed: {}' .format(function_name, feed)) - url = feed[0] - title = feed[1] + url = feed['url'] + title = feed['title'] sql = ( """ INSERT - INTO feeds( - name, url) + INTO feeds_properties( + title, url) VALUES( ?, ?) """ ) - par = ( - title, url - ) + par = (title, url) try: cur.execute(sql, par) except IntegrityError as e: @@ -395,16 +490,16 @@ async def add_metadata(db_file): sql = ( """ SELECT id - FROM feeds + FROM feeds_properties ORDER BY id ASC """ ) ixs = cur.execute(sql).fetchall() for ix in ixs: feed_id = ix[0] + # insert_feed_properties(cur, feed_id) insert_feed_status(cur, feed_id) - insert_feed_properties(cur, feed_id) - insert_feed_pubsub(cur, feed_id) + insert_feed_preferences(cur, feed_id) def insert_feed_status(cur, feed_id): @@ -437,6 +532,36 @@ def insert_feed_status(cur, feed_id): logger.error(e) +def insert_feed_preferences(cur, feed_id): + """ + Set feed preferences. + + Parameters + ---------- + cur : object + Cursor object. + """ + function_name = sys._getframe().f_code.co_name + logger.debug('{}: feed_id: {}' + .format(function_name, feed_id)) + sql = ( + """ + INSERT + INTO feeds_preferences( + feed_id) + VALUES( + ?) + """ + ) + par = (feed_id,) + try: + cur.execute(sql, par) + except IntegrityError as e: + logger.warning( + "Skipping feed_id {} for table feeds_preferences".format(feed_id)) + logger.error(e) + + def insert_feed_properties(cur, feed_id): """ Set feed properties. @@ -467,37 +592,7 @@ def insert_feed_properties(cur, feed_id): logger.error(e) -def insert_feed_pubsub(cur, feed_id): - """ - Set feed pubsub. - - Parameters - ---------- - cur : object - Cursor object. - """ - function_name = sys._getframe().f_code.co_name - logger.debug('{}: feed_id: {}' - .format(function_name, feed_id)) - sql = ( - """ - INSERT - INTO feeds_pubsub( - feed_id) - VALUES( - ?) - """ - ) - par = (feed_id,) - try: - cur.execute(sql, par) - except IntegrityError as e: - logger.warning( - "Skipping feed_id {} for table feeds_pubsub".format(feed_id)) - logger.error(e) - - -async def insert_feed(db_file, url, title, node, entries=None, version=None, +async def insert_feed(db_file, url, title, identifier, entries=None, version=None, encoding=None, language=None, status_code=None, updated=None): """ @@ -511,8 +606,8 @@ async def insert_feed(db_file, url, title, node, entries=None, version=None, URL. title : str Feed title. - node : str - Feed Node. + identifier : str + Feed identifier. entries : int, optional Number of entries. The default is None. version : str, optional @@ -535,20 +630,18 @@ async def insert_feed(db_file, url, title, node, entries=None, version=None, sql = ( """ INSERT - INTO feeds( - name, url) + INTO feeds_properties( + url, title, identifier, entries, version, encoding, language, updated) VALUES( - ?, ?) + ?, ?, ?, ?, ?, ?, ?, ?) """ ) - par = ( - title, url - ) + par = (url, title, identifier, entries, version, encoding, language, updated) cur.execute(sql, par) sql = ( """ SELECT id - FROM feeds + FROM feeds_properties WHERE url = :url """ ) @@ -558,46 +651,27 @@ async def insert_feed(db_file, url, title, node, entries=None, version=None, """ INSERT INTO feeds_state( - feed_id, enabled, updated, status_code, valid) + feed_id, status_code, valid) VALUES( - ?, ?, ?, ?, ?) + ?, ?, ?) """ ) - par = ( - feed_id, 1, updated, status_code, 1 - ) + par = (feed_id, status_code, 1) cur.execute(sql, par) sql = ( """ INSERT - INTO feeds_properties( - feed_id, entries, type, encoding, language) + INTO feeds_preferences( + feed_id) VALUES( - ?, ?, ?, ?, ?) + ?) """ ) - par = ( - feed_id, entries, version, encoding, language - ) - cur.execute(sql, par) - sql = ( - """ - INSERT - INTO feeds_pubsub( - feed_id, node) - VALUES( - ?, ?) - """ - ) - par = ( - feed_id, node - ) + par = (feed_id,) cur.execute(sql, par) -async def insert_feed_(db_file, url, title=None, entries=None, version=None, - encoding=None, language=None, status_code=None, - updated=None): +async def insert_feed_(db_file, url, title): """ Insert a new feed into the feeds table. @@ -609,22 +683,6 @@ async def insert_feed_(db_file, url, title=None, entries=None, version=None, URL. title : str, optional Feed title. The default is None. - entries : int, optional - Number of entries. The default is None. - version : str, optional - Type of feed. The default is None. - encoding : str, optional - Encoding of feed. The default is None. - language : str, optional - Language code of feed. The default is None. - status : str, optional - HTTP status code. The default is None. - updated : ???, optional - Date feed was last updated. The default is None. - status : str, optional - HTTP status code. The default is None. - updated : ???, optional - Date feed was last updated. The default is None. """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} url: {}' @@ -635,8 +693,8 @@ async def insert_feed_(db_file, url, title=None, entries=None, version=None, sql = ( """ INSERT - INTO feeds( - name, url) + INTO feeds_properties( + title, url) VALUES( ?, ?) """ @@ -648,17 +706,15 @@ async def insert_feed_(db_file, url, title=None, entries=None, version=None, sql = ( """ SELECT id - FROM feeds + FROM feeds_properties WHERE url = :url """ ) par = (url,) feed_id = cur.execute(sql, par).fetchone()[0] - insert_feed_properties( - cur, feed_id, entries=None, - version=None, encoding=None, language=None) - insert_feed_status( - cur, feed_id, status_code=None, updated=None) + # insert_feed_properties(cur, feed_id) + insert_feed_status(cur, feed_id) + insert_feed_preferences(cur, feed_id) async def remove_feed_by_url(db_file, url): @@ -681,7 +737,7 @@ async def remove_feed_by_url(db_file, url): sql = ( """ DELETE - FROM feeds + FROM feeds_properties WHERE url = ? """ ) @@ -724,7 +780,7 @@ async def remove_feed_by_index(db_file, ix): sql = ( """ DELETE - FROM feeds + FROM feeds_properties WHERE id = ? """ ) @@ -755,12 +811,12 @@ def get_feeds_by_tag_id(db_file, tag_id): cur = conn.cursor() sql = ( """ - SELECT feeds.* - FROM feeds - INNER JOIN feeds_tags ON feeds.id = feeds_tags.feed_id - INNER JOIN tags ON tags.id = feeds_tags.tag_id + SELECT feeds_properties.* + FROM feeds_properties + INNER JOIN tagged_feeds ON feeds_properties.id = tagged_feeds.feed_id + INNER JOIN tags ON tags.id = tagged_feeds.tag_id WHERE tags.id = ? - ORDER BY feeds.name; + ORDER BY feeds_properties.title; """ ) par = (tag_id,) @@ -793,9 +849,9 @@ def get_tags_by_feed_id(db_file, feed_id): """ SELECT tags.tag FROM tags - INNER JOIN feeds_tags ON tags.id = feeds_tags.tag_id - INNER JOIN feeds ON feeds.id = feeds_tags.feed_id - WHERE feeds.id = ? + INNER JOIN tagged_feeds ON tags.id = tagged_feeds.tag_id + INNER JOIN feeds_properties ON feeds_properties.id = tagged_feeds.feed_id + WHERE feeds_properties.id = ? ORDER BY tags.tag; """ ) @@ -826,7 +882,7 @@ async def set_feed_id_and_tag_id(db_file, feed_id, tag_id): sql = ( """ INSERT - INTO feeds_tags( + INTO tagged_feeds( feed_id, tag_id) VALUES( :feed_id, :tag_id) @@ -852,8 +908,8 @@ def get_feed_properties(db_file, feed_id): Returns ------- - node : str - Node name. + properties : list + List of properties. """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} feed_id: {}' @@ -868,13 +924,13 @@ def get_feed_properties(db_file, feed_id): """ ) par = (feed_id,) - name = cur.execute(sql, par).fetchone() - return name + properties = cur.execute(sql, par).fetchone() + return properties -def get_node_name(db_file, feed_id): +def get_feed_identifier(db_file, feed_id): """ - Get name of given node. + Get identifier of given feed ID. Parameters ---------- @@ -885,8 +941,8 @@ def get_node_name(db_file, feed_id): Returns ------- - node : str - Node name. + identifier : str + Identifier name. """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} feed_id: {}' @@ -895,26 +951,26 @@ def get_node_name(db_file, feed_id): cur = conn.cursor() sql = ( """ - SELECT node - FROM feeds_pubsub + SELECT identifier + FROM feeds_properties WHERE feed_id = ? """ ) par = (feed_id,) - name = cur.execute(sql, par).fetchone() - return name + identifier = cur.execute(sql, par).fetchone() + return identifier -def check_node_exist(db_file, node_name): +def check_identifier_exist(db_file, identifier): """ - Check whether node exist. + Check whether given identifier exist. Parameters ---------- db_file : str Path to database file. - node_name : str - Node name. + identifier : str + Identifier name. Returns ------- @@ -922,24 +978,24 @@ def check_node_exist(db_file, node_name): ID. feed_id : str Feed ID. - node : str - Node name. + identifier : str + Identifier name. """ function_name = sys._getframe().f_code.co_name - logger.debug('{}: db_file: {} node_name: {}' - .format(function_name, db_file, node_name)) + logger.debug('{}: db_file: {} identifier: {}' + .format(function_name, db_file, identifier)) with create_connection(db_file) as conn: cur = conn.cursor() sql = ( """ - SELECT id, feed_id, node - FROM feeds_pubsub - WHERE node = ? + SELECT identifier + FROM feeds_properties + WHERE identifier = ? """ ) - par = (node_name,) - name = cur.execute(sql, par).fetchone() - return name + par = (identifier,) + identifier = cur.execute(sql, par).fetchone() + return identifier def get_tag_id(db_file, tag_name): @@ -1032,7 +1088,7 @@ def is_tag_id_associated(db_file, tag_id): sql = ( """ SELECT tag_id - FROM feeds_tags + FROM tagged_feeds WHERE tag_id = :tag_id """ ) @@ -1089,7 +1145,7 @@ def is_tag_id_of_feed_id(db_file, tag_id, feed_id): sql = ( """ SELECT tag_id - FROM feeds_tags + FROM tagged_feeds WHERE tag_id = :tag_id AND feed_id = :feed_id """ ) @@ -1111,7 +1167,7 @@ async def delete_feed_id_tag_id(db_file, feed_id, tag_id): sql = ( """ DELETE - FROM feeds_tags + FROM tagged_feeds WHERE tag_id = :tag_id AND feed_id = :feed_id """ ) @@ -1179,8 +1235,8 @@ def get_feed_id_and_name(db_file, url): cur = conn.cursor() sql = ( """ - SELECT id, name - FROM feeds + SELECT id, title + FROM feeds_properties WHERE url = ? """ ) @@ -1198,7 +1254,7 @@ def get_number_of_items(db_file, table): db_file : str Path to database file. table : str - "entries" or "feeds". + "entries_properties" or "feeds_properties". Returns ------- @@ -1242,7 +1298,7 @@ def get_number_of_feeds_active(db_file): sql = ( """ SELECT count(id) - FROM feeds_state + FROM feeds_preferences WHERE enabled = 1 """ ) @@ -1271,16 +1327,9 @@ def get_number_of_entries_unread(db_file): cur = conn.cursor() sql = ( """ - SELECT - ( - SELECT count(id) - FROM entries - WHERE read = 0 - ) + ( - SELECT count(id) - FROM archive - ) - AS total_count + SELECT count(id) + FROM entries_state + WHERE read = 0 """ ) count = cur.execute(sql).fetchone()[0] @@ -1310,12 +1359,9 @@ def get_entries(db_file, num): cur = conn.cursor() sql = ( """ - SELECT id, title, link, summary, enclosure, feed_id, timestamp - FROM entries - UNION ALL - SELECT id, title, link, summary, enclosure, feed_id, timestamp - FROM archive - ORDER BY timestamp DESC + SELECT id, title, link, summary_text, feed_id, published + FROM entries_properties + ORDER BY published DESC LIMIT :num """ ) @@ -1347,14 +1393,11 @@ def get_entries_rejected(db_file, num): cur = conn.cursor() sql = ( """ - SELECT id, title, link, summary, enclosure, feed_id, timestamp - FROM entries - WHERE reject = 1 - UNION ALL - SELECT id, title, link, summary, enclosure, feed_id, timestamp - FROM archive - WHERE reject = 1 - ORDER BY timestamp DESC + SELECT entries_properties.id, title, link, summary_text, feed_id, published + FROM entries_properties + INNER JOIN entries_state ON entries_properties.id = entries_state.entry_id + WHERE entries_state.rejected = 1 + ORDER BY published DESC LIMIT :num """ ) @@ -1386,13 +1429,11 @@ def get_unread_entries(db_file, num): cur = conn.cursor() sql = ( """ - SELECT id, title, link, summary, enclosure, feed_id, timestamp - FROM entries - WHERE read = 0 - UNION ALL - SELECT id, title, link, summary, enclosure, feed_id, timestamp - FROM archive - ORDER BY timestamp DESC + SELECT entries_properties.id, title, link, summary_text, feed_id, published + FROM entries_properties + INNER JOIN entries_state ON entries_properties.id = entries_state.entry_id + WHERE entries_state.read = 0 + ORDER BY published DESC LIMIT :num """ ) @@ -1425,7 +1466,7 @@ def get_feed_id_by_entry_index(db_file, ix): sql = ( """ SELECT feed_id - FROM entries + FROM entries_properties WHERE id = :ix """ ) @@ -1458,7 +1499,7 @@ def get_feed_id(db_file, url): sql = ( """ SELECT id - FROM feeds + FROM feeds_properties WHERE url = :url """ ) @@ -1467,6 +1508,36 @@ def get_feed_id(db_file, url): return feed_id +def is_entry_archived(cur, ix): + """ + Check whether a given entry is archived. + + Parameters + ---------- + cur : object + Cursor object. + ix : str + Index of entry. + + Returns + ------- + result : tuple + Entry ID. + """ + function_name = sys._getframe().f_code.co_name + logger.debug('{}: ix: {}' + .format(function_name, ix)) + sql = ( + """ + SELECT id + FROM entries_state + WHERE archived = 1 AND entry_id = ? + """ + ) + par = (ix,) + result = cur.execute(sql, par).fetchone() + return result + async def mark_entry_as_read(cur, ix): """ Set read status of entry as read. @@ -1483,9 +1554,9 @@ async def mark_entry_as_read(cur, ix): .format(function_name, ix)) sql = ( """ - UPDATE entries + UPDATE entries_state SET read = 1 - WHERE id = ? + WHERE entry_id = ? """ ) par = (ix,) @@ -1539,8 +1610,9 @@ def get_unread_entries_of_feed(db_file, feed_id): sql = ( """ SELECT * - FROM entries - WHERE read = 0 AND feed_id = ? + FROM entries_properties + INNER JOIN entries_state ON entries_properties.id = entries_state.entry_id + WHERE entries_state.read = 0 AND feed_id = ? """ ) par = (feed_id,) @@ -1566,9 +1638,10 @@ def get_number_of_unread_entries_by_feed(db_file, feed_id): cur = conn.cursor() sql = ( """ - SELECT count(id) - FROM entries - WHERE read = 0 AND feed_id = ? + SELECT count(entries_properties.id) + FROM entries_properties + INNER JOIN entries_state ON entries_properties.id = entries_state.entry_id + WHERE entries_state.read = 0 AND feed_id = ? """ ) par = (feed_id,) @@ -1585,7 +1658,7 @@ async def mark_feed_as_read(db_file, feed_id): db_file : str Path to database file. feed_id : str - Feed Id. + Feed ID. """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} feed_id: {}' @@ -1595,13 +1668,24 @@ async def mark_feed_as_read(db_file, feed_id): cur = conn.cursor() sql = ( """ - UPDATE entries - SET read = 1 + SELECT id + FROM entries_properties WHERE feed_id = ? """ ) par = (feed_id,) - cur.execute(sql, par) + ixs = cur.execute(sql, par).fetchall() + sql = ( + """ + UPDATE entries_state + SET read = 1 + WHERE entry_id = ? + """ + ) + for ix in ixs: cur.execute(sql, ix) + # for ix in ixs: + # par = ix # Variable ix is already of type tuple + # cur.execute(sql, par) async def delete_entry_by_id(db_file, ix): @@ -1624,7 +1708,7 @@ async def delete_entry_by_id(db_file, ix): sql = ( """ DELETE - FROM entries + FROM entries_properties WHERE id = :ix """ ) @@ -1651,32 +1735,13 @@ async def archive_entry(db_file, ix): cur = conn.cursor() sql = ( """ - INSERT - INTO archive - SELECT * - FROM entries - WHERE entries.id = :ix + UPDATE entries_state + SET archived = 1 + WHERE entry_id = :ix """ ) par = (ix,) - try: - cur.execute(sql, par) - except Exception as e: - print('ERROR DB insert from entries into archive at index {} ' - 'for {}. Reason: {}'.format(ix, db_file, e)) - sql = ( - """ - DELETE - FROM entries - WHERE id = :ix - """ - ) - par = (ix,) - try: - cur.execute(sql, par) - except: - print('ERROR DB deleting items from table entries at index {} ' - 'for DB {}', ix, db_file) + cur.execute(sql, par) def get_feed_title(db_file, ix): @@ -1687,8 +1752,8 @@ def get_feed_title(db_file, ix): cur = conn.cursor() sql = ( """ - SELECT name - FROM feeds + SELECT title + FROM feeds_properties WHERE id = :ix """ ) @@ -1697,7 +1762,25 @@ def get_feed_title(db_file, ix): return title -async def set_feed_title(db_file, feed_id, name): +def get_feed_subtitle(db_file, feed_id): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: db_file: {} feed_id: {}' + .format(function_name, db_file, feed_id)) + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + SELECT subtitle + FROM feeds_properties + WHERE feed_id = :feed_id + """ + ) + par = (feed_id,) + title = cur.execute(sql, par).fetchone() + return title + + +async def set_feed_title(db_file, feed_id, title): """ Set new name for feed. @@ -1711,20 +1794,20 @@ async def set_feed_title(db_file, feed_id, name): New name. """ function_name = sys._getframe().f_code.co_name - logger.debug('{}: db_file: {} feed_id: {} name: {}' - .format(function_name, db_file, feed_id, name)) + logger.debug('{}: db_file: {} feed_id: {} title: {}' + .format(function_name, db_file, feed_id, title)) async with DBLOCK: with create_connection(db_file) as conn: cur = conn.cursor() sql = ( """ - UPDATE feeds - SET name = :name + UPDATE feeds_properties + SET title = :title WHERE id = :feed_id """ ) par = { - "name": name, + "title": title, "feed_id": feed_id } cur.execute(sql, par) @@ -1736,10 +1819,10 @@ def get_entry_title(db_file, ix): .format(function_name, db_file, ix)) with create_connection(db_file) as conn: cur = conn.cursor() - sql = ( # TODO Handletable archive too + sql = ( """ SELECT title - FROM entries + FROM entries_properties WHERE id = :ix """ ) @@ -1754,10 +1837,10 @@ def get_entry_url(db_file, ix): .format(function_name, db_file, ix)) with create_connection(db_file) as conn: cur = conn.cursor() - sql = ( # TODO Handletable archive too + sql = ( """ SELECT link - FROM entries + FROM entries_properties WHERE id = :ix """ ) @@ -1775,7 +1858,7 @@ def get_feed_url(db_file, ix): sql = ( """ SELECT url - FROM feeds + FROM feeds_properties WHERE id = :ix """ ) @@ -1787,7 +1870,7 @@ def get_feed_url(db_file, ix): async def mark_as_read(db_file, ix): function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} ix: {}' - .format(function_name, db_file, ix)) + .format(function_name, db_file, ix)) async with DBLOCK: with create_connection(db_file) as conn: cur = conn.cursor() @@ -1797,8 +1880,11 @@ async def mark_as_read(db_file, ix): # NOTE: We can use DBLOCK once for both # functions, because, due to exclusive # ID, only one can ever occur. - await mark_entry_as_read(cur, ix) - await delete_archived_entry(cur, ix) + if is_entry_archived(cur, ix): + await delete_entry(cur, ix) + else: + await mark_entry_as_read(cur, ix) + async def mark_all_as_read(db_file): @@ -1812,29 +1898,42 @@ async def mark_all_as_read(db_file): """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {}' - .format(function_name, db_file)) + .format(function_name, db_file)) async with DBLOCK: with create_connection(db_file) as conn: cur = conn.cursor() sql = ( """ - UPDATE entries + UPDATE entries_state SET read = 1 """ ) cur.execute(sql) + + sql = ( + """ + SELECT entries_properties.id, title, link, summary_text, feed_id, published + FROM entries_properties + INNER JOIN entries_state ON entries_properties.id = entries_state.entry_id + WHERE entries_state.archive = 1 + """ + ) + ixs = cur.execute(sql).fetchall() sql = ( """ DELETE - FROM archive + FROM entries_properties + WHERE id = ? """ ) + for ix in ixs: cur.execute(sql, ix) + cur.execute(sql) -async def delete_archived_entry(cur, ix): +async def delete_entry(cur, ix): """ - Delete entry from table archive. + Delete entry. Parameters ---------- @@ -1849,7 +1948,7 @@ async def delete_archived_entry(cur, ix): sql = ( """ DELETE - FROM archive + FROM entries_properties WHERE id = ? """ ) @@ -1869,8 +1968,8 @@ async def update_statistics(cur): function_name = sys._getframe().f_code.co_name logger.debug('{}'.format(function_name)) stat_dict = {} - stat_dict["feeds"] = get_number_of_items(cur, 'feeds') - stat_dict["entries"] = get_number_of_items(cur, 'entries') + stat_dict["feeds"] = get_number_of_items(cur, 'feeds_properties') + stat_dict["entries"] = get_number_of_items(cur, 'entries_properties') stat_dict["unread"] = get_number_of_entries_unread(cur=cur) for i in stat_dict: sql = ( @@ -1927,7 +2026,7 @@ async def set_enabled_status(db_file, feed_id, status): cur = conn.cursor() sql = ( """ - UPDATE feeds_state + UPDATE feeds_preferences SET enabled = :status WHERE feed_id = :feed_id """ @@ -1982,9 +2081,9 @@ async def add_entry(db_file, title, link, entry_id, feed_id, date, """ INSERT INTO entries( - title, link, entry_id, feed_id, timestamp, read) + title, link, entry_id, feed_id, published, read) VALUES( - :title, :link, :entry_id, :feed_id, :timestamp, :read) + :title, :link, :entry_id, :feed_id, :published, :read) """ ) par = { @@ -1992,7 +2091,7 @@ async def add_entry(db_file, title, link, entry_id, feed_id, date, "link": link, "entry_id": entry_id, "feed_id": feed_id, - "timestamp": date, + "published": date, "read": read_status } cur.execute(sql, par) @@ -2013,9 +2112,9 @@ async def add_entry(db_file, title, link, entry_id, feed_id, date, # # breakpoint() -async def add_entries_and_update_timestamp(db_file, feed_id, new_entries): +async def add_entries_and_update_feed_state(db_file, feed_id, new_entries): """ - Add new entries. + Add new entries and update feed state. Parameters ---------- @@ -2030,29 +2129,141 @@ async def add_entries_and_update_timestamp(db_file, feed_id, new_entries): async with DBLOCK: with create_connection(db_file) as conn: cur = conn.cursor() - for entry in new_entries: - logger.debug('{}: db_file: {} feed_id: {} entry: {}' - .format(function_name, db_file, feed_id, entry["title"])) + for new_entry in new_entries: + logger.debug('{}: db_file: {} feed_id: {}' + .format(function_name, db_file, feed_id)) sql = ( """ INSERT - INTO entries( - title, link, summary, enclosure, entry_id, feed_id, timestamp, read) + INTO entries_properties( + feed_id, identifier, link, title, title_type, summary_text, summary_lang, summary_type, summary_base, category, comments, published, updated) VALUES( - :title, :link, :summary, :enclosure, :entry_id, :feed_id, :timestamp, :read) + :feed_id, :identifier, :link, :title, :title_type, :summary_text, :summary_lang, :summary_type, :summary_base, :category, :comments, :published, :updated) + """ + ) + entry_properties = new_entry['entry_properties'] + par = { + "feed_id": feed_id, + "identifier": entry_properties["identifier"], + "link": entry_properties["link"], + "title": entry_properties["title"], + "title_type": entry_properties["title_type"], + 'summary_text' : entry_properties['summary_text'], + 'summary_lang' : entry_properties['summary_lang'], + 'summary_type' : entry_properties['summary_type'], + 'summary_base' : entry_properties['summary_base'], + 'category' : entry_properties['category'], + "comments": entry_properties["comments"], + "published": entry_properties["published"], + "updated": entry_properties["updated"], + } + cur.execute(sql, par) + entry_id = cur.lastrowid + sql = ( + """ + INSERT + INTO entries_state( + entry_id) + VALUES( + :entry_id) """ ) par = { - "title": entry["title"], - "link": entry["link"], - "summary": entry["summary"], - "enclosure": entry["enclosure"], - "entry_id": entry["entry_id"], - "feed_id": feed_id, - "timestamp": entry["date"], - "read": entry["read_status"] + "entry_id": entry_id, } cur.execute(sql, par) + entry_authors = new_entry['entry_authors'] + for entry_author in entry_authors: + sql = ( + """ + INSERT + INTO entries_properties_authors( + entry_id, name, url, email) + VALUES( + :entry_id, :name, :url, :email) + """ + ) + par = { + "entry_id": entry_id, + "name": entry_author['name'], + "url": entry_author['url'], + "email": entry_author['email'], + } + cur.execute(sql, par) + entry_contributors = new_entry['entry_contributors'] + for entry_contributor in entry_contributors: + sql = ( + """ + INSERT + INTO entries_properties_contributors( + entry_id, name, url, email) + VALUES( + :entry_id, :name, :url, :email) + """ + ) + par = { + "entry_id": entry_id, + "name": entry_contributor['name'], + "url": entry_contributor['url'], + "email": entry_contributor['email'], + } + cur.execute(sql, par) + entry_contents = new_entry['entry_contents'] + for entry_content in entry_contents: + sql = ( + """ + INSERT + INTO entries_properties_contents( + entry_id, text, type, base, lang) + VALUES( + :entry_id, :text, :type, :base, :lang) + """ + ) + par = { + "entry_id": entry_id, + "text": entry_content['text'], + "type": entry_content['type'], + "base": entry_content['base'], + "lang": entry_content['lang'], + } + cur.execute(sql, par) + entry_links = new_entry['entry_tags'] + for entry_link in entry_links: + sql = ( + """ + INSERT + INTO entries_properties_tags( + entry_id, term, scheme, label) + VALUES( + :entry_id, :term, :scheme, :label) + """ + ) + par = { + "entry_id": entry_id, + "term": entry_link['term'], + "scheme": entry_link['scheme'], + "label": entry_link['label'], + } + cur.execute(sql, par) + entry_links = new_entry['entry_links'] + for entry_link in entry_links: + sql = ( + """ + INSERT + INTO entries_properties_links( + entry_id, url, type, rel, size) + VALUES( + :entry_id, :url, :type, :rel, :size) + """ + ) + par = { + "entry_id": entry_id, + "url": entry_link['url'], + "rel": entry_link['rel'], + "type": entry_link['type'], + "size": entry_link['length'], + } + cur.execute(sql, par) sql = ( """ UPDATE feeds_state @@ -2099,6 +2310,27 @@ async def set_date(db_file, feed_id): cur.execute(sql, par) +async def update_feed_identifier(db_file, feed_id, identifier): + function_name = sys._getframe().f_code.co_name + logger.debug('{}: db_file: {} feed_id: {} identifier: {}' + .format(function_name, db_file, feed_id, identifier)) + async with DBLOCK: + with create_connection(db_file) as conn: + cur = conn.cursor() + sql = ( + """ + UPDATE feeds_properties + SET identifier = :identifier + WHERE feed_id = :feed_id + """ + ) + par = { + "identifier": identifier, + "feed_id": feed_id + } + cur.execute(sql, par) + + async def update_feed_status(db_file, feed_id, status_code): """ Set status_code of feed_id in table status. @@ -2113,8 +2345,11 @@ async def update_feed_status(db_file, feed_id, status_code): Status ID or message. """ function_name = sys._getframe().f_code.co_name - print('{}: db_file: {} feed_id: {} status_code: {}' + logger.debug('{}: db_file: {} feed_id: {} status_code: {}' .format(function_name, db_file, feed_id, status_code)) + if status_code != 200: + print('{}: db_file: {} feed_id: {} status_code: {}' + .format(function_name, db_file, feed_id, status_code)) async with DBLOCK: with create_connection(db_file) as conn: cur = conn.cursor() @@ -2166,7 +2401,7 @@ async def update_feed_validity(db_file, feed_id, valid): cur.execute(sql, par) -async def update_feed_properties(db_file, feed_id, entries, updated): +async def update_feed_properties(db_file, feed_id, feed_properties): """ Update properties of url in table feeds. @@ -2176,26 +2411,36 @@ async def update_feed_properties(db_file, feed_id, entries, updated): Path to database file. url : str Feed URL. - entries : int - Number of entries. - updated : ??? - Date feed was last updated. + feed_properties : dict + Feed properties. """ function_name = sys._getframe().f_code.co_name - logger.debug('{}: db_file: {} feed_id: {} entries: {} updated: {}' - .format(function_name, db_file, feed_id, entries, updated)) + logger.debug('{}: db_file: {} feed_id: {} feed_properties: {}' + .format(function_name, db_file, feed_id, feed_properties)) async with DBLOCK: with create_connection(db_file) as conn: cur = conn.cursor() sql = ( """ UPDATE feeds_properties - SET entries = :entries - WHERE feed_id = :feed_id + SET version = :version, encoding = :encoding, + language = :language, rating = :rating, + entries = :entries, icon = :icon, image = :image, + logo = :logo, ttl = :ttl, updated = :updated + WHERE id = :feed_id """ ) par = { - "entries" : entries, + "version" : feed_properties['version'], + "encoding" : feed_properties['encoding'], + "language" : feed_properties['language'], + "rating" : feed_properties['rating'], + "entries" : feed_properties['entries_count'], + "icon" : feed_properties['icon'], + "image" : feed_properties['image'], + "logo" : feed_properties['logo'], + "ttl" : feed_properties['ttl'], + "updated" : feed_properties['updated'], "feed_id": feed_id } cur.execute(sql, par) @@ -2221,7 +2466,8 @@ async def maintain_archive(db_file, limit): sql = ( """ SELECT count(id) - FROM archive + FROM entries_state + WHERE archived = 1 """ ) count = cur.execute(sql).fetchone()[0] @@ -2236,12 +2482,14 @@ async def maintain_archive(db_file, limit): sql = ( """ DELETE - FROM archive + FROM entries_properties WHERE id IN ( - SELECT id - FROM archive - ORDER BY timestamp ASC + SELECT entry_id + FROM entries_state + INNER JOIN entries_properties ON entries_state.entry_id = entries_properties.id + WHERE archived = 1 + ORDER BY published ASC LIMIT :difference ) """ @@ -2274,10 +2522,10 @@ def get_entries_of_feed(db_file, feed_id): cur = conn.cursor() sql = ( """ - SELECT id, title, link, entry_id, timestamp, read - FROM entries + SELECT id, title, link, identifier, published, read + FROM entries_properties WHERE feed_id = ? - ORDER BY timestamp DESC + ORDER BY published DESC """ ) par = (feed_id,) @@ -2333,7 +2581,7 @@ def get_feeds_url(db_file): sql = ( """ SELECT url - FROM feeds + FROM feeds_properties """ ) result = cur.execute(sql).fetchall() @@ -2367,10 +2615,10 @@ def get_feeds_by_enabled_state(db_file, enabled_state): cur = conn.cursor() sql = ( """ - SELECT feeds.* - FROM feeds - INNER JOIN feeds_state ON feeds.id = feeds_state.feed_id - WHERE feeds_state.enabled = ? + SELECT feeds_properties.* + FROM feeds_properties + INNER JOIN feeds_preferences ON feeds_properties.id = feeds_preferences.feed_id + WHERE feeds_preferences.enabled = ? """ ) par = (enabled_state,) @@ -2399,10 +2647,10 @@ def get_feeds_and_enabled_state(db_file): cur = conn.cursor() sql = ( """ - SELECT feeds.*, feeds_state.enabled - FROM feeds - INNER JOIN feeds_state ON feeds.id = feeds_state.feed_id - ORDER BY feeds.name ASC + SELECT feeds_properties.*, feeds_preferences.enabled + FROM feeds_properties + INNER JOIN feeds_preferences ON feeds_properties.id = feeds_preferences.feed_id + ORDER BY feeds_properties.title ASC """ ) result = cur.execute(sql).fetchall() @@ -2430,10 +2678,10 @@ def get_active_feeds_url(db_file): cur = conn.cursor() sql = ( """ - SELECT feeds.url - FROM feeds - INNER JOIN feeds_state ON feeds.id = feeds_state.feed_id - WHERE feeds_state.enabled = 1 + SELECT feeds_properties.url + FROM feeds_properties + INNER JOIN feeds_preferences ON feeds_properties.id = feeds_preferences.feed_id + WHERE feeds_preferences.enabled = 1 """ ) result = cur.execute(sql).fetchall() @@ -2496,9 +2744,9 @@ def get_feeds(db_file): cur = conn.cursor() sql = ( """ - SELECT id, name, url - FROM feeds - ORDER BY name + SELECT id, title, url + FROM feeds_properties + ORDER BY title """ ) result = cur.execute(sql).fetchall() @@ -2534,14 +2782,11 @@ def get_last_entries(db_file, num): # ) sql = ( """ - SELECT title, link, timestamp - FROM entries - WHERE read = 0 - UNION ALL - SELECT title, link, timestamp - FROM archive - WHERE read = 0 - ORDER BY timestamp DESC + SELECT title, link, published + FROM entries_properties + INNER JOIN entries_state ON entries_properties.id = entries_state.entry_id + WHERE entries_state.read = 0 + ORDER BY published DESC LIMIT :num """ ) @@ -2573,9 +2818,9 @@ def search_feeds(db_file, query): cur = conn.cursor() sql = ( """ - SELECT name, id, url - FROM feeds - WHERE name LIKE ? + SELECT title, id, url + FROM feeds_properties + WHERE title LIKE ? OR url LIKE ? LIMIT 50 """ @@ -2609,11 +2854,7 @@ async def search_entries(db_file, query): sql = ( """ SELECT title, link - FROM entries - WHERE title LIKE ? - UNION ALL - SELECT title, link - FROM archive + FROM entries_properties WHERE title LIKE ? LIMIT 50 """ @@ -2645,12 +2886,12 @@ ERROR DATE: result = https://blog.heckel.io/feed/ 19:32:06 ERROR DATE: result = https://mwl.io/feed """ -def check_entry_exist(db_file, feed_id, entry_id=None, title=None, link=None, - date=None): +def check_entry_exist(db_file, feed_id, identifier=None, title=None, link=None, + published=None): """ Check whether an entry exists. If entry has an ID, check by ID. - If entry has timestamp, check by title, link and date. + If entry has timestamp (published), check by title, link and date. Otherwise, check by title and link. Parameters @@ -2659,13 +2900,13 @@ def check_entry_exist(db_file, feed_id, entry_id=None, title=None, link=None, Path to database file. feed_id : str Feed Id. - entry_id : str, optional + identifier : str, optional Entry ID. The default is None. title : str, optional Entry title. The default is None. link : str, optional Entry URL. The default is None. - date : str, optional + published : str, optional Entry Timestamp. The default is None. Returns @@ -2679,44 +2920,44 @@ def check_entry_exist(db_file, feed_id, entry_id=None, title=None, link=None, with create_connection(db_file) as conn: cur = conn.cursor() exist = False - if entry_id: + if identifier: sql = ( """ SELECT id - FROM entries - WHERE entry_id = :entry_id and feed_id = :feed_id + FROM entries_properties + WHERE identifier = :identifier and feed_id = :feed_id """ ) par = { - "entry_id": entry_id, + "identifier": identifier, "feed_id": feed_id } result = cur.execute(sql, par).fetchone() if result: exist = True - elif date: + elif published: sql = ( """ SELECT id - FROM entries - WHERE title = :title AND link = :link AND timestamp = :date + FROM entries_properties + WHERE title = :title AND link = :link AND published = :published """ ) par = { "title": title, "link": link, - "date": date + "date": published } try: result = cur.execute(sql, par).fetchone() if result: exist = True except: logger.error("source =", feed_id) - logger.error("date =", date) + logger.error("published =", published) else: sql = ( """ SELECT id - FROM entries + FROM entries_properties WHERE title = :title AND link = :link """ ) diff --git a/slixfeed/task.py b/slixfeed/task.py index 1c992b5..431df19 100644 --- a/slixfeed/task.py +++ b/slixfeed/task.py @@ -68,12 +68,15 @@ except Exception as exc: """ import asyncio +from feedparser import parse import logging import os import slixfeed.action as action import slixfeed.config as config from slixfeed.config import Config # from slixfeed.dt import current_time +import slixfeed.dt as dt +import slixfeed.fetch as fetch import slixfeed.sqlite as sqlite # from xmpp import Slixfeed from slixfeed.xmpp.presence import XmppPresence @@ -339,14 +342,67 @@ async def check_updates(self, jid_bare): jid : str Jabber ID. """ + print('Scanning for updates for JID {}'.format(jid_bare)) logging.info('Scanning for updates for JID {}'.format(jid_bare)) while True: jid_file = jid_bare.replace('/', '_') db_file = config.get_pathname_to_database(jid_file) urls = sqlite.get_active_feeds_url(db_file) for url in urls: - await action.scan(self, jid_bare, db_file, url) - await asyncio.sleep(50) + url = url[0] + print('STA',url) + + result = await fetch.http(url) + status_code = result['status_code'] + feed_id = sqlite.get_feed_id(db_file, url) + feed_id = feed_id[0] + if not result['error']: + await sqlite.update_feed_status(db_file, feed_id, status_code) + document = result['content'] + feed = parse(document) + + feed_valid = 0 if feed.bozo else 1 + await sqlite.update_feed_validity(db_file, feed_id, feed_valid) + + if feed.has_key('updated_parsed'): + feed_updated = feed.updated_parsed + try: + feed_updated = dt.convert_struct_time_to_iso8601(feed_updated) + except: + feed_updated = '' + else: + feed_updated = '' + + entries_count = len(feed.entries) + + feed_version = feed.version if feed.has_key('version') else '' + feed_encoding = feed.encoding if feed.has_key('encoding') else '' + feed_language = feed.feed.language if feed.feed.has_key('language') else '' + feed_icon = feed.feed.icon if feed.feed.has_key('icon') else '' + feed_image = feed.feed.image if feed.feed.has_key('image') else '' + feed_logo = feed.feed.logo if feed.feed.has_key('logo') else '' + feed_ttl = feed.feed.ttl if feed.feed.has_key('ttl') else '' + + feed_properties = { + "version" : feed_version, + "encoding" : feed_encoding, + "language" : feed_language, + "rating" : '', + "entries_count" : entries_count, + "icon" : feed_icon, + "image" : feed_image, + "logo" : feed_logo, + "ttl" : feed_ttl, + "updated" : feed_updated, + } + await sqlite.update_feed_properties(db_file, feed_id, + feed_properties) + new_entries = action.get_properties_of_entries( + self, jid_bare, db_file, url, feed_id, feed) + if new_entries: await sqlite.add_entries_and_update_feed_state( + db_file, feed_id, new_entries) + print('END', url) + await asyncio.sleep(5) val = Config.get_setting_value(self.settings, jid_bare, 'check') await asyncio.sleep(60 * float(val)) # Schedule to call this function again in 90 minutes diff --git a/slixfeed/url.py b/slixfeed/url.py index aa628d2..ef0d459 100644 --- a/slixfeed/url.py +++ b/slixfeed/url.py @@ -50,7 +50,7 @@ def get_hostname(url): return hostname -def replace_hostname(url, url_type): +async def replace_hostname(url, url_type): """ Replace hostname. @@ -79,6 +79,8 @@ def replace_hostname(url, url_type): proxy = proxies[proxy_name] if hostname in proxy['hostname'] and url_type in proxy['type']: while not url_new: + print('>>>') + print(url_new) proxy_type = 'clearnet' proxy_list = proxy[proxy_type] if len(proxy_list): @@ -89,10 +91,13 @@ def replace_hostname(url, url_type): hostname_new = parted_proxy_url.netloc url_new = urlunsplit([protocol_new, hostname_new, pathname, queries, fragment]) - response = fetch.http_response(url_new) + print(proxy_url) + print(url_new) + print('>>>') + response = await fetch.http(url_new) if (response and - response.status_code == 200 and - response.reason == 'OK' and + response['status_code'] == 200 and + # response.reason == 'OK' and url_new.startswith(proxy_url)): break else: @@ -104,13 +109,16 @@ def replace_hostname(url, url_type): config.backup_obsolete(proxies_obsolete_file, proxy_name, proxy_type, proxy_url) - config.update_proxies(proxies_file, proxy_name, - proxy_type, proxy_url) + try: + config.update_proxies(proxies_file, proxy_name, + proxy_type, proxy_url) + except ValueError as e: + logging.error([str(e), proxy_url]) url_new = None else: logging.warning( - "No proxy URLs for {}." - "Update proxies.toml".format(proxy_name)) + "No proxy URLs for {}. Please update proxies.toml" + .format(proxy_name)) url_new = url break return url_new diff --git a/slixfeed/version.py b/slixfeed/version.py index 1eaf6c0..992dc2c 100644 --- a/slixfeed/version.py +++ b/slixfeed/version.py @@ -1,2 +1,2 @@ -__version__ = '0.1.52' -__version_info__ = (0, 1, 52) +__version__ = '0.1.53' +__version_info__ = (0, 1, 53) diff --git a/slixfeed/xmpp/bookmark.py b/slixfeed/xmpp/bookmark.py index 3fec8ee..c2befa7 100644 --- a/slixfeed/xmpp/bookmark.py +++ b/slixfeed/xmpp/bookmark.py @@ -15,7 +15,7 @@ from slixmpp.plugins.xep_0048.stanza import Bookmarks class XmppBookmark: - async def get(self): + async def get_bookmarks(self): result = await self.plugin['xep_0048'].get_bookmarks() conferences = result['private']['bookmarks']['conferences'] return conferences diff --git a/slixfeed/xmpp/client.py b/slixfeed/xmpp/client.py index 48dd241..e04a27b 100644 --- a/slixfeed/xmpp/client.py +++ b/slixfeed/xmpp/client.py @@ -283,13 +283,14 @@ class Slixfeed(slixmpp.ClientXMPP): await self['xep_0115'].update_caps() # self.send_presence() await self.get_roster() - bookmarks = await XmppBookmark.get(self) + bookmarks = await XmppBookmark.get_bookmarks(self) await action.xmpp_muc_autojoin(self, bookmarks) - jids = await XmppPubsub.get_pubsub_services(self) - for jid_bare in jids: + results = await XmppPubsub.get_pubsub_services(self) + for result in results + [{'jid' : self.boundjid.bare, + 'name' : self.alias}]: + jid_bare = result['jid'] if jid_bare not in self.settings: - jid_file = jid_bare - db_file = config.get_pathname_to_database(jid_file) + db_file = config.get_pathname_to_database(jid_bare) Config.add_settings_jid(self.settings, jid_bare, db_file) await task.start_tasks_xmpp_pubsub(self, jid_bare) # XmppCommand.adhoc_commands(self) @@ -309,7 +310,7 @@ class Slixfeed(slixmpp.ClientXMPP): # self.send_presence() profile.set_identity(self, 'client') self['xep_0115'].update_caps() - bookmarks = await XmppBookmark.get(self) + bookmarks = await XmppBookmark.get_bookmarks(self) await action.xmpp_muc_autojoin(self, bookmarks) time_end = time.time() difference = time_end - time_begin @@ -747,9 +748,9 @@ class Slixfeed(slixmpp.ClientXMPP): jid_bare = session['from'].bare if is_operator(self, jid_bare): form = self['xep_0004'].make_form('form', 'PubSub') - form['instructions'] = 'Manage nodes and publish news into nodes.' - options = form.add_field(desc=('Send a set of selected posts or ' - 'set a new subscription.'), + form['instructions'] = 'Manage PubSub nodes and publish news items.' + options = form.add_field(desc='Send a set of selected posts or ' + 'set a new subscription.', ftype='list-single', label='Choose', required=True, @@ -780,24 +781,24 @@ class Slixfeed(slixmpp.ClientXMPP): form['instructions'] = ('Choose a PubSub Jabber ID and verify ' 'that Slixfeed has the necessary ' 'permissions to publish into it.') - form.add_field(var='url', + form.add_field(desc='Enter a subscription URL.', # TODO Make it possible to add several subscriptions at once; # Similarly to BitTorrent trackers list # ftype='text-multi', # label='Subscription URLs', - # desc=('Add subscriptions one time per ' - # 'subscription.'), + # desc='Add subscriptions one time per ' + # 'subscription.', ftype='text-single', label='URL', - desc='Enter a subscription URL.', + required=True, value='http://', - required=True) - options = form.add_field(var='jid', - ftype='list-single', - label='PubSub', - desc='Select a PubSub Service.', - value=self.boundjid.bare, - required=True) + var='url') + options = form.add_field(desc='Select a PubSub Service.', + ftype='list-single', + label='PubSub', + required=True, + value=self.boundjid.bare, + var='jid') options.addOption(self.boundjid.bare, self.boundjid.bare) iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) items = iq['disco_items']['items'] @@ -811,23 +812,23 @@ class Slixfeed(slixmpp.ClientXMPP): elif item[2]: name = item[2] else: name = jid options.addOption(jid, name) - # form.add_field(var='jid', + # form.add_field(desc='Enter a PubSub Jabber ID.', # ftype='text-single', # label='PubSub', - # desc='Enter a PubSub Jabber ID.', + # required=True, # value=self.boundjid.bare, # # value='pubsub.' + self.boundjid.host, - # required=True) - form.add_field(var='node', + # var='jid') + form.add_field(desc='Enter a node to publish to.', ftype='text-single', label='Node', - desc='Enter a node to publish to.') - # options = form.add_field(var='xep', + var='node') + # options = form.add_field(desc='Select XMPP Extension Protocol.', # ftype='list-single', # label='Protocol', - # desc='Select XMPP Extension Protocol.', + # required=True, # value='0060', - # required=True) + # var='xep') # options.addOption('XEP-0060: Publish-Subscribe', '0060') # options.addOption('XEP-0277: Microblogging over XMPP', '0277') # options.addOption('XEP-0472: Pubsub Social Feed', '0472') @@ -838,18 +839,18 @@ class Slixfeed(slixmpp.ClientXMPP): form['instructions'] = ('Choose a PubSub Jabber ID and verify ' 'that Slixfeed has the necessary ' 'permissions to publish into it.') - form.add_field(var='url', + form.add_field(desc='Enter a subscription URL.', ftype='text-single', label='URL', - desc='Enter a subscription URL.', + required=True, value='http://', - required=True) - options = form.add_field(var='jid', - ftype='list-single', - label='PubSub', - desc='Select a PubSub Service.', - value=self.boundjid.bare, - required=True) + var='url') + options = form.add_field(desc='Select a PubSub Service.', + ftype='list-single', + label='PubSub', + required=True, + value=self.boundjid.bare, + var='jid') options.addOption(self.boundjid.bare, self.boundjid.bare) iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) items = iq['disco_items']['items'] @@ -863,23 +864,23 @@ class Slixfeed(slixmpp.ClientXMPP): elif item[2]: name = item[2] else: name = jid options.addOption(jid, name) - # form.add_field(var='jid', + # form.add_field(desc='Enter a PubSub Jabber ID.', # ftype='text-single', # label='PubSub', - # desc='Enter a PubSub Jabber ID.', - # value=self.boundjid.bare, + # required=True, # # value='pubsub.' + self.boundjid.host, - # required=True) - form.add_field(var='node', + # value=self.boundjid.bare, + # var='jid') + form.add_field(desc='Enter a node to publish to.', ftype='text-single', label='Node', - desc='Enter a node to publish to.') - # options = form.add_field(var='xep', + var='node') + # options = form.add_field(desc='Select XMPP Extension Protocol.', # ftype='list-single', # label='Protocol', - # desc='Select XMPP Extension Protocol.', + # required=True, # value='0060', - # required=True) + # var='xep') # options.addOption('XEP-0060: Publish-Subscribe', '0060') # options.addOption('XEP-0277: Microblogging over XMPP', '0277') # options.addOption('XEP-0472: Pubsub Social Feed', '0472') @@ -926,11 +927,11 @@ class Slixfeed(slixmpp.ClientXMPP): # if is_feed(url, feed): if action.is_feed(feed): form['instructions'] = 'Select entries to publish.' - options = form.add_field(var='entries', + options = form.add_field(desc='Select entries to post.', ftype='list-multi', label='Titles', - desc='Select entries to post.', - required=True) + required=True, + var='entries') if "title" in feed["feed"].keys(): title = feed["feed"]["title"] else: @@ -967,11 +968,11 @@ class Slixfeed(slixmpp.ClientXMPP): form['instructions'] = ('Discovered {} subscriptions ' 'for {}' .format(len(results), url)) - options = form.add_field(var='url', + options = form.add_field(desc='Select a feed.', ftype='list-single', label='Feeds', - desc='Select a feed.', - required=True) + required=True, + var='url') for result in results: title = result['name'] url = result['link'] @@ -1202,18 +1203,18 @@ class Slixfeed(slixmpp.ClientXMPP): form['instructions'] = 'Editing filters' # 🪄️ 🛡️ value = sqlite.get_filter_value(db_file, 'allow') if value: value = str(value[0]) - form.add_field(var='allow', + form.add_field(desc='Keywords to allow (comma-separated keywords).', ftype='text-single', label='Allow list', value=value, - desc='Keywords to allow (comma-separated keywords).') + var='allow') value = sqlite.get_filter_value(db_file, 'deny') if value: value = str(value[0]) - form.add_field(var='deny', + form.add_field(desc='Keywords to deny (comma-separated keywords).', ftype='text-single', label='Deny list', value=value, - desc='Keywords to deny (comma-separated keywords).') + var='deny') session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_filters_complete @@ -1288,32 +1289,32 @@ class Slixfeed(slixmpp.ClientXMPP): if chat_type == 'chat' or moderator: form = self['xep_0004'].make_form('form', 'Subscription') form['instructions'] = 'Adding subscription' - form.add_field(var='subscription', + form.add_field(desc='Enter a subscription URL.', # TODO Make it possible to add several subscriptions at once; # Similarly to BitTorrent trackers list # ftype='text-multi', # label='Subscription URLs', - # desc=('Add subscriptions one time per ' - # 'subscription.'), + # desc='Add subscriptions one time per ' + # 'subscription.', ftype='text-single', label='URL', - desc='Enter a subscription URL.', + required=True, value='http://', - required=True) + var='subscription') if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to add the ' - 'subscription to (The default Jabber ID ' - 'is your own).'), + form.add_field(desc='Enter a Jabber ID to add the ' + 'subscription to (The default Jabber ID is ' + 'your own).', ftype='text-single', label='Jabber ID', var='jid') - # form.add_field(var='scan', + # form.add_field(desc='Scan URL for validity (recommended).', # ftype='boolean', # label='Scan', - # desc='Scan URL for validity (recommended).', - # value=True) + # value=True, + # var='scan') session['allow_prev'] = False session['has_next'] = True session['next'] = self._handle_subscription_new @@ -1331,18 +1332,40 @@ class Slixfeed(slixmpp.ClientXMPP): function_name = sys._getframe().f_code.co_name logger.debug('{}: jid_full: {}' .format(function_name, jid_full)) + jid_bare = session['from'].bare form = self['xep_0004'].make_form('form', 'Updates') form['instructions'] = 'Browse and read news' - options = form.add_field(var='action', + options = form.add_field(desc='What would you want to read?', ftype='list-single', label='Read', - desc=('What would you want to read?'), - required=True) + required=True, + var='action') options.addOption('All news', 'all') # options.addOption('News by subscription', 'feed') # options.addOption('News by tag', 'tag') options.addOption('Rejected news', 'reject') options.addOption('Unread news', 'unread') + if is_operator(self, jid_bare): + form.add_field(ftype='fixed', + label='Subscriber') + options = form.add_field(desc='Select a Jabber ID (The default ' + 'Jabber ID is your own).', + ftype='list-single', + label='Jabber ID', + value=jid_bare, + var='jid') + jids = [] + contacts = await XmppRoster.get_contacts(self) + for contact in contacts: + jids.extend([contact]) + conferences = await XmppBookmark.get_bookmarks(self) + for conference in conferences: + jids.extend([conference['jid']]) + pubsubs = await XmppPubsub.get_pubsub_services(self) + for pubsub in pubsubs: + jids.extend([pubsub['jid']]) + for jid_bare in sorted(jids): + options.addOption(jid_bare, jid_bare) session['allow_prev'] = False # Cheogram changes style if that button - which should not be on this form - is present session['has_next'] = True session['next'] = self._handle_recent_result @@ -1357,10 +1380,19 @@ class Slixfeed(slixmpp.ClientXMPP): logger.debug('{}: jid_full: {}' .format(function_name, jid_full)) jid_bare = session['from'].bare - jid_file = jid_bare + values = payload['values'] + form = self['xep_0004'].make_form('form', 'Updates') + if is_operator(self, jid_bare) and 'jid' in values: + jid = values['jid'] + jid_file = jid + form.add_field(var='jid', + ftype='hidden', + value=jid) + else: + jid_file = jid_bare db_file = config.get_pathname_to_database(jid_file) num = 100 - match payload['values']['action']: + match values['action']: case 'all': results = sqlite.get_entries(db_file, num) subtitle = 'Recent {} updates'.format(num) @@ -1374,13 +1406,12 @@ class Slixfeed(slixmpp.ClientXMPP): subtitle = 'Recent {} updates (unread)'.format(num) message = 'There are no unread news.' if results: - form = self['xep_0004'].make_form('form', 'Updates') form['instructions'] = subtitle - options = form.add_field(var='update', + options = form.add_field(desc='Select a news item to read.', ftype='list-single', label='News', - desc=('Select a news item to read.'), - required=True) + required=True, + var='update') for result in results: title = result[1] ix = str(result[0]) @@ -1409,18 +1440,25 @@ class Slixfeed(slixmpp.ClientXMPP): values = payload['values'] ix = values['update'] jid_bare = session['from'].bare - jid_file = jid_bare + form = self['xep_0004'].make_form('form', 'Article') + if is_operator(self, jid_bare) and 'jid' in values: + jid = values['jid'] + jid_file = jid[0] if isinstance(jid, list) else jid + form.add_field(var='jid', + ftype='hidden', + value=jid) + else: + jid_file = jid_bare db_file = config.get_pathname_to_database(jid_file) title = sqlite.get_entry_title(db_file, ix) title = title[0] if title else 'Untitled' - form = self['xep_0004'].make_form('form', 'Article') form['instructions'] = title url = sqlite.get_entry_url(db_file, ix) - url = url[0] + url = url[0] # TODO Handle a situation when index is no longer exist logger.debug('Original URL: {}'.format(url)) url = uri.remove_tracking_parameters(url) logger.debug('Processed URL (tracker removal): {}'.format(url)) - url = (uri.replace_hostname(url, 'link')) or url + url = (await uri.replace_hostname(url, 'link')) or url logger.debug('Processed URL (replace hostname): {}'.format(url)) result = await fetch.http(url) if 'content' in result: @@ -1431,29 +1469,29 @@ class Slixfeed(slixmpp.ClientXMPP): form.add_field(ftype="text-multi", label='Article', value=summary) - field_url = form.add_field(var='url', - ftype='hidden', - value=url) - field_url = form.add_field(var='url_link', + field_url = form.add_field(ftype='hidden', + value=url, + var='url') + field_url = form.add_field(ftype='text-single', label='Link', - ftype='text-single', - value=url) + value=url, + var='url_link') field_url['validate']['datatype'] = 'xs:anyURI' feed_id = sqlite.get_feed_id_by_entry_index(db_file, ix) feed_id = feed_id[0] feed_url = sqlite.get_feed_url(db_file, feed_id) feed_url = feed_url[0] - field_feed = form.add_field(var='url_feed', + field_feed = form.add_field(ftype='text-single', label='Source', - ftype='text-single', - value=feed_url) + value=feed_url, + var='url_feed') field_feed['validate']['datatype'] = 'xs:anyURI' - options = form.add_field(var='filetype', + options = form.add_field(desc='Select file type.', ftype='list-single', label='Save as', - desc=('Select file type.'), + required=True, value='pdf', - required=True) + var='filetype') options.addOption('ePUB', 'epub') options.addOption('HTML', 'html') options.addOption('Markdown', 'md') @@ -1476,15 +1514,13 @@ class Slixfeed(slixmpp.ClientXMPP): ext = payload['values']['filetype'] url = payload['values']['url'][0] jid_bare = session['from'].bare - jid_file = jid_bare - db_file = config.get_pathname_to_database(jid_file) cache_dir = config.get_default_cache_directory() if not os.path.isdir(cache_dir): os.mkdir(cache_dir) if not os.path.isdir(cache_dir + '/readability'): os.mkdir(cache_dir + '/readability') url = uri.remove_tracking_parameters(url) - url = (uri.replace_hostname(url, 'link')) or url + url = (await uri.replace_hostname(url, 'link')) or url result = await fetch.http(url) if not result['error']: data = result['content'] @@ -1532,7 +1568,7 @@ class Slixfeed(slixmpp.ClientXMPP): form = self['xep_0004'].make_form('form', 'Subscription') # scan = values['scan'] values = payload['values'] - node = values['node'] if 'node' in values else None + identifier = values['identifier'] if 'identifier' in values else None url = values['subscription'] jid_bare = session['from'].bare if is_operator(self, jid_bare) and 'jid' in values: @@ -1544,33 +1580,34 @@ class Slixfeed(slixmpp.ClientXMPP): else: jid_file = jid_bare db_file = config.get_pathname_to_database(jid_file) - if node and sqlite.check_node_exist(db_file, node): + if identifier and sqlite.check_identifier_exist(db_file, identifier): form['title'] = 'Conflict' form['instructions'] = ('Name "{}" already exists. Choose a ' 'different name.') - form.add_field(desc='Enter a node to publish to.', + form.add_field(desc='Enter a unique identifier. The identifier ' + 'is realized to distinct PubSub nodes.', ftype='text-single', - label='Node', - value=node, - var='node') - form.add_field(var='subscription', - ftype='hidden', - value=url) - form.add_field(var='node', - ftype='hidden', - value=node) + label='identifier', + value=identifier, + var='identifier') + form.add_field(ftype='hidden', + value=url, + var='subscription') + form.add_field(ftype='hidden', + value=identifier, + var='identifier') session['allow_prev'] = False session['next'] = self._handle_subscription_new # session['payload'] = None session['prev'] = None - # elif not node: + # elif not identifier: # counter = 0 # hostname = uri.get_hostname(url) - # node = hostname + ':' + str(counter) + # identifier = hostname + ':' + str(counter) # while True: - # if sqlite.check_node_exist(db_file, node): + # if sqlite.check_identifier_exist(db_file, identifier): # counter += 1 - # node = hostname + ':' + str(counter) + # identifier = hostname + ':' + str(counter) # else: # break # Several URLs to subscribe @@ -1583,14 +1620,15 @@ class Slixfeed(slixmpp.ClientXMPP): for url in urls: counter = 0 hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) while True: - if sqlite.check_node_exist(db_file, node): + if sqlite.check_identifier_exist(db_file, identifier): counter += 1 - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) else: break - result = await action.add_feed(self, jid_bare, db_file, url, node) + result = await action.add_feed(self, jid_bare, db_file, url, + identifier) if result['error']: error_count += 1 elif result['exist']: @@ -1615,24 +1653,25 @@ class Slixfeed(slixmpp.ClientXMPP): url = url[0] counter = 0 hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) while True: - if sqlite.check_node_exist(db_file, node): + if sqlite.check_identifier_exist(db_file, identifier): counter += 1 - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) else: break - result = await action.add_feed(self, jid_bare, db_file, url, node) + result = await action.add_feed(self, jid_bare, db_file, url, + identifier) # URL is not a feed and URL has returned to feeds if isinstance(result, list): results = result form['instructions'] = ('Discovered {} subscriptions for {}' .format(len(results), url)) - options = form.add_field(var='subscription', + options = form.add_field(desc='Select subscriptions to add.', ftype='list-multi', label='Subscribe', - desc='Select subscriptions to add.', - required=True) + required=True, + var='subscription') for result in results: options.addOption(result['name'], result['link']) # NOTE Disabling "allow_prev" until Cheogram would allow to display @@ -1812,11 +1851,11 @@ class Slixfeed(slixmpp.ClientXMPP): if chat_type == 'chat' or moderator: form = self['xep_0004'].make_form('form', 'Discover & Search') form['instructions'] = 'Discover news subscriptions of all kinds' - options = form.add_field(var='search_type', + options = form.add_field(desc='Select type of search.', ftype='list-single', label='Browse', - desc=('Select type of search.'), - required=True) + required=True, + var='search_type') options.addOption('All', 'all') options.addOption('Categories', 'cat') # Should we write this in a singular form # options.addOption('Tags', 'tag') @@ -1846,12 +1885,12 @@ class Slixfeed(slixmpp.ClientXMPP): match search_type: case 'all': form['instructions'] = 'Browsing subscriptions' - options = form.add_field(var='subscription', + options = form.add_field(desc='Select a subscription to add.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to add.'), - required=True) + required=True, + var='subscription') results = sqlite.get_titles_tags_urls(db_file) for result in results: title = result[0] @@ -1864,11 +1903,11 @@ class Slixfeed(slixmpp.ClientXMPP): case 'cat': form['instructions'] = 'Browsing categories' session['next'] = self._handle_discover_category - options = form.add_field(var='category', + options = form.add_field(desc='Select a category to browse.', ftype='list-single', label='Categories', - desc=('Select a category to browse.'), - required=True) # NOTE Uncategories or no option for entries without category + required=True, + var='category') # NOTE Uncategories or no option for entries without category categories = sqlite.get_categories(db_file) for category in categories: category = category[0] @@ -1899,12 +1938,12 @@ class Slixfeed(slixmpp.ClientXMPP): db_file = config_dir + '/feeds.sqlite' form = self['xep_0004'].make_form('form', 'Discover & Search') form['instructions'] = 'Browsing category "{}"'.format(category) - options = form.add_field(var='subscription', + options = form.add_field(desc='Select a subscription to add.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to add.'), - required=True) + required=True, + var='subscription') results = sqlite.get_titles_tags_urls_by_category(db_file, category) for result in results: title = result[0] @@ -1931,12 +1970,12 @@ class Slixfeed(slixmpp.ClientXMPP): if chat_type == 'chat' or moderator: form = self['xep_0004'].make_form('form', 'Subscriptions') form['instructions'] = 'Managing subscriptions' - options = form.add_field(var='action', + options = form.add_field(desc='Select action type.', ftype='list-single', label='Action', - desc='Select action type.', required=True, - value='browse') + value='browse', + var='action') options.addOption('Browse subscriptions', 'browse') options.addOption('Browse tags', 'tag') options.addOption('Remove subscriptions', 'delete') @@ -1944,12 +1983,24 @@ class Slixfeed(slixmpp.ClientXMPP): if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to add the ' - 'subscription to (The default Jabber ID ' - 'is your own).'), - ftype='text-single', - label='Jabber ID', - var='jid') + options = form.add_field(desc='Select a Jabber ID (The ' + 'default Jabber ID is your own).', + ftype='list-single', + label='Jabber ID', + value=jid_bare, + var='jid') + jids = [] + contacts = await XmppRoster.get_contacts(self) + for contact in contacts: + jids.extend([contact]) + conferences = await XmppBookmark.get_bookmarks(self) + for conference in conferences: + jids.extend([conference['jid']]) + pubsubs = await XmppPubsub.get_pubsub_services(self) + for pubsub in pubsubs: + jids.extend([pubsub['jid']]) + for jid_bare in sorted(jids): + options.addOption(jid_bare, jid_bare) session['payload'] = form session['next'] = self._handle_subscriptions_result session['has_next'] = True @@ -1980,12 +2031,12 @@ class Slixfeed(slixmpp.ClientXMPP): match values['action']: case 'browse': form['instructions'] = 'Editing subscriptions' - options = form.add_field(var='subscriptions', + options = form.add_field(desc='Select a subscription to edit.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to edit.'), - required=True) + required=True, + var='subscriptions') subscriptions = sqlite.get_feeds(db_file) # subscriptions = sorted(subscriptions, key=lambda x: x[1]) for subscription in subscriptions: @@ -2000,11 +2051,11 @@ class Slixfeed(slixmpp.ClientXMPP): # form.addField(var='interval', # ftype='text-single', # label='Interval period') - options = form.add_field(var='subscriptions', + options = form.add_field(desc='Select subscriptions to remove.', ftype='list-multi', label='Subscriptions', - desc=('Select subscriptions to remove.'), - required=True) + required=True, + var='subscriptions') subscriptions = sqlite.get_feeds(db_file) # subscriptions = sorted(subscriptions, key=lambda x: x[1]) for subscription in subscriptions: @@ -2026,9 +2077,9 @@ class Slixfeed(slixmpp.ClientXMPP): url = subscription[2] enabled_state = True if subscription[3] else False enabled_state = subscription[3] - form.add_field(ftype='boolean', + form.add_field(desc=url, + ftype='boolean', label=title, - desc=url, value=enabled_state, var=ix) session['cancel'] = self._handle_cancel @@ -2037,11 +2088,11 @@ class Slixfeed(slixmpp.ClientXMPP): session['allow_complete'] = True case 'tag': form['instructions'] = 'Browsing tags' - options = form.add_field(var='tag', + options = form.add_field(desc='Select a tag to browse.', ftype='list-single', label='Tag', - desc=('Select a tag to browse.'), - required=True) + required=True, + var='tag') tags = sqlite.get_tags(db_file) # tags = sorted(tags, key=lambda x: x[0]) for tag in tags: @@ -2077,12 +2128,12 @@ class Slixfeed(slixmpp.ClientXMPP): tag_id = values['tag'] tag_name = sqlite.get_tag_name(db_file, tag_id)[0] form['instructions'] = 'Subscriptions tagged with "{}"'.format(tag_name) - options = form.add_field(var='subscriptions', + options = form.add_field(desc='Select a subscription to edit.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to edit.'), - required=True) + required=True, + var='subscriptions') subscriptions = sqlite.get_feeds_by_tag_id(db_file, tag_id) # subscriptions = sorted(subscriptions, key=lambda x: x[1]) for subscription in subscriptions: @@ -2165,20 +2216,20 @@ class Slixfeed(slixmpp.ClientXMPP): label='ID #', value=feed_id_str) options.addOption(feed_id_str, feed_id_str) - form.add_field(var='tags_new', + form.add_field(desc='Comma-separated tags.', ftype='text-single', label='Tags', - desc='Comma-separated tags.', - value=tags) - form.add_field(var='tags_old', - ftype='hidden', - value=tags) + value=tags, + var='tags_new') + form.add_field(ftype='hidden', + value=tags, + var='tags_old') form.add_field(ftype='fixed', label='Options') - options = form.add_field(var='priority', - ftype='list-single', + options = form.add_field(ftype='list-single', label='Priority', - value='0') + value='0', + var='priority') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 5 } i = 0 @@ -2186,10 +2237,10 @@ class Slixfeed(slixmpp.ClientXMPP): num = str(i) options.addOption(num, num) i += 1 - form.add_field(var='enabled', - ftype='boolean', + form.add_field(ftype='boolean', label='Enabled', - value=True) + value=True, + var='enabled') session['allow_complete'] = True # session['allow_prev'] = True session['cancel'] = self._handle_cancel @@ -2335,12 +2386,12 @@ class Slixfeed(slixmpp.ClientXMPP): else: form = self['xep_0004'].make_form('form', 'Admin Panel') form['instructions'] = 'Administration actions' - options = form.add_field(var='action', + options = form.add_field(desc='Select action type.', ftype='list-single', label='Manage', - desc='Select action type.', + required=True, value='subscribers', - required=True) + var='action') options.addOption('Bookmarks', 'bookmarks') options.addOption('Contacts', 'roster') options.addOption('PubSub', 'pubsub') @@ -2373,18 +2424,18 @@ class Slixfeed(slixmpp.ClientXMPP): case 'import': form = self['xep_0004'].make_form('form', 'Import') form['instructions'] = 'Importing feeds' - url = form.add_field(var='url', + url = form.add_field(desc='Enter URL to an OPML file.', ftype='text-single', label='URL', - desc='Enter URL to an OPML file.', - required=True) + required=True, + var='url') url['validate']['datatype'] = 'xs:anyURI' if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to import ' - 'subscriptions to (The default ' - 'Jabber ID is your own).'), + form.add_field(desc='Enter a Jabber ID to import ' + 'subscriptions to (The default Jabber ID ' + 'is your own).', ftype='text-single', label='Jabber ID', var='jid') @@ -2401,12 +2452,12 @@ class Slixfeed(slixmpp.ClientXMPP): 'About -> Software for a list of ' 'News Readers offered for desktop and ' 'mobile devices.') - options = form.add_field(var='filetype', + options = form.add_field(desc='Choose export format.', ftype='list-multi', label='Format', - desc='Choose export format.', + required=True, value='opml', - required=True) + var='filetype') options.addOption('Markdown', 'md') options.addOption('OPML', 'opml') # options.addOption('HTML', 'html') @@ -2414,12 +2465,26 @@ class Slixfeed(slixmpp.ClientXMPP): if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to export ' - 'subscriptions from (The default ' - 'Jabber ID is your own).'), - ftype='text-single', - label='Jabber ID', - var='jid') + options = form.add_field(desc='Select a Jabber ID to ' + 'export subscriptions from (The ' + 'default Jabber ID is your own).', + ftype='list-single', + label='Jabber ID', + value=jid_bare, + var='jid') + # options.addOption(self.boundjid.bare, self.boundjid.bare) + jids = [] + contacts = await XmppRoster.get_contacts(self) + for contact in contacts: + jids.extend([contact]) + conferences = await XmppBookmark.get_bookmarks(self) + for conference in conferences: + jids.extend([conference['jid']]) + pubsubs = await XmppPubsub.get_pubsub_services(self) + for pubsub in pubsubs: + jids.extend([pubsub['jid']]) + for jid_bare in sorted(jids): + options.addOption(jid_bare, jid_bare) session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_export_complete @@ -2664,17 +2729,17 @@ class Slixfeed(slixmpp.ClientXMPP): # NOTE Refresh button would be of use form['instructions'] = 'Featured subscriptions' url = action.pick_a_feed() - # options = form.add_field(var='choice', + # options = form.add_field(desc='Click to subscribe.', # ftype="boolean", # label='Subscribe to {}?'.format(url['name']), - # desc='Click to subscribe.') + # var='choice') # form.add_field(var='subscription', # ftype='hidden', # value=url['link']) - options = form.add_field(var='subscription', - ftype="list-single", - label='Subscribe', - desc='Click to subscribe.') + options = form.add_field(desc='Click to subscribe.', + ftype="list-single", + label='Subscribe', + var='subscription') for i in range(10): url = action.pick_a_feed() options.addOption(url['name'], url['link']) @@ -2725,12 +2790,12 @@ class Slixfeed(slixmpp.ClientXMPP): case 'bookmarks': form = self['xep_0004'].make_form('form', 'Bookmarks') form['instructions'] = 'Managing bookmarks' - options = form.add_field(var='jid', + options = form.add_field(desc='Select a bookmark to edit.', ftype='list-single', label='Jabber ID', - desc='Select a bookmark to edit.', - required=True) - conferences = await XmppBookmark.get(self) + required=True, + var='jid') + conferences = await XmppBookmark.get_bookmarks(self) for conference in conferences: options.addOption(conference['name'], conference['jid']) session['has_next'] = True @@ -2738,20 +2803,20 @@ class Slixfeed(slixmpp.ClientXMPP): case 'roster': form = self['xep_0004'].make_form('form', 'Contacts') form['instructions'] = 'Organizing contacts' - options = form.add_field(var='jid', + options = form.add_field(desc='Select a contact.', ftype='list-single', label='Contact', - desc='Select a contact.', - required=True) + required=True, + var='jid') contacts = await XmppRoster.get_contacts(self) for contact in contacts: contact_name = contacts[contact]['name'] contact_name = contact_name if contact_name else contact options.addOption(contact_name, contact) - options = form.add_field(var='action', - ftype='list-single', + options = form.add_field(ftype='list-single', label='Action', - required=True) + required=True, + var='action') options.addOption('Display', 'view') options.addOption('Edit', 'edit') session['has_next'] = True @@ -2759,20 +2824,20 @@ class Slixfeed(slixmpp.ClientXMPP): case 'subscribers': form = self['xep_0004'].make_form('form', 'Subscribers') form['instructions'] = 'Committing subscriber action' - options = form.add_field(var='action', - ftype='list-single', + options = form.add_field(ftype='list-single', label='Action', + required=True, value='message', - required=True) + var='action') options.addOption('Request authorization From', 'from') options.addOption('Resend authorization To', 'to') options.addOption('Send message', 'message') options.addOption('Remove', 'remove') - options = form.add_field(var='jid', + options = form.add_field(desc='Select a contact.', ftype='list-single', label='Jabber ID', - desc='Select a contact.', - required=True) + required=True, + var='jid') contacts = await XmppRoster.get_contacts(self) for contact in contacts: contact_name = contacts[contact]['name'] @@ -2781,10 +2846,10 @@ class Slixfeed(slixmpp.ClientXMPP): form.add_field(var='subject', ftype='text-single', label='Subject') - form.add_field(var='message', + form.add_field(desc='Add a descriptive message.', ftype='text-multi', label='Message', - desc='Add a descriptive message.') + var='message') session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_subscribers_complete @@ -2799,28 +2864,19 @@ class Slixfeed(slixmpp.ClientXMPP): label='Jabber ID') # jid_bare = self.boundjid.bare # enabled_state = Config.get_setting_value(self.settings, jid_bare, 'enabled') - # form.add_field(ftype='boolean', - # label=jid_bare, - # value=enabled_state, - # var=jid_bare) - iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) - items = iq['disco_items']['items'] - for item in items: - iq = await self['xep_0030'].get_info(jid=item[0]) - identities = iq['disco_info']['identities'] - for identity in identities: - if identity[0] == 'pubsub' and identity[1] == 'service': - jid_bare = item[0] - if item[1]: name = item[1] - elif item[2]: name = item[2] - else: name = jid_bare - enabled_state = Config.get_setting_value(self.settings, jid_bare, 'enabled') - form.add_field(desc=jid_bare, - ftype='boolean', - label=name, - value=enabled_state, - var=jid_bare) - print(Config.get_setting_value(self.settings, jid_bare, 'enabled')) + + results = await XmppPubsub.get_pubsub_services(self) + for result in results + [{'jid' : self.boundjid.bare, + 'name' : self.alias}]: + jid_bare = result['jid'] + name = result['name'] + enabled_state = Config.get_setting_value( + self.settings, jid_bare, 'enabled') + form.add_field(desc=jid_bare, + ftype='boolean', + label=name, + value=enabled_state, + var=jid_bare) session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_pubsubs_complete @@ -2845,7 +2901,8 @@ class Slixfeed(slixmpp.ClientXMPP): db_file = config.get_pathname_to_database(jid_file) if jid_bare not in self.settings: Config.add_settings_jid(self.settings, jid_bare, db_file) - await Config.set_setting_value(self.settings, jid_bare, db_file, 'enabled', value) + await Config.set_setting_value(self.settings, jid_bare, + db_file, 'enabled', value) print(self.settings) text_note = 'Done.' session['has_next'] = False @@ -3118,11 +3175,11 @@ class Slixfeed(slixmpp.ClientXMPP): value = True else: value = False - form.add_field(var='enabled', + form.add_field(desc='Enable news updates.', ftype='boolean', label='Enabled', - desc='Enable news updates.', - value=value) + value=value, + var='enabled') value = Config.get_setting_value(self.settings, jid_bare, 'media') value = str(value) value = int(value) @@ -3130,11 +3187,11 @@ class Slixfeed(slixmpp.ClientXMPP): value = True else: value = False - form.add_field(var='media', + form.add_field(desc='Send audio, images or videos if found.', ftype='boolean', - desc='Send audio, images or videos if found.', label='Display media', - value=value) + value=value, + var='media') value = Config.get_setting_value(self.settings, jid_bare, 'old') value = str(value) value = int(value) @@ -3142,23 +3199,24 @@ class Slixfeed(slixmpp.ClientXMPP): value = True else: value = False - form.add_field(var='old', + form.add_field(desc='Treat all items of newly added subscriptions ' + 'as new.', ftype='boolean', - desc='Treat all items of newly added subscriptions as new.', # label='Send only new items', label='Include old news', - value=value) + value=value, + var='old') value = Config.get_setting_value(self.settings, jid_bare, 'interval') value = str(value) value = int(value) value = value/60 value = int(value) value = str(value) - options = form.add_field(var='interval', + options = form.add_field(desc='Interval update (in hours).', ftype='list-single', label='Interval', - desc='Interval update (in hours).', - value=value) + value=value, + var='interval') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 48 } i = 1 @@ -3171,11 +3229,11 @@ class Slixfeed(slixmpp.ClientXMPP): i += 1 value = Config.get_setting_value(self.settings, jid_bare, 'quantum') value = str(value) - options = form.add_field(var='quantum', + options = form.add_field(desc='Amount of items per update.', ftype='list-single', label='Amount', - desc='Amount of items per update.', - value=value) + value=value, + var='quantum') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 5 } i = 1 @@ -3185,11 +3243,11 @@ class Slixfeed(slixmpp.ClientXMPP): i += 1 value = Config.get_setting_value(self.settings, jid_bare, 'archive') value = str(value) - options = form.add_field(var='archive', + options = form.add_field(desc='Number of news items to archive.', ftype='list-single', label='Archive', - desc='Number of news items to archive.', - value=value) + value=value, + var='archive') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 0, 'maximum': 500 } i = 0 diff --git a/slixfeed/xmpp/component.py b/slixfeed/xmpp/component.py index 9e5bb6d..34534b8 100644 --- a/slixfeed/xmpp/component.py +++ b/slixfeed/xmpp/component.py @@ -709,9 +709,9 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): jid_bare = session['from'].bare if is_operator(self, jid_bare): form = self['xep_0004'].make_form('form', 'PubSub') - form['instructions'] = 'Manage nodes and publish news into nodes.' - options = form.add_field(desc=('Send a set of selected posts or ' - 'set a new subscription.'), + form['instructions'] = 'Manage PubSub nodes and publish news items.' + options = form.add_field(desc='Send a set of selected posts or ' + 'set a new subscription.', ftype='list-single', label='Choose', required=True, @@ -742,24 +742,24 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form['instructions'] = ('Choose a PubSub Jabber ID and verify ' 'that Slixfeed has the necessary ' 'permissions to publish into it.') - form.add_field(var='url', + form.add_field(desc='Enter a subscription URL.', # TODO Make it possible to add several subscriptions at once; # Similarly to BitTorrent trackers list # ftype='text-multi', # label='Subscription URLs', - # desc=('Add subscriptions one time per ' - # 'subscription.'), + # desc='Add subscriptions one time per ' + # 'subscription.', ftype='text-single', label='URL', - desc='Enter a subscription URL.', + required=True, value='http://', - required=True) - options = form.add_field(var='jid', - ftype='list-single', - label='PubSub', - desc='Select a PubSub Service.', - value=self.boundjid.bare, - required=True) + var='url') + options = form.add_field(desc='Select a PubSub Service.', + ftype='list-single', + label='PubSub', + required=True, + value=self.boundjid.bare, + var='jid') options.addOption(self.boundjid.bare, self.boundjid.bare) iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) items = iq['disco_items']['items'] @@ -773,23 +773,23 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): elif item[2]: name = item[2] else: name = jid options.addOption(jid, name) - # form.add_field(var='jid', + # form.add_field(desc='Enter a PubSub Jabber ID.', # ftype='text-single', # label='PubSub', - # desc='Enter a PubSub Jabber ID.', + # required=True, # value=self.boundjid.bare, # # value='pubsub.' + self.boundjid.host, - # required=True) - form.add_field(var='node', + # var='jid') + form.add_field(desc='Enter a node to publish to.', ftype='text-single', label='Node', - desc='Enter a node to publish to.') - # options = form.add_field(var='xep', + var='node') + # options = form.add_field(desc='Select XMPP Extension Protocol.', # ftype='list-single', # label='Protocol', - # desc='Select XMPP Extension Protocol.', + # required=True, # value='0060', - # required=True) + # var='xep') # options.addOption('XEP-0060: Publish-Subscribe', '0060') # options.addOption('XEP-0277: Microblogging over XMPP', '0277') # options.addOption('XEP-0472: Pubsub Social Feed', '0472') @@ -800,18 +800,18 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form['instructions'] = ('Choose a PubSub Jabber ID and verify ' 'that Slixfeed has the necessary ' 'permissions to publish into it.') - form.add_field(var='url', + form.add_field(desc='Enter a subscription URL.', ftype='text-single', label='URL', - desc='Enter a subscription URL.', + required=True, value='http://', - required=True) - options = form.add_field(var='jid', - ftype='list-single', - label='PubSub', - desc='Select a PubSub Service.', - value=self.boundjid.bare, - required=True) + var='url') + options = form.add_field(desc='Select a PubSub Service.', + ftype='list-single', + label='PubSub', + required=True, + value=self.boundjid.bare, + var='jid') options.addOption(self.boundjid.bare, self.boundjid.bare) iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) items = iq['disco_items']['items'] @@ -825,23 +825,23 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): elif item[2]: name = item[2] else: name = jid options.addOption(jid, name) - # form.add_field(var='jid', + # form.add_field(desc='Enter a PubSub Jabber ID.', # ftype='text-single', # label='PubSub', - # desc='Enter a PubSub Jabber ID.', - # value=self.boundjid.bare, + # required=True, # # value='pubsub.' + self.boundjid.host, - # required=True) - form.add_field(var='node', + # value=self.boundjid.bare, + # var='jid') + form.add_field(desc='Enter a node to publish to.', ftype='text-single', label='Node', - desc='Enter a node to publish to.') - # options = form.add_field(var='xep', + var='node') + # options = form.add_field(desc='Select XMPP Extension Protocol.', # ftype='list-single', # label='Protocol', - # desc='Select XMPP Extension Protocol.', + # required=True, # value='0060', - # required=True) + # var='xep') # options.addOption('XEP-0060: Publish-Subscribe', '0060') # options.addOption('XEP-0277: Microblogging over XMPP', '0277') # options.addOption('XEP-0472: Pubsub Social Feed', '0472') @@ -888,11 +888,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): # if is_feed(url, feed): if action.is_feed(feed): form['instructions'] = 'Select entries to publish.' - options = form.add_field(var='entries', + options = form.add_field(desc='Select entries to post.', ftype='list-multi', label='Titles', - desc='Select entries to post.', - required=True) + required=True, + var='entries') if "title" in feed["feed"].keys(): title = feed["feed"]["title"] else: @@ -929,11 +929,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form['instructions'] = ('Discovered {} subscriptions ' 'for {}' .format(len(results), url)) - options = form.add_field(var='url', + options = form.add_field(desc='Select a feed.', ftype='list-single', label='Feeds', - desc='Select a feed.', - required=True) + required=True, + var='url') for result in results: title = result['name'] url = result['link'] @@ -1164,18 +1164,18 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form['instructions'] = 'Editing filters' # 🪄️ 🛡️ value = sqlite.get_filter_value(db_file, 'allow') if value: value = str(value[0]) - form.add_field(var='allow', + form.add_field(desc='Keywords to allow (comma-separated keywords).', ftype='text-single', label='Allow list', value=value, - desc='Keywords to allow (comma-separated keywords).') + var='allow') value = sqlite.get_filter_value(db_file, 'deny') if value: value = str(value[0]) - form.add_field(var='deny', + form.add_field(desc='Keywords to deny (comma-separated keywords).', ftype='text-single', label='Deny list', value=value, - desc='Keywords to deny (comma-separated keywords).') + var='deny') session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_filters_complete @@ -1250,32 +1250,32 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): if chat_type == 'chat' or moderator: form = self['xep_0004'].make_form('form', 'Subscription') form['instructions'] = 'Adding subscription' - form.add_field(var='subscription', + form.add_field(desc='Enter a subscription URL.', # TODO Make it possible to add several subscriptions at once; # Similarly to BitTorrent trackers list # ftype='text-multi', # label='Subscription URLs', - # desc=('Add subscriptions one time per ' - # 'subscription.'), + # desc='Add subscriptions one time per ' + # 'subscription.', ftype='text-single', label='URL', - desc='Enter a subscription URL.', + required=True, value='http://', - required=True) + var='subscription') if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to add the ' - 'subscription to (The default Jabber ID ' - 'is your own).'), + form.add_field(desc='Enter a Jabber ID to add the ' + 'subscription to (The default Jabber ID is ' + 'your own).', ftype='text-single', label='Jabber ID', var='jid') - # form.add_field(var='scan', + # form.add_field(desc='Scan URL for validity (recommended).', # ftype='boolean', # label='Scan', - # desc='Scan URL for validity (recommended).', - # value=True) + # value=True, + # var='scan') session['allow_prev'] = False session['has_next'] = True session['next'] = self._handle_subscription_new @@ -1293,18 +1293,40 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): function_name = sys._getframe().f_code.co_name logger.debug('{}: jid_full: {}' .format(function_name, jid_full)) + jid_bare = session['from'].bare form = self['xep_0004'].make_form('form', 'Updates') form['instructions'] = 'Browse and read news' - options = form.add_field(var='action', + options = form.add_field(desc='What would you want to read?', ftype='list-single', label='Read', - desc=('What would you want to read?'), - required=True) + required=True, + var='action') options.addOption('All news', 'all') # options.addOption('News by subscription', 'feed') # options.addOption('News by tag', 'tag') options.addOption('Rejected news', 'reject') options.addOption('Unread news', 'unread') + if is_operator(self, jid_bare): + form.add_field(ftype='fixed', + label='Subscriber') + options = form.add_field(desc='Select a Jabber ID (The default ' + 'Jabber ID is your own).', + ftype='list-single', + label='Jabber ID', + value=jid_bare, + var='jid') + jids = [] + contacts = await XmppRoster.get_contacts(self) + for contact in contacts: + jids.extend([contact]) + conferences = await XmppBookmark.get_bookmarks(self) + for conference in conferences: + jids.extend([conference['jid']]) + pubsubs = await XmppPubsub.get_pubsub_services(self) + for pubsub in pubsubs: + jids.extend([pubsub['jid']]) + for jid_bare in sorted(jids): + options.addOption(jid_bare, jid_bare) session['allow_prev'] = False # Cheogram changes style if that button - which should not be on this form - is present session['has_next'] = True session['next'] = self._handle_recent_result @@ -1319,10 +1341,19 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): logger.debug('{}: jid_full: {}' .format(function_name, jid_full)) jid_bare = session['from'].bare - jid_file = jid_bare + values = payload['values'] + form = self['xep_0004'].make_form('form', 'Updates') + if is_operator(self, jid_bare) and 'jid' in values: + jid = values['jid'] + jid_file = jid + form.add_field(var='jid', + ftype='hidden', + value=jid) + else: + jid_file = jid_bare db_file = config.get_pathname_to_database(jid_file) num = 100 - match payload['values']['action']: + match values['action']: case 'all': results = sqlite.get_entries(db_file, num) subtitle = 'Recent {} updates'.format(num) @@ -1336,13 +1367,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): subtitle = 'Recent {} updates (unread)'.format(num) message = 'There are no unread news.' if results: - form = self['xep_0004'].make_form('form', 'Updates') form['instructions'] = subtitle - options = form.add_field(var='update', + options = form.add_field(desc='Select a news item to read.', ftype='list-single', label='News', - desc=('Select a news item to read.'), - required=True) + required=True, + var='update') for result in results: title = result[1] ix = str(result[0]) @@ -1371,18 +1401,25 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): values = payload['values'] ix = values['update'] jid_bare = session['from'].bare - jid_file = jid_bare + form = self['xep_0004'].make_form('form', 'Article') + if is_operator(self, jid_bare) and 'jid' in values: + jid = values['jid'] + jid_file = jid[0] if isinstance(jid, list) else jid + form.add_field(var='jid', + ftype='hidden', + value=jid) + else: + jid_file = jid_bare db_file = config.get_pathname_to_database(jid_file) title = sqlite.get_entry_title(db_file, ix) title = title[0] if title else 'Untitled' - form = self['xep_0004'].make_form('form', 'Article') form['instructions'] = title url = sqlite.get_entry_url(db_file, ix) - url = url[0] + url = url[0] # TODO Handle a situation when index is no longer exist logger.debug('Original URL: {}'.format(url)) url = uri.remove_tracking_parameters(url) logger.debug('Processed URL (tracker removal): {}'.format(url)) - url = (uri.replace_hostname(url, 'link')) or url + url = (await uri.replace_hostname(url, 'link')) or url logger.debug('Processed URL (replace hostname): {}'.format(url)) result = await fetch.http(url) if 'content' in result: @@ -1393,29 +1430,29 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form.add_field(ftype="text-multi", label='Article', value=summary) - field_url = form.add_field(var='url', - ftype='hidden', - value=url) - field_url = form.add_field(var='url_link', + field_url = form.add_field(ftype='hidden', + value=url, + var='url') + field_url = form.add_field(ftype='text-single', label='Link', - ftype='text-single', - value=url) + value=url, + var='url_link') field_url['validate']['datatype'] = 'xs:anyURI' feed_id = sqlite.get_feed_id_by_entry_index(db_file, ix) feed_id = feed_id[0] feed_url = sqlite.get_feed_url(db_file, feed_id) feed_url = feed_url[0] - field_feed = form.add_field(var='url_feed', + field_feed = form.add_field(ftype='text-single', label='Source', - ftype='text-single', - value=feed_url) + value=feed_url, + var='url_feed') field_feed['validate']['datatype'] = 'xs:anyURI' - options = form.add_field(var='filetype', + options = form.add_field(desc='Select file type.', ftype='list-single', label='Save as', - desc=('Select file type.'), + required=True, value='pdf', - required=True) + var='filetype') options.addOption('ePUB', 'epub') options.addOption('HTML', 'html') options.addOption('Markdown', 'md') @@ -1438,15 +1475,13 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): ext = payload['values']['filetype'] url = payload['values']['url'][0] jid_bare = session['from'].bare - jid_file = jid_bare - db_file = config.get_pathname_to_database(jid_file) cache_dir = config.get_default_cache_directory() if not os.path.isdir(cache_dir): os.mkdir(cache_dir) if not os.path.isdir(cache_dir + '/readability'): os.mkdir(cache_dir + '/readability') url = uri.remove_tracking_parameters(url) - url = (uri.replace_hostname(url, 'link')) or url + url = (await uri.replace_hostname(url, 'link')) or url result = await fetch.http(url) if not result['error']: data = result['content'] @@ -1494,7 +1529,7 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form = self['xep_0004'].make_form('form', 'Subscription') # scan = values['scan'] values = payload['values'] - node = values['node'] if 'node' in values else None + identifier = values['identifier'] if 'identifier' in values else None url = values['subscription'] jid_bare = session['from'].bare if is_operator(self, jid_bare) and 'jid' in values: @@ -1506,33 +1541,34 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): else: jid_file = jid_bare db_file = config.get_pathname_to_database(jid_file) - if node and sqlite.check_node_exist(db_file, node): + if identifier and sqlite.check_identifier_exist(db_file, identifier): form['title'] = 'Conflict' form['instructions'] = ('Name "{}" already exists. Choose a ' 'different name.') - form.add_field(desc='Enter a node to publish to.', + form.add_field(desc='Enter a unique identifier. The identifier ' + 'is realized to distinct PubSub nodes.', ftype='text-single', - label='Node', - value=node, - var='node') - form.add_field(var='subscription', - ftype='hidden', - value=url) - form.add_field(var='node', - ftype='hidden', - value=node) + label='identifier', + value=identifier, + var='identifier') + form.add_field(ftype='hidden', + value=url, + var='subscription') + form.add_field(ftype='hidden', + value=identifier, + var='identifier') session['allow_prev'] = False session['next'] = self._handle_subscription_new # session['payload'] = None session['prev'] = None - # elif not node: + # elif not identifier: # counter = 0 # hostname = uri.get_hostname(url) - # node = hostname + ':' + str(counter) + # identifier = hostname + ':' + str(counter) # while True: - # if sqlite.check_node_exist(db_file, node): + # if sqlite.check_identifier_exist(db_file, identifier): # counter += 1 - # node = hostname + ':' + str(counter) + # identifier = hostname + ':' + str(counter) # else: # break # Several URLs to subscribe @@ -1545,14 +1581,15 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): for url in urls: counter = 0 hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) while True: - if sqlite.check_node_exist(db_file, node): + if sqlite.check_identifier_exist(db_file, identifier): counter += 1 - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) else: break - result = await action.add_feed(self, jid_bare, db_file, url, node) + result = await action.add_feed(self, jid_bare, db_file, url, + identifier) if result['error']: error_count += 1 elif result['exist']: @@ -1577,24 +1614,25 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): url = url[0] counter = 0 hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) while True: - if sqlite.check_node_exist(db_file, node): + if sqlite.check_identifier_exist(db_file, identifier): counter += 1 - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) else: break - result = await action.add_feed(self, jid_bare, db_file, url, node) + result = await action.add_feed(self, jid_bare, db_file, url, + identifier) # URL is not a feed and URL has returned to feeds if isinstance(result, list): results = result form['instructions'] = ('Discovered {} subscriptions for {}' .format(len(results), url)) - options = form.add_field(var='subscription', + options = form.add_field(desc='Select subscriptions to add.', ftype='list-multi', label='Subscribe', - desc='Select subscriptions to add.', - required=True) + required=True, + var='subscription') for result in results: options.addOption(result['name'], result['link']) # NOTE Disabling "allow_prev" until Cheogram would allow to display @@ -1774,11 +1812,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): if chat_type == 'chat' or moderator: form = self['xep_0004'].make_form('form', 'Discover & Search') form['instructions'] = 'Discover news subscriptions of all kinds' - options = form.add_field(var='search_type', + options = form.add_field(desc='Select type of search.', ftype='list-single', label='Browse', - desc=('Select type of search.'), - required=True) + required=True, + var='search_type') options.addOption('All', 'all') options.addOption('Categories', 'cat') # Should we write this in a singular form # options.addOption('Tags', 'tag') @@ -1808,12 +1846,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): match search_type: case 'all': form['instructions'] = 'Browsing subscriptions' - options = form.add_field(var='subscription', + options = form.add_field(desc='Select a subscription to add.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to add.'), - required=True) + required=True, + var='subscription') results = sqlite.get_titles_tags_urls(db_file) for result in results: title = result[0] @@ -1826,11 +1864,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): case 'cat': form['instructions'] = 'Browsing categories' session['next'] = self._handle_discover_category - options = form.add_field(var='category', + options = form.add_field(desc='Select a category to browse.', ftype='list-single', label='Categories', - desc=('Select a category to browse.'), - required=True) # NOTE Uncategories or no option for entries without category + required=True, + var='category') # NOTE Uncategories or no option for entries without category categories = sqlite.get_categories(db_file) for category in categories: category = category[0] @@ -1861,12 +1899,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): db_file = config_dir + '/feeds.sqlite' form = self['xep_0004'].make_form('form', 'Discover & Search') form['instructions'] = 'Browsing category "{}"'.format(category) - options = form.add_field(var='subscription', + options = form.add_field(desc='Select a subscription to add.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to add.'), - required=True) + required=True, + var='subscription') results = sqlite.get_titles_tags_urls_by_category(db_file, category) for result in results: title = result[0] @@ -1893,12 +1931,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): if chat_type == 'chat' or moderator: form = self['xep_0004'].make_form('form', 'Subscriptions') form['instructions'] = 'Managing subscriptions' - options = form.add_field(var='action', + options = form.add_field(desc='Select action type.', ftype='list-single', label='Action', - desc='Select action type.', required=True, - value='browse') + value='browse', + var='action') options.addOption('Browse subscriptions', 'browse') options.addOption('Browse tags', 'tag') options.addOption('Remove subscriptions', 'delete') @@ -1906,12 +1944,24 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to add the ' - 'subscription to (The default Jabber ID ' - 'is your own).'), - ftype='text-single', - label='Jabber ID', - var='jid') + options = form.add_field(desc='Select a Jabber ID (The ' + 'default Jabber ID is your own).', + ftype='list-single', + label='Jabber ID', + value=jid_bare, + var='jid') + jids = [] + contacts = await XmppRoster.get_contacts(self) + for contact in contacts: + jids.extend([contact]) + conferences = await XmppBookmark.get_bookmarks(self) + for conference in conferences: + jids.extend([conference['jid']]) + pubsubs = await XmppPubsub.get_pubsub_services(self) + for pubsub in pubsubs: + jids.extend([pubsub['jid']]) + for jid_bare in sorted(jids): + options.addOption(jid_bare, jid_bare) session['payload'] = form session['next'] = self._handle_subscriptions_result session['has_next'] = True @@ -1942,12 +1992,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): match values['action']: case 'browse': form['instructions'] = 'Editing subscriptions' - options = form.add_field(var='subscriptions', + options = form.add_field(desc='Select a subscription to edit.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to edit.'), - required=True) + required=True, + var='subscriptions') subscriptions = sqlite.get_feeds(db_file) # subscriptions = sorted(subscriptions, key=lambda x: x[1]) for subscription in subscriptions: @@ -1962,11 +2012,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): # form.addField(var='interval', # ftype='text-single', # label='Interval period') - options = form.add_field(var='subscriptions', + options = form.add_field(desc='Select subscriptions to remove.', ftype='list-multi', label='Subscriptions', - desc=('Select subscriptions to remove.'), - required=True) + required=True, + var='subscriptions') subscriptions = sqlite.get_feeds(db_file) # subscriptions = sorted(subscriptions, key=lambda x: x[1]) for subscription in subscriptions: @@ -1988,9 +2038,9 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): url = subscription[2] enabled_state = True if subscription[3] else False enabled_state = subscription[3] - form.add_field(ftype='boolean', + form.add_field(desc=url, + ftype='boolean', label=title, - desc=url, value=enabled_state, var=ix) session['cancel'] = self._handle_cancel @@ -1999,11 +2049,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): session['allow_complete'] = True case 'tag': form['instructions'] = 'Browsing tags' - options = form.add_field(var='tag', + options = form.add_field(desc='Select a tag to browse.', ftype='list-single', label='Tag', - desc=('Select a tag to browse.'), - required=True) + required=True, + var='tag') tags = sqlite.get_tags(db_file) # tags = sorted(tags, key=lambda x: x[0]) for tag in tags: @@ -2039,12 +2089,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): tag_id = values['tag'] tag_name = sqlite.get_tag_name(db_file, tag_id)[0] form['instructions'] = 'Subscriptions tagged with "{}"'.format(tag_name) - options = form.add_field(var='subscriptions', + options = form.add_field(desc='Select a subscription to edit.', # ftype='list-multi', # TODO To be added soon ftype='list-single', label='Subscription', - desc=('Select a subscription to edit.'), - required=True) + required=True, + var='subscriptions') subscriptions = sqlite.get_feeds_by_tag_id(db_file, tag_id) # subscriptions = sorted(subscriptions, key=lambda x: x[1]) for subscription in subscriptions: @@ -2127,20 +2177,20 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): label='ID #', value=feed_id_str) options.addOption(feed_id_str, feed_id_str) - form.add_field(var='tags_new', + form.add_field(desc='Comma-separated tags.', ftype='text-single', label='Tags', - desc='Comma-separated tags.', - value=tags) - form.add_field(var='tags_old', - ftype='hidden', - value=tags) + value=tags, + var='tags_new') + form.add_field(ftype='hidden', + value=tags, + var='tags_old') form.add_field(ftype='fixed', label='Options') - options = form.add_field(var='priority', - ftype='list-single', + options = form.add_field(ftype='list-single', label='Priority', - value='0') + value='0', + var='priority') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 5 } i = 0 @@ -2148,10 +2198,10 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): num = str(i) options.addOption(num, num) i += 1 - form.add_field(var='enabled', - ftype='boolean', + form.add_field(ftype='boolean', label='Enabled', - value=True) + value=True, + var='enabled') session['allow_complete'] = True # session['allow_prev'] = True session['cancel'] = self._handle_cancel @@ -2297,12 +2347,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): else: form = self['xep_0004'].make_form('form', 'Admin Panel') form['instructions'] = 'Administration actions' - options = form.add_field(var='action', + options = form.add_field(desc='Select action type.', ftype='list-single', label='Manage', - desc='Select action type.', + required=True, value='subscribers', - required=True) + var='action') options.addOption('Bookmarks', 'bookmarks') options.addOption('Contacts', 'roster') options.addOption('PubSub', 'pubsub') @@ -2335,18 +2385,18 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): case 'import': form = self['xep_0004'].make_form('form', 'Import') form['instructions'] = 'Importing feeds' - url = form.add_field(var='url', + url = form.add_field(desc='Enter URL to an OPML file.', ftype='text-single', label='URL', - desc='Enter URL to an OPML file.', - required=True) + required=True, + var='url') url['validate']['datatype'] = 'xs:anyURI' if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to import ' - 'subscriptions to (The default ' - 'Jabber ID is your own).'), + form.add_field(desc='Enter a Jabber ID to import ' + 'subscriptions to (The default Jabber ID ' + 'is your own).', ftype='text-single', label='Jabber ID', var='jid') @@ -2363,12 +2413,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): 'About -> Software for a list of ' 'News Readers offered for desktop and ' 'mobile devices.') - options = form.add_field(var='filetype', + options = form.add_field(desc='Choose export format.', ftype='list-multi', label='Format', - desc='Choose export format.', + required=True, value='opml', - required=True) + var='filetype') options.addOption('Markdown', 'md') options.addOption('OPML', 'opml') # options.addOption('HTML', 'html') @@ -2376,12 +2426,26 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): if is_operator(self, jid_bare): form.add_field(ftype='fixed', label='Subscriber') - form.add_field(desc=('Enter a Jabber ID to export ' - 'subscriptions from (The default ' - 'Jabber ID is your own).'), - ftype='text-single', - label='Jabber ID', - var='jid') + options = form.add_field(desc='Select a Jabber ID to ' + 'export subscriptions from (The ' + 'default Jabber ID is your own).', + ftype='list-single', + label='Jabber ID', + value=jid_bare, + var='jid') + # options.addOption(self.boundjid.bare, self.boundjid.bare) + jids = [] + contacts = await XmppRoster.get_contacts(self) + for contact in contacts: + jids.extend([contact]) + conferences = await XmppBookmark.get_bookmarks(self) + for conference in conferences: + jids.extend([conference['jid']]) + pubsubs = await XmppPubsub.get_pubsub_services(self) + for pubsub in pubsubs: + jids.extend([pubsub['jid']]) + for jid_bare in sorted(jids): + options.addOption(jid_bare, jid_bare) session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_export_complete @@ -2626,17 +2690,17 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): # NOTE Refresh button would be of use form['instructions'] = 'Featured subscriptions' url = action.pick_a_feed() - # options = form.add_field(var='choice', + # options = form.add_field(desc='Click to subscribe.', # ftype="boolean", # label='Subscribe to {}?'.format(url['name']), - # desc='Click to subscribe.') + # var='choice') # form.add_field(var='subscription', # ftype='hidden', # value=url['link']) - options = form.add_field(var='subscription', - ftype="list-single", - label='Subscribe', - desc='Click to subscribe.') + options = form.add_field(desc='Click to subscribe.', + ftype="list-single", + label='Subscribe', + var='subscription') for i in range(10): url = action.pick_a_feed() options.addOption(url['name'], url['link']) @@ -2687,12 +2751,12 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): case 'bookmarks': form = self['xep_0004'].make_form('form', 'Bookmarks') form['instructions'] = 'Managing bookmarks' - options = form.add_field(var='jid', + options = form.add_field(desc='Select a bookmark to edit.', ftype='list-single', label='Jabber ID', - desc='Select a bookmark to edit.', - required=True) - conferences = await XmppBookmark.get(self) + required=True, + var='jid') + conferences = await XmppBookmark.get_bookmarks(self) for conference in conferences: options.addOption(conference['name'], conference['jid']) session['has_next'] = True @@ -2700,20 +2764,20 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): case 'roster': form = self['xep_0004'].make_form('form', 'Contacts') form['instructions'] = 'Organizing contacts' - options = form.add_field(var='jid', + options = form.add_field(desc='Select a contact.', ftype='list-single', label='Contact', - desc='Select a contact.', - required=True) + required=True, + var='jid') contacts = await XmppRoster.get_contacts(self) for contact in contacts: contact_name = contacts[contact]['name'] contact_name = contact_name if contact_name else contact options.addOption(contact_name, contact) - options = form.add_field(var='action', - ftype='list-single', + options = form.add_field(ftype='list-single', label='Action', - required=True) + required=True, + var='action') options.addOption('Display', 'view') options.addOption('Edit', 'edit') session['has_next'] = True @@ -2721,20 +2785,20 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): case 'subscribers': form = self['xep_0004'].make_form('form', 'Subscribers') form['instructions'] = 'Committing subscriber action' - options = form.add_field(var='action', - ftype='list-single', + options = form.add_field(ftype='list-single', label='Action', + required=True, value='message', - required=True) + var='action') options.addOption('Request authorization From', 'from') options.addOption('Resend authorization To', 'to') options.addOption('Send message', 'message') options.addOption('Remove', 'remove') - options = form.add_field(var='jid', + options = form.add_field(desc='Select a contact.', ftype='list-single', label='Jabber ID', - desc='Select a contact.', - required=True) + required=True, + var='jid') contacts = await XmppRoster.get_contacts(self) for contact in contacts: contact_name = contacts[contact]['name'] @@ -2743,10 +2807,10 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): form.add_field(var='subject', ftype='text-single', label='Subject') - form.add_field(var='message', + form.add_field(desc='Add a descriptive message.', ftype='text-multi', label='Message', - desc='Add a descriptive message.') + var='message') session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_subscribers_complete @@ -2761,28 +2825,19 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): label='Jabber ID') # jid_bare = self.boundjid.bare # enabled_state = Config.get_setting_value(self.settings, jid_bare, 'enabled') - # form.add_field(ftype='boolean', - # label=jid_bare, - # value=enabled_state, - # var=jid_bare) - iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) - items = iq['disco_items']['items'] - for item in items: - iq = await self['xep_0030'].get_info(jid=item[0]) - identities = iq['disco_info']['identities'] - for identity in identities: - if identity[0] == 'pubsub' and identity[1] == 'service': - jid_bare = item[0] - if item[1]: name = item[1] - elif item[2]: name = item[2] - else: name = jid_bare - enabled_state = Config.get_setting_value(self.settings, jid_bare, 'enabled') - form.add_field(desc=jid_bare, - ftype='boolean', - label=name, - value=enabled_state, - var=jid_bare) - print(Config.get_setting_value(self.settings, jid_bare, 'enabled')) + + results = await XmppPubsub.get_pubsub_services(self) + for result in results + [{'jid' : self.boundjid.bare, + 'name' : self.alias}]: + jid_bare = result['jid'] + name = result['name'] + enabled_state = Config.get_setting_value( + self.settings, jid_bare, 'enabled') + form.add_field(desc=jid_bare, + ftype='boolean', + label=name, + value=enabled_state, + var=jid_bare) session['allow_complete'] = True session['has_next'] = False session['next'] = self._handle_pubsubs_complete @@ -2807,7 +2862,8 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): db_file = config.get_pathname_to_database(jid_file) if jid_bare not in self.settings: Config.add_settings_jid(self.settings, jid_bare, db_file) - await Config.set_setting_value(self.settings, jid_bare, db_file, 'enabled', value) + await Config.set_setting_value(self.settings, jid_bare, + db_file, 'enabled', value) print(self.settings) text_note = 'Done.' session['has_next'] = False @@ -3080,11 +3136,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): value = True else: value = False - form.add_field(var='enabled', + form.add_field(desc='Enable news updates.', ftype='boolean', label='Enabled', - desc='Enable news updates.', - value=value) + value=value, + var='enabled') value = Config.get_setting_value(self.settings, jid_bare, 'media') value = str(value) value = int(value) @@ -3092,11 +3148,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): value = True else: value = False - form.add_field(var='media', + form.add_field(desc='Send audio, images or videos if found.', ftype='boolean', - desc='Send audio, images or videos if found.', label='Display media', - value=value) + value=value, + var='media') value = Config.get_setting_value(self.settings, jid_bare, 'old') value = str(value) value = int(value) @@ -3104,23 +3160,24 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): value = True else: value = False - form.add_field(var='old', + form.add_field(desc='Treat all items of newly added subscriptions ' + 'as new.', ftype='boolean', - desc='Treat all items of newly added subscriptions as new.', # label='Send only new items', label='Include old news', - value=value) + value=value, + var='old') value = Config.get_setting_value(self.settings, jid_bare, 'interval') value = str(value) value = int(value) value = value/60 value = int(value) value = str(value) - options = form.add_field(var='interval', + options = form.add_field(desc='Interval update (in hours).', ftype='list-single', label='Interval', - desc='Interval update (in hours).', - value=value) + value=value, + var='interval') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 48 } i = 1 @@ -3133,11 +3190,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): i += 1 value = Config.get_setting_value(self.settings, jid_bare, 'quantum') value = str(value) - options = form.add_field(var='quantum', + options = form.add_field(desc='Amount of items per update.', ftype='list-single', label='Amount', - desc='Amount of items per update.', - value=value) + value=value, + var='quantum') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 1, 'maximum': 5 } i = 1 @@ -3147,11 +3204,11 @@ class SlixfeedComponent(slixmpp.ComponentXMPP): i += 1 value = Config.get_setting_value(self.settings, jid_bare, 'archive') value = str(value) - options = form.add_field(var='archive', + options = form.add_field(desc='Number of news items to archive.', ftype='list-single', label='Archive', - desc='Number of news items to archive.', - value=value) + value=value, + var='archive') options['validate']['datatype'] = 'xs:integer' options['validate']['range'] = { 'minimum': 0, 'maximum': 500 } i = 0 diff --git a/slixfeed/xmpp/muc.py b/slixfeed/xmpp/muc.py index 9582c95..09ecfad 100644 --- a/slixfeed/xmpp/muc.py +++ b/slixfeed/xmpp/muc.py @@ -20,20 +20,6 @@ import logging class XmppGroupchat: - # async def accept_muc_invite(self, message, ctr=None): - # # if isinstance(message, str): - # if not ctr: - # ctr = message["from"].bare - # jid = message['groupchat_invite']['jid'] - # else: - # jid = message - def accept_invitation(self, message): - # operator muc_chat - inviter = message["from"].bare - jid = message['groupchat_invite']['jid'] - self.join(self, inviter, jid) - - async def join(self, jid, alias=None, password=None): # token = await initdb( # muc_jid, diff --git a/slixfeed/xmpp/process.py b/slixfeed/xmpp/process.py index 86f058f..f88cc96 100644 --- a/slixfeed/xmpp/process.py +++ b/slixfeed/xmpp/process.py @@ -319,22 +319,49 @@ async def message(self, message): message_text = message_text[4:] url = message_text.split(' ')[0] title = ' '.join(message_text.split(' ')[1:]) - if not title: - title = uri.get_hostname(url) - counter = 0 - hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) - while True: - if sqlite.check_node_exist(db_file, node): - counter += 1 - node = hostname + ':' + str(counter) - else: - break if url.startswith('http'): + if not title: + title = uri.get_hostname(url) db_file = config.get_pathname_to_database(jid_file) + counter = 0 + hostname = uri.get_hostname(url) + hostname = hostname.replace('.','-') + identifier = hostname + ':' + str(counter) + while True: + if sqlite.check_identifier_exist(db_file, identifier): + counter += 1 + identifier = hostname + ':' + str(counter) + else: + break exist = sqlite.get_feed_id_and_name(db_file, url) if not exist: - await sqlite.insert_feed(db_file, url, title, node) + await sqlite.insert_feed(db_file, url, title, + identifier) + feed_id = sqlite.get_feed_id(db_file, url) + feed_id = feed_id[0] + document = result['content'] + feed = parse(document) + feed_valid = 0 if feed.bozo else 1 + await sqlite.update_feed_validity(db_file, feed_id, feed_valid) + if feed.has_key('updated_parsed'): + feed_updated = feed.updated_parsed + try: + feed_updated = dt.convert_struct_time_to_iso8601(feed_updated) + except: + feed_updated = None + else: + feed_updated = None + entries_count = len(feed.entries) + await sqlite.update_feed_properties(db_file, feed_id, + entries_count, + feed_updated) + feed_id = sqlite.get_feed_id(db_file, url) + feed_id = feed_id[0] + new_entries = action.get_properties_of_entries( + self, jid_bare, db_file, url, feed_id, feed) + if new_entries: + await sqlite.add_entries_and_update_feed_state( + db_file, feed_id, new_entries) await action.scan(self, jid_bare, db_file, url) if jid_bare not in self.settings: Config.add_settings_jid(self.settings, jid_bare, @@ -477,7 +504,8 @@ async def message(self, message): XmppMessage.send_reply(self, message, response) case 'bookmarks': if is_operator(self, jid_bare): - response = await action.list_bookmarks(self) + conferences = await XmppBookmark.get_bookmarks(self) + response = action.list_bookmarks(self, conferences) else: response = ('This action is restricted. ' 'Type: viewing bookmarks.') @@ -608,7 +636,7 @@ async def message(self, message): url = ix_url if url: url = uri.remove_tracking_parameters(url) - url = (uri.replace_hostname(url, 'link')) or url + url = (await uri.replace_hostname(url, 'link')) or url result = await fetch.http(url) if not result['error']: data = result['content'] @@ -696,15 +724,17 @@ async def message(self, message): url = info[1] db_file = config.get_pathname_to_database(jid) if len(info) > 2: - node = info[2] + identifier = info[2] else: counter = 0 hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) + hostname = hostname.replace('.','-') + identifier = hostname + ':' + str(counter) while True: - if sqlite.check_node_exist(db_file, node): + if sqlite.check_identifier_exist( + db_file, identifier): counter += 1 - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) else: break # task.clean_tasks_xmpp_chat(self, jid_bare, ['status']) @@ -720,8 +750,10 @@ async def message(self, message): status_type=status_type) if url.startswith('feed:'): url = uri.feed_to_http(url) - url = (uri.replace_hostname(url, 'feed')) or url - result = await action.add_feed(self, jid_bare, db_file, url, node) + url = (await uri.replace_hostname(url, 'feed')) or url + result = await action.add_feed(self, jid_bare, + db_file, url, + identifier) if isinstance(result, list): results = result response = ("Web feeds found for {}\n\n```\n" @@ -740,11 +772,11 @@ async def message(self, message): .format(result['link'], result['name'], result['index'])) - elif result['node']: - response = ('> {}\nNode "{}" is already ' + elif result['identifier']: + response = ('> {}\nIdentifier "{}" is already ' 'allocated to index {}' .format(result['link'], - result['node'], + result['identifier'], result['index'])) elif result['error']: response = ('> {}\nFailed to find subscriptions. ' @@ -776,10 +808,10 @@ async def message(self, message): '\n' 'Missing argument. ' 'Enter PubSub JID and subscription URL ' - '(and optionally: NodeName).') + '(and optionally: Identifier Name).') else: response = ('This action is restricted. ' - 'Type: adding node.') + 'Type: publishing to node.') XmppMessage.send_reply(self, message, response) case _ if (message_lowercase.startswith('http') or message_lowercase.startswith('feed:')): @@ -797,19 +829,21 @@ async def message(self, message): status_type=status_type) if url.startswith('feed:'): url = uri.feed_to_http(url) - url = (uri.replace_hostname(url, 'feed')) or url + url = (await uri.replace_hostname(url, 'feed')) or url db_file = config.get_pathname_to_database(jid_file) counter = 0 hostname = uri.get_hostname(url) - node = hostname + ':' + str(counter) + hostname = hostname.replace('.','-') + identifier = hostname + ':' + str(counter) while True: - if sqlite.check_node_exist(db_file, node): + if sqlite.check_identifier_exist(db_file, identifier): counter += 1 - node = hostname + ':' + str(counter) + identifier = hostname + ':' + str(counter) else: break # try: - result = await action.add_feed(self, jid_bare, db_file, url, node) + result = await action.add_feed(self, jid_bare, db_file, url, + identifier) if isinstance(result, list): results = result response = ("Web feeds found for {}\n\n```\n" @@ -1122,15 +1156,15 @@ async def message(self, message): status_type=status_type) if url.startswith('feed:'): url = uri.feed_to_http(url) - url = (uri.replace_hostname(url, 'feed')) or url + url = (await uri.replace_hostname(url, 'feed')) or url match len(data): case 1: if url.startswith('http'): while True: result = await fetch.http(url) + status = result['status_code'] if not result['error']: document = result['content'] - status = result['status_code'] feed = parse(document) # if is_feed(url, feed): if action.is_feed(feed): @@ -1151,7 +1185,7 @@ async def message(self, message): .format(len(results))) break else: - url = result[0] + url = result['link'] else: response = ('> {}\nFailed to load URL. Reason: {}' .format(url, status)) @@ -1188,7 +1222,7 @@ async def message(self, message): .format(len(results))) break else: - url = result[0] + url = result['link'] else: response = ('> {}\nFailed to load URL. Reason: {}' .format(url, status)) diff --git a/slixfeed/xmpp/publish.py b/slixfeed/xmpp/publish.py index 1a8f61c..5f4ce4d 100644 --- a/slixfeed/xmpp/publish.py +++ b/slixfeed/xmpp/publish.py @@ -15,7 +15,7 @@ class XmppPubsub: async def get_pubsub_services(self): - jids = [self.boundjid.bare] + results = [] iq = await self['xep_0030'].get_items(jid=self.boundjid.domain) items = iq['disco_items']['items'] for item in items: @@ -23,9 +23,13 @@ class XmppPubsub: identities = iq['disco_info']['identities'] for identity in identities: if identity[0] == 'pubsub' and identity[1] == 'service': - jid = item[0] - jids.extend([jid]) - return jids + result = {} + result['jid'] = item[0] + if item[1]: result['name'] = item[1] + elif item[2]: result['name'] = item[2] + else: result['name'] = item[0] + results.extend([result]) + return results def delete_node(self, jid, node): @@ -44,7 +48,7 @@ class XmppPubsub: # TODO Make use of var "xep" with match/case (XEP-0060, XEP-0277, XEP-0472) - def create_node(self, jid, node, xep ,title=None, summary=None): + def create_node(self, jid, node, xep ,title=None, subtitle=None): jid_from = str(self.boundjid) if self.is_component else None iq = self.Iq(stype='set', sto=jid, @@ -57,7 +61,7 @@ class XmppPubsub: value=title) form.addField('pubsub#description', ftype='text-single', - value=summary) + value=subtitle) form.addField('pubsub#notify_retract', ftype='boolean', value=1) diff --git a/slixfeed/xmpp/utility.py b/slixfeed/xmpp/utility.py index 5e53e6d..c874026 100644 --- a/slixfeed/xmpp/utility.py +++ b/slixfeed/xmpp/utility.py @@ -17,6 +17,7 @@ def is_operator(self, jid_bare): break return result + def is_moderator(self, jid_bare, jid_full): alias = jid_full[jid_full.index('/')+1:] role = self.plugin['xep_0045'].get_jid_property(jid_bare, alias, 'role') @@ -27,6 +28,16 @@ def is_moderator(self, jid_bare, jid_full): return result +def is_member(self, jid_bare, jid_full): + alias = jid_full[jid_full.index('/')+1:] + affiliation = self.plugin['xep_0045'].get_jid_property(jid_bare, alias, 'affiliation') + if affiliation == 'member': + result = True + else: + result = False + return result + + # TODO Rename to get_jid_type async def get_chat_type(self, jid): """