#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ TODO 1) Function scan at "for entry in entries" Suppress directly calling function "add_entry" (accept db_file) Pass a list of valid entries to a new function "add_entries" (accept db_file) which would call function "add_entry" (accept cur). * accelerate adding of large set of entries at once. * prevent (or mitigate halt of consequent actions). * reduce I/O. 2) Call sqlite function from function statistics. Returning a list of values doesn't' seem to be a good practice. 3) Special statistics for operator: * Size of database(s); * Amount of JIDs subscribed; * Amount of feeds of all JIDs; * Amount of entries of all JIDs. """ import asyncio from feedparser import parse import os from slixfeed.config import Config import slixfeed.fetch as fetch from slixfeed.log import Logger,Message import slixfeed.sqlite as sqlite from slixfeed.utilities import DateAndTime, Html, MD, String, Url, Utilities from slixmpp.xmlstream import ET import sys from urllib.parse import urlsplit import xml.etree.ElementTree as ETR logger = Logger(__name__) class Feed: # NOTE Consider removal of MD (and any other option HTML and XBEL) def export_feeds(dir_data, dir_cache, jid_bare, ext): function_name = sys._getframe().f_code.co_name logger.debug('{}: jid_bare: {}: ext: {}'.format(function_name, jid_bare, ext)) if not os.path.isdir(dir_cache): os.mkdir(dir_cache) if not os.path.isdir(dir_cache + '/' + ext): os.mkdir(dir_cache + '/' + ext) filename = os.path.join( dir_cache, ext, 'slixfeed_' + DateAndTime.timestamp() + '.' + ext) db_file = os.path.join(dir_data, 'sqlite', f'{jid_bare}.db') results = sqlite.get_feeds(db_file) match ext: # case 'html': # response = 'Not yet implemented.' case 'md': MD.export_to_markdown(jid_bare, filename, results) case 'opml': Opml.export_to_file(jid_bare, filename, results) # case 'xbel': # response = 'Not yet implemented.' return filename def pack_entry_into_dict(db_file, entry): entry_id = entry[0] authors = sqlite.get_authors_by_entry_id(db_file, entry_id) entry_authors = [] for author in authors: entry_author = { 'name': author[2], 'email': author[3], 'url': author[4]} entry_authors.extend([entry_author]) contributors = sqlite.get_contributors_by_entry_id(db_file, entry_id) entry_contributors = [] for contributor in contributors: entry_contributor = { 'name': contributor[2], 'email': contributor[3], 'url': contributor[4]} entry_contributors.extend([entry_contributor]) links = sqlite.get_links_by_entry_id(db_file, entry_id) entry_links = [] for link in links: entry_link = { 'url': link[2], 'type': link[3], 'rel': link[4], 'size': link[5]} entry_links.extend([entry_link]) tags = sqlite.get_tags_by_entry_id(db_file, entry_id) entry_tags = [] for tag in tags: entry_tag = { 'term': tag[2], 'scheme': tag[3], 'label': tag[4]} entry_tags.extend([entry_tag]) contents = sqlite.get_contents_by_entry_id(db_file, entry_id) entry_contents = [] for content in contents: entry_content = { 'text': content[2], 'type': content[3], 'base': content[4], 'lang': content[5]} entry_contents.extend([entry_content]) feed_entry = { 'authors' : entry_authors, 'category' : entry[10], 'comments' : entry[12], 'contents' : entry_contents, 'contributors' : entry_contributors, 'summary_base' : entry[9], 'summary_lang' : entry[7], 'summary_text' : entry[6], 'summary_type' : entry[8], 'enclosures' : entry[13], 'href' : entry[11], 'link' : entry[3], 'links' : entry_links, 'published' : entry[14], 'rating' : entry[13], 'tags' : entry_tags, 'title' : entry[4], 'title_type' : entry[3], 'updated' : entry[15]} return feed_entry def create_rfc4287_entry(feed_entry): node_entry = ET.Element('entry') node_entry.set('xmlns', 'http://www.w3.org/2005/Atom') # Title title = ET.SubElement(node_entry, 'title') if feed_entry['title']: if feed_entry['title_type']: title.set('type', feed_entry['title_type']) title.text = feed_entry['title'] elif feed_entry['summary_text']: if feed_entry['summary_type']: title.set('type', feed_entry['summary_type']) title.text = feed_entry['summary_text'] # if feed_entry['summary_base']: title.set('base', feed_entry['summary_base']) # if feed_entry['summary_lang']: title.set('lang', feed_entry['summary_lang']) else: title.text = feed_entry['published'] # Some feeds have identical content for contents and summary # So if content is present, do not add summary if feed_entry['contents']: # Content for feed_entry_content in feed_entry['contents']: content = ET.SubElement(node_entry, 'content') # if feed_entry_content['base']: content.set('base', feed_entry_content['base']) if feed_entry_content['lang']: content.set('lang', feed_entry_content['lang']) if feed_entry_content['type']: content.set('type', feed_entry_content['type']) content.text = feed_entry_content['text'] else: # Summary summary = ET.SubElement(node_entry, 'summary') # TODO Try 'content' # if feed_entry['summary_base']: summary.set('base', feed_entry['summary_base']) # TODO Check realization of "lang" if feed_entry['summary_type']: summary.set('type', feed_entry['summary_type']) if feed_entry['summary_lang']: summary.set('lang', feed_entry['summary_lang']) summary.text = feed_entry['summary_text'] # Authors for feed_entry_author in feed_entry['authors']: author = ET.SubElement(node_entry, 'author') name = ET.SubElement(author, 'name') name.text = feed_entry_author['name'] if feed_entry_author['url']: uri = ET.SubElement(author, 'uri') uri.text = feed_entry_author['url'] if feed_entry_author['email']: email = ET.SubElement(author, 'email') email.text = feed_entry_author['email'] # Contributors for feed_entry_contributor in feed_entry['contributors']: contributor = ET.SubElement(node_entry, 'author') name = ET.SubElement(contributor, 'name') name.text = feed_entry_contributor['name'] if feed_entry_contributor['url']: uri = ET.SubElement(contributor, 'uri') uri.text = feed_entry_contributor['url'] if feed_entry_contributor['email']: email = ET.SubElement(contributor, 'email') email.text = feed_entry_contributor['email'] # Category category = ET.SubElement(node_entry, "category") category.set('category', feed_entry['category']) # Tags for feed_entry_tag in feed_entry['tags']: tag = ET.SubElement(node_entry, 'category') tag.set('term', feed_entry_tag['term']) # Link link = ET.SubElement(node_entry, "link") link.set('href', feed_entry['link']) # Links for feed_entry_link in feed_entry['links']: link = ET.SubElement(node_entry, "link") link.set('href', feed_entry_link['url']) link.set('type', feed_entry_link['type']) link.set('rel', feed_entry_link['rel']) # Date updated if feed_entry['updated']: updated = ET.SubElement(node_entry, 'updated') updated.text = feed_entry['updated'] # Date published if feed_entry['published']: published = ET.SubElement(node_entry, 'published') published.text = feed_entry['published'] return node_entry # Look into the type ("atom", "rss2" etc.) def is_feed(url, feed): """ Determine whether document is feed or not. Parameters ---------- feed : dict Parsed feed. Returns ------- val : boolean True or False. """ function_name = sys._getframe().f_code.co_name logger.debug('{}'.format(function_name)) value = False # message = None if not feed.entries: if "version" in feed.keys(): # feed["version"] if feed.version: value = True # message = ( # "Empty feed for {}" # ).format(url) elif "title" in feed["feed"].keys(): value = True # message = ( # "Empty feed for {}" # ).format(url) else: value = False # message = ( # "No entries nor title for {}" # ).format(url) elif feed.bozo: # NOTE Consider valid even when is not-well-formed value = True logger.warning('Bozo detected for {}'.format(url)) else: value = True # message = ( # "Good feed for {}" # ).format(url) return value async def add_feed(self, jid_bare, db_file, url, identifier): function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} url: {}' .format(function_name, db_file, url)) while True: feed_id = sqlite.get_feed_id(db_file, url) if not feed_id: if not sqlite.check_identifier_exist(db_file, identifier): result = await fetch.http(self.settings_network, url) message = result['message'] status_code = result['status_code'] if not result['error']: await sqlite.update_feed_status(db_file, feed_id, status_code) document = result['content'] feed = parse(document) # if document and status_code == 200: if Feed.is_feed(url, feed): if "title" in feed["feed"].keys(): title = feed["feed"]["title"] else: title = urlsplit(url).netloc if "language" in feed["feed"].keys(): language = feed["feed"]["language"] else: language = '' if "encoding" in feed.keys(): encoding = feed["encoding"] else: encoding = '' if "updated_parsed" in feed["feed"].keys(): updated = feed["feed"]["updated_parsed"] try: updated = DateAndTime.convert_struct_time_to_iso8601(updated) except Exception as e: logger.error(str(e)) updated = '' else: updated = '' version = feed.version entries_count = len(feed.entries) await sqlite.insert_feed(db_file, url, title, identifier, entries=entries_count, version=version, encoding=encoding, language=language, status_code=status_code, updated=updated) feed_valid = 0 if feed.bozo else 1 await sqlite.update_feed_validity( db_file, feed_id, feed_valid) if feed.has_key('updated_parsed'): feed_updated = feed.updated_parsed try: feed_updated = DateAndTime.convert_struct_time_to_iso8601(feed_updated) except Exception as e: logger.error(str(e)) feed_updated = None else: feed_updated = None feed_properties = Feed.get_properties_of_feed( db_file, feed_id, feed) await sqlite.update_feed_properties( db_file, feed_id, feed_properties) feed_id = sqlite.get_feed_id(db_file, url) feed_id = feed_id[0] new_entries = [] for entry in feed.entries: if entry.has_key("link"): entry_link = Url.join_url(url, entry.link) entry_link = Url.trim_url(entry_link) entry_identifier = String.md5_hash(entry_link) if not sqlite.get_entry_id_by_identifier( db_file, entry_identifier): new_entry = Feed.get_properties_of_entry( url, entry_identifier, entry) new_entries.extend([new_entry]) if new_entries: await sqlite.add_entries_and_update_feed_state( db_file, feed_id, new_entries) old = self.settings[jid_bare]['old'] or self.defaults['default']['old'] if not old: await sqlite.mark_feed_as_read(db_file, feed_id) result_final = {'link' : url, 'index' : feed_id, 'name' : title, 'code' : status_code, 'error' : False, 'message': message, 'exist' : False, 'identifier' : None} break else: # NOTE Do not be tempted to return a compact dictionary. # That is, dictionary within dictionary # Return multiple dictionaries in a list or tuple. result = await FeedDiscovery.probe_page( self.settings_network, self.pathnames, url, document) if not result: # Get out of the loop with dict indicating error. result_final = {'link' : url, 'index' : None, 'name' : None, 'code' : status_code, 'error' : True, 'message': message, 'exist' : False, 'identifier' : None} break elif isinstance(result, list): # Get out of the loop and deliver a list of dicts. result_final = result break else: # Go back up to the while loop and try again. url = result['link'] else: await sqlite.update_feed_status(db_file, feed_id, status_code) result_final = {'link' : url, 'index' : None, 'name' : None, 'code' : status_code, 'error' : True, 'message': message, 'exist' : False, 'identifier' : None} break else: ix = sqlite.get_entry_id_by_identifier(db_file, identifier) message = ('Identifier "{}" is already allocated.' .format(identifier)) result_final = {'link' : url, 'index' : ix, 'name' : None, 'code' : None, 'error' : False, 'message': message, 'exist' : False, 'identifier' : identifier} break else: feed_id = feed_id[0] title = sqlite.get_feed_title(db_file, feed_id) title = title[0] message = 'URL already exist.' result_final = {'link' : url, 'index' : feed_id, 'name' : title, 'code' : None, 'error' : False, 'message': message, 'exist' : True, 'identifier' : None} break return result_final def view_feed(url, feed): function_name = sys._getframe().f_code.co_name logger.debug('{}: url: {}' .format(function_name, url)) if "title" in feed["feed"].keys(): title = feed["feed"]["title"] else: title = urlsplit(url).netloc entries = feed.entries response = "Preview of {}:\n\n```\n".format(title) counter = 0 for entry in entries: counter += 1 if entry.has_key("title"): title = entry.title else: title = "*** No title ***" if entry.has_key("link"): # link = complete_url(source, entry.link) link = Url.join_url(url, entry.link) link = Url.trim_url(link) else: link = "*** No link ***" if entry.has_key("published"): date = entry.published date = DateAndTime.rfc2822_to_iso8601(date) elif entry.has_key("updated"): date = entry.updated date = DateAndTime.rfc2822_to_iso8601(date) else: date = "*** No date ***" response += ("Title : {}\n" "Date : {}\n" "Link : {}\n" "Count : {}\n" "\n" .format(title, date, link, counter)) if counter > 4: break response += ( "```\nSource: {}" ).format(url) return response def view_entry(url, feed, num): function_name = sys._getframe().f_code.co_name logger.debug('{}: url: {} num: {}' .format(function_name, url, num)) if "title" in feed["feed"].keys(): title = feed["feed"]["title"] else: title = urlsplit(url).netloc entries = feed.entries num = int(num) - 1 entry = entries[num] response = "Preview of {}:\n\n```\n".format(title) if entry.has_key("title"): title = entry.title else: title = '*** No title ***' if entry.has_key("published"): date = entry.published date = DateAndTime.rfc2822_to_iso8601(date) elif entry.has_key("updated"): date = entry.updated date = DateAndTime.rfc2822_to_iso8601(date) else: date = '*** No date ***' if entry.has_key("summary"): summary = entry.summary # Remove HTML tags if summary: summary = Html.remove_html_tags(summary) # TODO Limit text length summary = summary.replace("\n\n\n", "\n\n") else: summary = '*** No summary ***' else: summary = '*** No summary ***' if entry.has_key("link"): # link = complete_url(source, entry.link) link = Url.join_url(url, entry.link) link = Url.trim_url(link) else: link = '*** No link ***' response = ("{}\n" "\n" # "> {}\n" "{}\n" "\n" "{}\n" "\n" .format(title, summary, link)) return response # NOTE This function is not being utilized async def download_feed(settings_network, db_file, feed_url): """ Process feed content. Parameters ---------- db_file : str Path to database file. feed_url : str URL of feed. """ function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {} url: {}' .format(function_name, db_file, feed_url)) if isinstance(feed_url, tuple): feed_url = feed_url[0] result = await fetch.http(settings_network, feed_url) feed_id = sqlite.get_feed_id(db_file, feed_url) feed_id = feed_id[0] status_code = result['status_code'] await sqlite.update_feed_status(db_file, feed_id, status_code) def get_properties_of_feed(db_file, feed_id, feed): if feed.has_key('updated_parsed'): feed_updated = feed.updated_parsed try: feed_updated = DateAndTime.convert_struct_time_to_iso8601(feed_updated) except: feed_updated = '' else: feed_updated = '' entries_count = len(feed.entries) feed_version = feed.version if feed.has_key('version') else '' feed_encoding = feed.encoding if feed.has_key('encoding') else '' feed_language = feed.feed.language if feed.feed.has_key('language') else '' feed_icon = feed.feed.icon if feed.feed.has_key('icon') else '' # (Pdb) feed.feed.image # {} # (Pdb) feed.version # 'rss10' # (Pdb) feed.feed.image # {'links': [{'rel': 'alternate', 'type': 'text/html'}]} # (Pdb) feed.version # '' feed_image = feed.feed.image.href if feed.feed.has_key('image') and feed.feed.image.has_key('href') else '' feed_logo = feed.feed.logo if feed.feed.has_key('logo') else '' feed_ttl = feed.feed.ttl if feed.feed.has_key('ttl') else '' feed_properties = { "version" : feed_version, "encoding" : feed_encoding, "language" : feed_language, "rating" : '', "entries_count" : entries_count, "icon" : feed_icon, "image" : feed_image, "logo" : feed_logo, "ttl" : feed_ttl, "updated" : feed_updated, } return feed_properties # TODO get all active feeds of active accounts and scan the feed with the earliest scanned time # TODO Rename function name (idea: scan_and_populate) def get_properties_of_entry(feed_url, entry_identifier, entry): """ Process entry content. Parameters ---------- feed_url : str URL of feed. entry : Object of entry. """ function_name = sys._getframe().f_code.co_name logger.debug('{} feed_url: {}' .format(function_name, feed_url)) read_status = 0 if entry.has_key("published"): entry_published = entry.published entry_published = DateAndTime.rfc2822_to_iso8601(entry_published) else: entry_published = '' if entry.has_key("updated"): entry_updated = entry.updated entry_updated = DateAndTime.rfc2822_to_iso8601(entry_updated) else: entry_updated = DateAndTime.now() if entry.has_key("link"): # link = complete_url(source, entry.link) entry_link = Url.join_url(feed_url, entry.link) entry_link = Url.trim_url(entry_link) else: entry_link = feed_url # title = feed["feed"]["title"] # title = "{}: *{}*".format(feed["feed"]["title"], entry.title) entry_title = entry.title if entry.has_key("title") else entry_published # entry_id = entry.id if entry.has_key("id") else entry_link # # Filter # pathname = urlsplit(link).path # string = ( # "{} {} {}" # ).format( # title, summary, pathname) # if self.settings['default']['filter']: # print('Filter is now processing data.') # allow_list = config.is_include_keyword(db_file, # "allow", string) # if not allow_list: # reject_list = config.is_include_keyword(db_file, # "deny", # string) # if reject_list: # read_status = 1 # logger.debug('Rejected : {}' # '\n' # 'Keyword : {}' # .format(link, reject_list)) if isinstance(entry_published, int): logger.error('Variable "published" is int: {}'.format(entry_published)) if isinstance(entry_updated, int): logger.error('Variable "updated" is int: {}'.format(entry_updated)) # Authors entry_authors =[] if entry.has_key('authors'): for author in entry.authors: author_properties = { 'name' : author.name if author.has_key('name') else '', 'url' : author.href if author.has_key('href') else '', 'email' : author.email if author.has_key('email') else '', } entry_authors.extend([author_properties]) elif entry.has_key('author_detail'): author_properties = { 'name' : entry.author_detail.name if entry.author_detail.has_key('name') else '', 'url' : entry.author_detail.href if entry.author_detail.has_key('href') else '', 'email' : entry.author_detail.email if entry.author_detail.has_key('email') else '', } entry_authors.extend([author_properties]) elif entry.has_key('author'): author_properties = { 'name' : entry.author, 'url' : '', 'email' : '', } entry_authors.extend([author_properties]) # Contributors entry_contributors = [] if entry.has_key('contributors'): for contributor in entry.contributors: contributor_properties = { 'name' : contributor.name if contributor.has_key('name') else '', 'url' : contributor.href if contributor.has_key('href') else '', 'email' : contributor.email if contributor.has_key('email') else '', } entry_contributors.extend([contributor_properties]) # Tags entry_tags = [] if entry.has_key('tags'): for tag in entry.tags: tag_properties = { 'term' : tag.term if tag.has_key('term') else '', 'scheme' : tag.scheme if tag.has_key('scheme') else '', 'label' : tag.label if tag.has_key('label') else '', } entry_tags.extend([tag_properties]) # Content entry_contents = [] if entry.has_key('content'): for content in entry.content: text = content.value if content.has_key('value') else '' type = content.type if content.has_key('type') else '' lang = content.lang if content.has_key('lang') else '' base = content.base if content.has_key('base') else '' entry_content = { 'text' : text, 'lang' : lang, 'type' : type, 'base' : base, } entry_contents.extend([entry_content]) # Links and Enclosures entry_links = [] if entry.has_key('links'): for link in entry.links: link_properties = { 'url' : link.href if link.has_key('href') else '', 'rel' : link.rel if link.has_key('rel') else '', 'type' : link.type if link.has_key('type') else '', 'length' : '', } entry_links.extend([link_properties]) # Element media:content is utilized by Mastodon if entry.has_key('media_content'): for link in entry.media_content: link_properties = { 'url' : link['url'] if 'url' in link else '', 'rel' : 'enclosure', 'type' : link['type'] if 'type' in link else '', # 'medium' : link['medium'] if 'medium' in link else '', 'length' : link['filesize'] if 'filesize' in link else '', } entry_links.extend([link_properties]) if entry.has_key('media_thumbnail'): for link in entry.media_thumbnail: link_properties = { 'url' : link['url'] if 'url' in link else '', 'rel' : 'enclosure', 'type' : '', # 'medium' : 'image', 'length' : '', } entry_links.extend([link_properties]) # Category entry_category = entry.category if entry.has_key('category') else '' # Comments entry_comments = entry.comments if entry.has_key('comments') else '' # href entry_href = entry.href if entry.has_key('href') else '' # Link: Same as entry.links[0].href in most if not all cases entry_link = entry.link if entry.has_key('link') else '' # Rating entry_rating = entry.rating if entry.has_key('rating') else '' # Summary entry_summary_text = entry.summary if entry.has_key('summary') else '' if entry.has_key('summary_detail'): entry_summary_type = entry.summary_detail.type if entry.summary_detail.has_key('type') else '' entry_summary_lang = entry.summary_detail.lang if entry.summary_detail.has_key('lang') else '' entry_summary_base = entry.summary_detail.base if entry.summary_detail.has_key('base') else '' else: entry_summary_type = '' entry_summary_lang = '' entry_summary_base = '' # Title entry_title = entry.title if entry.has_key('title') else '' if entry.has_key('title_detail'): entry_title_type = entry.title_detail.type if entry.title_detail.has_key('type') else '' else: entry_title_type = '' ########################################################### # media_type = e_link.type[:e_link.type.index("/")] # if (e_link.rel == "enclosure" and # media_type in ("audio", "image", "video")): # media_link = e_link.href # media_link = Url.join_url(url, e_link.href) # media_link = Url.trim_url(media_link) ########################################################### entry_properties = { "identifier": entry_identifier, "link": entry_link, "href": entry_href, "title": entry_title, "title_type": entry_title_type, 'summary_text' : entry_summary_text, 'summary_lang' : entry_summary_lang, 'summary_type' : entry_summary_type, 'summary_base' : entry_summary_base, 'category' : entry_category, "comments": entry_comments, "rating": entry_rating, "published": entry_published, "updated": entry_updated, "read_status": read_status} new_entry = { "entry_properties" : entry_properties, "entry_authors" : entry_authors, "entry_contributors" : entry_contributors, "entry_contents" : entry_contents, "entry_links" : entry_links, "entry_tags" : entry_tags} # await sqlite.add_entry( # db_file, title, link, entry_id, # url, date, read_status) # await sqlite.set_date(db_file, url) return new_entry """ FIXME 1) https://wiki.pine64.org File "/slixfeed/crawl.py", line 178, in feed_mode_guess address = Url.join_url(url, parted_url.path.split('/')[1] + path) ~~~~~~~~~~~~~~~~~~~~~~~~~~^^^ IndexError: list index out of range TODO 1.1) Attempt to scan more paths: /blog/, /news/ etc., including root / Attempt to scan sub domains https://esmailelbob.xyz/en/ https://blog.esmailelbob.xyz/feed/ 1.2) Consider utilizing fetch.http_response 2) DeviantArt https://www.deviantart.com/nedesem/gallery https://backend.deviantart.com/rss.xml?q=gallery:nedesem https://backend.deviantart.com/rss.xml?q=nedesem https://www.deviantart.com/search?q= https://backend.deviantart.com/rss.xml?q=search: FEEDS CRAWLER PROJECT 3) Mark redirects for manual check Title : JSON Feed Link : https://www.jsonfeed.org/feed.json.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/atom.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/feed.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/feeds/rss/news.xml.php Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/jekyll/feed.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/news.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/news.xml.php Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/rdf.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/rss.xml Title : JSON Feed Link : https://www.jsonfeed.org/feed.json/videos.xml """ from aiohttp import ClientError, ClientSession, ClientTimeout from lxml import etree from lxml import html from lxml.etree import fromstring class FeedDiscovery: # TODO Use boolean as a flag to determine whether a single URL was found # async def probe_page( # callback, url, document, num=None, db_file=None): # result = None # try: # # tree = etree.fromstring(res[0]) # etree is for xml # tree = html.fromstring(document) # except: # result = ( # "> {}\nFailed to parse URL as feed." # ).format(url) # if not result: # print("RSS Auto-Discovery Engaged") # result = await feed_mode_auto_discovery(url, tree) # if not result: # print("RSS Scan Mode Engaged") # result = await feed_mode_scan(url, tree) # if not result: # print("RSS Arbitrary Mode Engaged") # result = await feed_mode_request(url, tree) # if not result: # result = ( # "> {}\nNo news feeds were found for URL." # ).format(url) # # elif msg: # else: # if isinstance(result, str): # return result # elif isinstance(result, list): # url = result[0] # if db_file: # # print("if db_file", db_file) # return await callback(db_file, url) # elif num: # return await callback(url, num) # else: # return await callback(url) async def probe_page(settings_network, pathnames, url, document=None): """ Parameters ---------- url : str URL. document : TYPE DESCRIPTION. Returns ------- result : list or str Single URL as list or selection of URLs as str. """ if not document: response = await fetch.http(settings_network, url) if not response['error']: document = response['content'] try: # tree = etree.fromstring(res[0]) # etree is for xml tree = html.fromstring(document) result = None except Exception as e: logger.error(str(e)) try: # /questions/15830421/xml-unicode-strings-with-encoding-declaration-are-not-supported # xml = html.fromstring(document.encode('utf-8')) # parser = etree.XMLParser(ns_clean=True, recover=True, encoding='utf-8') # tree = fromstring(xml, parser=parser) # /questions/57833080/how-to-fix-unicode-strings-with-encoding-declaration-are-not-supported #tree = html.fromstring(bytes(document, encoding='utf8')) # https://twigstechtips.blogspot.com/2013/06/python-lxml-strings-with-encoding.html #parser = etree.XMLParser(recover=True) #tree = etree.fromstring(document, parser) tree = html.fromstring(document.encode('utf-8')) result = None except Exception as e: logger.error(str(e)) logger.warning(f"Failed to parse URL as feed for {url}.") result = {'link' : None, 'index' : None, 'name' : None, 'code' : None, 'error' : True, 'exist' : None} if not result: logger.debug(f"Feed auto-discovery engaged for {url}") result = FeedDiscovery.feed_mode_auto_discovery(url, tree) if not result: logger.debug(f"Feed link scan mode engaged for {url}") result = FeedDiscovery.feed_mode_scan(url, tree, pathnames) if not result: logger.debug(f"Feed arbitrary mode engaged for {url}") result = FeedDiscovery.feed_mode_guess(url, pathnames) if not result: logger.debug(f"No feeds were found for {url}") result = None result = await FeedDiscovery.process_feed_selection(settings_network, url, result) return result # TODO Improve scan by gradual decreasing of path def feed_mode_guess(url, pathnames): """ Lookup for feeds by pathname using HTTP Requests. Parameters ---------- db_file : str Path to database file. url : str URL. pathnames : list pathnames. Returns ------- msg : str Message with URLs. """ urls = [] parted_url = urlsplit(url) # Check whether URL has path (i.e. not root) # Check parted_url.path to avoid error in case root wasn't given # TODO Make more tests if parted_url.path and parted_url.path.split('/')[1]: pathnames.extend( [".atom", ".feed", ".rdf", ".rss"] ) if '.rss' not in pathnames else -1 # if paths.index('.rss'): # paths.extend([".atom", ".feed", ".rdf", ".rss"]) parted_url_path = parted_url.path if parted_url.path else '/' for path in pathnames: address = Url.join_url(url, parted_url_path.split('/')[1] + path) if address not in urls: urls.extend([address]) # breakpoint() # print("feed_mode_guess") return urls def feed_mode_scan(url, tree, pathnames): """ Scan page for potential feeds by pathname. Parameters ---------- db_file : str Path to database file. url : str URL. tree : TYPE DESCRIPTION. Returns ------- msg : str Message with URLs. """ urls = [] for path in pathnames: # xpath_query = "//*[@*[contains(.,'{}')]]".format(path) # xpath_query = "//a[contains(@href,'{}')]".format(path) num = 5 xpath_query = ( "(//a[contains(@href,'{}')])[position()<={}]" ).format(path, num) addresses = tree.xpath(xpath_query) xpath_query = ( "(//a[contains(@href,'{}')])[position()>last()-{}]" ).format(path, num) addresses += tree.xpath(xpath_query) # NOTE Should number of addresses be limited or # perhaps be N from the start and N from the end for address in addresses: address = Url.join_url(url, address.xpath('@href')[0]) if address not in urls: urls.extend([address]) # breakpoint() # print("feed_mode_scan") return urls def feed_mode_auto_discovery(url, tree): """ Lookup for feeds using RSS autodiscovery technique. See: https://www.rssboard.org/rss-autodiscovery Parameters ---------- db_file : str Path to database file. url : str URL. tree : TYPE DESCRIPTION. Returns ------- msg : str Message with URLs. """ xpath_query = ( '//link[(@rel="alternate") and ' '(@type="application/atom+xml" or ' '@type="application/rdf+xml" or ' '@type="application/rss+xml")]' ) # xpath_query = """//link[(@rel="alternate") and (@type="application/atom+xml" or @type="application/rdf+xml" or @type="application/rss+xml")]/@href""" # xpath_query = "//link[@rel='alternate' and @type='application/atom+xml' or @rel='alternate' and @type='application/rss+xml' or @rel='alternate' and @type='application/rdf+xml']/@href" feeds = tree.xpath(xpath_query) if feeds: urls = [] for feed in feeds: # # The following code works; # # The following code will catch # # only valid resources (i.e. not 404); # # The following code requires more bandwidth. # res = await fetch.http(feed) # if res[0]: # disco = parse(res[0]) # title = disco["feed"]["title"] # msg += "{} \n {} \n\n".format(title, feed) # feed_name = feed.xpath('@title')[0] # feed_addr = Url.join_url(url, feed.xpath('@href')[0]) # if feed_addr.startswith("/"): # feed_addr = url + feed_addr address = Url.join_url(url, feed.xpath('@href')[0]) if address not in urls: urls.extend([address]) # breakpoint() # print("feed_mode_auto_discovery") return urls # TODO Segregate function into function that returns # URLs (string) and Feeds (dict) and function that # composes text message (string). # Maybe that's not necessary. async def process_feed_selection(settings_network, url, urls): feeds = {} for i in urls: result = await fetch.http(settings_network, i) if not result['error']: document = result['content'] status_code = result['status_code'] if status_code == 200: # NOTE This line might be redundant try: feeds[i] = [parse(document)] except: continue message = ( "Web feeds found for {}\n\n```\n" ).format(url) urls = [] for feed_url in feeds: # try: # res = await fetch.http(feed) # except: # continue feed_name = None if "title" in feeds[feed_url][0]["feed"].keys(): feed_name = feeds[feed_url][0].feed.title feed_name = feed_name if feed_name else "Untitled" # feed_name = feed_name if feed_name else urlsplit(feed_url).netloc # AttributeError: 'str' object has no attribute 'entries' if "entries" in feeds[feed_url][0].keys(): feed_amnt = feeds[feed_url][0].entries else: continue if feed_amnt: # NOTE Because there could be many false positives # which are revealed in second phase of scan, we # could end with a single feed, which would be # listed instead of fetched, so feed_url_mark is # utilized in order to make fetch possible. # NOTE feed_url_mark was a variable which stored # single URL (probably first accepted as valid) # in order to get an indication whether a single # URL has been fetched, so that the receiving # function will scan that single URL instead of # listing it as a message. url = {'link' : feed_url, 'index' : None, 'name' : feed_name, 'code' : status_code, 'error' : False, 'exist' : None} urls.extend([url]) count = len(urls) if count > 1: result = urls elif count: result = urls[0] else: result = None return result # def get_discovered_feeds(url, urls): # message = ( # "Found {} web feeds:\n\n```\n" # ).format(len(urls)) # if len(urls) > 1: # for urls in urls: # message += ( # "Title : {}\n" # "Link : {}\n" # "\n" # ).format(url, url.title) # message += ( # "```\nThe above feeds were extracted from\n{}" # ).format(url) # elif len(urls) > 0: # result = urls # else: # message = ( # "No feeds were found for {}" # ).format(url) # return result # Test module # TODO ModuleNotFoundError: No module named 'slixfeed' # import slixfeed.fetch as fetch # from slixfeed.action import is_feed, process_feed_selection # async def start(url): # while True: # result = await fetch.http(url) # document = result[0] # status = result[1] # if document: # feed = parse(document) # if is_feed(feed): # print(url) # else: # urls = await probe_page( # url, document) # if len(urls) > 1: # await process_feed_selection(urls) # elif urls: # url = urls[0] # else: # response = ( # "> {}\nFailed to load URL. Reason: {}" # ).format(url, status) # break # return response # url = "https://www.smh.com.au/rssheadlines" # start(url) class FeedTask: # TODO Take this function out of # async def check_updates(self, jid_bare): """ Start calling for update check up. Parameters ---------- jid : str Jabber ID. """ # print('Scanning for updates for JID {}'.format(jid_bare)) logger.info('Scanning for updates for JID {}'.format(jid_bare)) while True: db_file = os.path.join(self.dir_data, 'sqlite', f'{jid_bare}.db') urls = sqlite.get_active_feeds_url_sorted_by_last_scanned(db_file) for url in urls: #Message.printer('Scanning updates for URL {} ...'.format(url)) url = url[0] # print('STA',url) # # Skip Reddit # if 'reddit.com' in str(url).lower(): # print('Reddit Atom Syndication feeds are not supported by Slixfeed.') # print('Skipping URL:', url) # continue result = await fetch.http(self.settings_network, url) status_code = result['status_code'] feed_id = sqlite.get_feed_id(db_file, url) feed_id = feed_id[0] if not result['error']: identifier = sqlite.get_feed_identifier(db_file, feed_id) identifier = identifier[0] if not identifier: counter = 0 while True: identifier = String.generate_identifier(url, counter) if sqlite.check_identifier_exist(db_file, identifier): counter += 1 else: break await sqlite.update_feed_identifier(db_file, feed_id, identifier) # identifier = sqlite.get_feed_identifier(db_file, feed_id) # identifier = identifier[0] await sqlite.update_feed_status(db_file, feed_id, status_code) document = result['content'] feed = parse(document) feed_valid = 0 if feed.bozo else 1 await sqlite.update_feed_validity(db_file, feed_id, feed_valid) feed_properties = Feed.get_properties_of_feed( db_file, feed_id, feed) await sqlite.update_feed_properties( db_file, feed_id, feed_properties) new_entries = [] for entry in feed.entries: if entry.has_key("link"): # link = complete_url(source, entry.link) entry_link = Url.join_url(url, entry.link) entry_link = Url.trim_url(entry_link) entry_identifier = String.md5_hash(entry_link) # if 'f-droid.org' in url: # breakpoint() # print(entry.link) # print(entry_identifier) # Check if an entry identifier exists if not sqlite.get_entry_id_by_identifier( db_file, entry_identifier): new_entry = Feed.get_properties_of_entry( url, entry_identifier, entry) # new_entries.append(new_entry) new_entries.extend([new_entry]) if new_entries: await sqlite.add_entries_and_update_feed_state(db_file, feed_id, new_entries) limit = Config.get_setting_value(self, jid_bare, 'archive') ixs = sqlite.get_entries_id_of_feed(db_file, feed_id) ixs_invalid = {} for ix in ixs: ix = ix[0] read_status = sqlite.is_entry_read(db_file, ix) read_status = read_status[0] entry_identifier_local = sqlite.get_entry_identifier(db_file, ix) entry_identifier_local = entry_identifier_local[0] valid = False for entry in feed.entries: if entry.has_key("link"): entry_link = Url.join_url(url, entry.link) entry_link = Url.trim_url(entry_link) entry_identifier_external = Utilities.hash_url_to_md5( entry_link) if entry_identifier_local == entry_identifier_external: valid = True continue if not valid: ixs_invalid[ix] = read_status if len(ixs_invalid): await sqlite.process_invalid_entries(db_file, ixs_invalid) # TODO return number of archived entries and add if statement to run archive maintainence function await sqlite.maintain_archive(db_file, limit) # await sqlite.process_invalid_entries(db_file, ixs) await asyncio.sleep(60 * 2) val = Config.get_setting_value(self, jid_bare, 'check') await asyncio.sleep(60 * float(val)) # Schedule to call this function again in 90 minutes # loop.call_at( # loop.time() + 60 * 90, # loop.create_task, # self.check_updates(jid) # ) # Consider an endless loop. See XmppPubsubTask.loop_task # def restart_task(self, jid_bare): async def loop_task(self, jid_bare): await asyncio.sleep(60) while True: logger.info('Looping task "check" for JID {}'.format(jid_bare)) print('Looping task "check" for JID {}'.format(jid_bare)) await FeedTask.check_updates(self, jid_bare) await asyncio.sleep(60 * 60) def restart_task(self, jid_bare): if jid_bare not in self.task_manager: self.task_manager[jid_bare] = {} logger.info('Creating new task manager for JID {}'.format(jid_bare)) logger.info('Stopping task "check" for JID {}'.format(jid_bare)) try: self.task_manager[jid_bare]['check'].cancel() except: logger.info('No task "check" for JID {} (FeedTask.check_updates)' .format(jid_bare)) logger.info('Starting tasks "check" for JID {}'.format(jid_bare)) self.task_manager[jid_bare]['check'] = asyncio.create_task( FeedTask.check_updates(self, jid_bare)) class Opml: # TODO Consider adding element jid as a pointer of import def export_to_file(jid, filename, results): # print(jid, filename, results) function_name = sys._getframe().f_code.co_name logger.debug('{} jid: {} filename: {}' .format(function_name, jid, filename)) root = ETR.Element("opml") root.set("version", "1.0") head = ETR.SubElement(root, "head") ETR.SubElement(head, "title").text = "{}".format(jid) ETR.SubElement(head, "description").text = ( "Set of subscriptions exported by Slixfeed") ETR.SubElement(head, "generator").text = "Slixfeed" ETR.SubElement(head, "urlPublic").text = ( "https://slixfeed.woodpeckersnest.space/") time_stamp = DateAndTime.current_time() ETR.SubElement(head, "dateCreated").text = time_stamp ETR.SubElement(head, "dateModified").text = time_stamp body = ETR.SubElement(root, "body") for result in results: outline = ETR.SubElement(body, "outline") outline.set("text", result[1]) outline.set("xmlUrl", result[2]) # outline.set("type", result[2]) tree = ETR.ElementTree(root) tree.write(filename) async def import_from_file(db_file, result): function_name = sys._getframe().f_code.co_name logger.debug('{}: db_file: {}' .format(function_name, db_file)) if not result['error']: document = result['content'] root = ETR.fromstring(document) before = sqlite.get_number_of_items(db_file, 'feeds_properties') feeds = [] for child in root.findall(".//outline"): url = child.get("xmlUrl") title = child.get("text") # feed = (url, title) # feeds.extend([feed]) feed = { 'title' : title, 'url' : url, } feeds.extend([feed]) await sqlite.import_feeds(db_file, feeds) await sqlite.add_metadata(db_file) after = sqlite.get_number_of_items(db_file, 'feeds_properties') difference = int(after) - int(before) return difference