From 77ac4c0ed9da11689d90da9605903185a6a3e621 Mon Sep 17 00:00:00 2001 From: "Schimon Jehudah, Adv." Date: Wed, 13 Nov 2024 16:41:33 +0200 Subject: [PATCH] Modularize code; Update document README. --- README.md | 57 +- blasta/__init__.py | 3 + blasta/__main__.py | 61 + {graphic => blasta/assets/graphic}/blasta.svg | 0 .../assets/graphic}/blasta_syndicate.svg | 0 blasta/assets/graphic/syndicate.svg | 19 + {graphic => blasta/assets/graphic}/xmpp.svg | 0 {script => blasta/assets/script}/regulator.js | 0 .../assets/stylesheet}/stylesheet.css | 0 .../assets/stylesheet}/stylesheet.xsl | 0 .../assets/template}/about.xhtml | 43 +- .../assets/template}/ask.xhtml | 0 .../assets/template}/atomsub.xhtml | 8 +- .../assets/template}/browse.atom | 0 .../assets/template}/browse.xhtml | 0 .../assets/template}/connect.xhtml | 2 +- .../assets/template}/contact.xhtml | 0 .../assets/template}/edit.xhtml | 0 .../assets/template}/feeds.xhtml | 0 .../assets/template}/folksonomy.xhtml | 0 .../assets/template}/help.xhtml | 0 .../assets/template}/ideas.xhtml | 0 .../assets/template}/libervia.xhtml | 0 .../assets/template}/movim.xhtml | 4 +- .../assets/template}/now.xhtml | 0 .../assets/template}/people.xhtml | 0 .../assets/template}/philosophy.xhtml | 0 .../assets/template}/profile.xhtml | 0 .../assets/template}/projects.xhtml | 0 .../assets/template}/pubsub.xhtml | 18 +- .../assets/template}/questions.xhtml | 0 .../assets/template}/register.xhtml | 5 +- .../assets/template}/result.xhtml | 0 .../assets/template}/search.xhtml | 0 .../assets/template}/software.xhtml | 0 .../assets/template}/syndication.xhtml | 0 .../assets/template}/tag.xhtml | 0 .../assets/template}/thanks.xhtml | 0 .../assets/template}/utilities.xhtml | 0 .../assets/template}/xmpp.xhtml | 0 blasta/config.py | 116 + blasta/helpers/data.py | 257 ++ blasta/helpers/utilities.py | 27 + blasta/helpers/xml.py | 16 + blasta.py => blasta/http/instance.py | 3313 +---------------- {screenshot => blasta/screenshot}/browse.png | Bin {screenshot => blasta/screenshot}/tag.png | Bin blasta/sqlite.py | 2482 ++++++++++++ blasta/version.py | 2 + blasta/xml/syndication.py | 88 + blasta/xmpp/instance.py | 31 + blasta/xmpp/message.py | 22 + blasta/xmpp/pubsub.py | 231 ++ data/README.txt | 1 - export/README.txt | 1 - graphic/syndicate.svg | 18 - items/README.txt | 1 - pyproject.toml | 60 + configuration.toml => settings.toml | 12 +- 59 files changed, 3609 insertions(+), 3289 deletions(-) create mode 100644 blasta/__init__.py create mode 100644 blasta/__main__.py rename {graphic => blasta/assets/graphic}/blasta.svg (100%) rename {graphic => blasta/assets/graphic}/blasta_syndicate.svg (100%) create mode 100644 blasta/assets/graphic/syndicate.svg rename {graphic => blasta/assets/graphic}/xmpp.svg (100%) rename {script => blasta/assets/script}/regulator.js (100%) rename {stylesheet => blasta/assets/stylesheet}/stylesheet.css (100%) rename {stylesheet => blasta/assets/stylesheet}/stylesheet.xsl (100%) rename {template => blasta/assets/template}/about.xhtml (91%) rename {template => blasta/assets/template}/ask.xhtml (100%) rename {template => blasta/assets/template}/atomsub.xhtml (98%) rename {template => blasta/assets/template}/browse.atom (100%) rename {template => blasta/assets/template}/browse.xhtml (100%) rename {template => blasta/assets/template}/connect.xhtml (98%) rename {template => blasta/assets/template}/contact.xhtml (100%) rename {template => blasta/assets/template}/edit.xhtml (100%) rename {template => blasta/assets/template}/feeds.xhtml (100%) rename {template => blasta/assets/template}/folksonomy.xhtml (100%) rename {template => blasta/assets/template}/help.xhtml (100%) rename {template => blasta/assets/template}/ideas.xhtml (100%) rename {template => blasta/assets/template}/libervia.xhtml (100%) rename {template => blasta/assets/template}/movim.xhtml (98%) rename {template => blasta/assets/template}/now.xhtml (100%) rename {template => blasta/assets/template}/people.xhtml (100%) rename {template => blasta/assets/template}/philosophy.xhtml (100%) rename {template => blasta/assets/template}/profile.xhtml (100%) rename {template => blasta/assets/template}/projects.xhtml (100%) rename {template => blasta/assets/template}/pubsub.xhtml (93%) rename {template => blasta/assets/template}/questions.xhtml (100%) rename {template => blasta/assets/template}/register.xhtml (95%) rename {template => blasta/assets/template}/result.xhtml (100%) rename {template => blasta/assets/template}/search.xhtml (100%) rename {template => blasta/assets/template}/software.xhtml (100%) rename {template => blasta/assets/template}/syndication.xhtml (100%) rename {template => blasta/assets/template}/tag.xhtml (100%) rename {template => blasta/assets/template}/thanks.xhtml (100%) rename {template => blasta/assets/template}/utilities.xhtml (100%) rename {template => blasta/assets/template}/xmpp.xhtml (100%) create mode 100644 blasta/config.py create mode 100644 blasta/helpers/data.py create mode 100644 blasta/helpers/utilities.py create mode 100644 blasta/helpers/xml.py rename blasta.py => blasta/http/instance.py (51%) rename {screenshot => blasta/screenshot}/browse.png (100%) rename {screenshot => blasta/screenshot}/tag.png (100%) create mode 100644 blasta/sqlite.py create mode 100644 blasta/version.py create mode 100644 blasta/xml/syndication.py create mode 100644 blasta/xmpp/instance.py create mode 100644 blasta/xmpp/message.py create mode 100644 blasta/xmpp/pubsub.py delete mode 100644 data/README.txt delete mode 100644 export/README.txt delete mode 100644 graphic/syndicate.svg delete mode 100644 items/README.txt create mode 100644 pyproject.toml rename configuration.toml => settings.toml (77%) diff --git a/README.md b/README.md index c92d584..14f9f3f 100644 --- a/README.md +++ b/README.md @@ -17,8 +17,8 @@ desire. ## Screenshots -[browse view](screenshot/browse.png) -[tags view](screenshot/tag.png) +[browse view](blasta/screenshot/browse.png) +[tags view](blasta/screenshot/tag.png) ## Technicalities @@ -61,21 +61,60 @@ The connection to the Blasta system is made with XMPP accounts. * Python >= 3.5 * fastapi * lxml +* python-dateutil +* python-multipart * slixmpp * tomllib (Python <= 3.10) * uvicorn -## Instructions +## Installation -Use the following commands to start Blasta. +It is possible to install Blasta using pip and pipx. -```shell -$ git clone https://git.xmpp-it.net/sch/Blasta -$ cd Blasta/ -$ python -m uvicorn blasta:app --reload +#### pip inside venv + +``` +$ python3 -m venv .venv +$ source .venv/bin/activate ``` -Open URL http://localhost:8000/ and connect with your Jabber ID. +##### Install + +``` +$ pip install git+https://git.xmpp-it.net/sch/Blasta +``` + +#### pipx + +##### Install + +``` +$ pipx install git+https://git.xmpp-it.net/sch/Blasta +``` + +##### Update + +``` +$ pipx uninstall blasta +$ pipx install git+https://git.xmpp-it.net/sch/Blasta +``` + +### Configure + +Copy file`settings.toml` to `~/.config/blasta/`. + +Copy directories `stylesheet`, `graphic`, `template`, and `script` to +`~/.local/share/blasta/`. + +Create directories `data`, `export`, `items` under `~/.cache/blasta/`. + +### Start + +``` +$ blasta +``` + +Open URL http://localhost:8000 and connect with your Jabber ID. ## License diff --git a/blasta/__init__.py b/blasta/__init__.py new file mode 100644 index 0000000..355b2ed --- /dev/null +++ b/blasta/__init__.py @@ -0,0 +1,3 @@ +from blasta.version import __version__, __version_info__ + +print('Blasta', __version__) diff --git a/blasta/__main__.py b/blasta/__main__.py new file mode 100644 index 0000000..a22f789 --- /dev/null +++ b/blasta/__main__.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" + +TODO + +* Delete cookie if session does not match + +* Delete entry/tag/jid combination row upon removal of a tag. + +""" + +import argparse +from blasta.http.instance import HttpInstance +from blasta.sqlite import SQLite +import json +import logging +from os.path import getsize, exists +import sys +import time +from typing import Optional +import urllib.parse +import uvicorn +import webbrowser + + +try: + import tomllib +except: + import tomli as tomllib + + +def main(): + if not exists('main.sqlite') or not getsize('main.sqlite'): + SQLite.instantiate_database('main.sqlite') + accounts = {} + sessions = {} + http_instance = HttpInstance(accounts, sessions) + return http_instance.app + +app = main() + +# FIXME +if __name__ == 'blasta.__main__': + parser = argparse.ArgumentParser( + prog='blasta', + description='Blasta - A collaborative annotation system.', + usage='%(prog)s [OPTION]...') + parser.add_argument('-v', '--version', help='print version', + action='version', version='0.1') + parser.add_argument('-p', '--port', help='port number', dest='port') + parser.add_argument('-o', '--open', help='open an html browser', action='store_const', const=True, dest='open') + args = parser.parse_args() + port = args.port if args.port else 8000 + uvicorn.run(app, host='localhost', port=port) + if args.open: + # TODO Check first time + webbrowser.open('http://localhost:{}/help/about'.format(port)) + webbrowser.open_new_tab('http://localhost:{}'.format(port)) + diff --git a/graphic/blasta.svg b/blasta/assets/graphic/blasta.svg similarity index 100% rename from graphic/blasta.svg rename to blasta/assets/graphic/blasta.svg diff --git a/graphic/blasta_syndicate.svg b/blasta/assets/graphic/blasta_syndicate.svg similarity index 100% rename from graphic/blasta_syndicate.svg rename to blasta/assets/graphic/blasta_syndicate.svg diff --git a/blasta/assets/graphic/syndicate.svg b/blasta/assets/graphic/syndicate.svg new file mode 100644 index 0000000..907df8b --- /dev/null +++ b/blasta/assets/graphic/syndicate.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/graphic/xmpp.svg b/blasta/assets/graphic/xmpp.svg similarity index 100% rename from graphic/xmpp.svg rename to blasta/assets/graphic/xmpp.svg diff --git a/script/regulator.js b/blasta/assets/script/regulator.js similarity index 100% rename from script/regulator.js rename to blasta/assets/script/regulator.js diff --git a/stylesheet/stylesheet.css b/blasta/assets/stylesheet/stylesheet.css similarity index 100% rename from stylesheet/stylesheet.css rename to blasta/assets/stylesheet/stylesheet.css diff --git a/stylesheet/stylesheet.xsl b/blasta/assets/stylesheet/stylesheet.xsl similarity index 100% rename from stylesheet/stylesheet.xsl rename to blasta/assets/stylesheet/stylesheet.xsl diff --git a/template/about.xhtml b/blasta/assets/template/about.xhtml similarity index 91% rename from template/about.xhtml rename to blasta/assets/template/about.xhtml index 8f832ba..3d1a638 100644 --- a/template/about.xhtml +++ b/blasta/assets/template/about.xhtml @@ -76,7 +76,9 @@ » Information and resources about Blasta, collaborative bookmarks with an Irish manner.

-

About Blasta

+

+ About Blasta +

Blasta is a collaborative bookmarks manager for organizing online content. It allows you to add links to your personal @@ -114,7 +116,12 @@ monero, mms, news, sip, udp, xmpp and any scheme and type that you desire.

-

Why Blasta?

+

+ Blasta was inspired by projects Movim and Rivista. +

+

+ Why Blasta? +

Corporate search engines are archaic and outdated, and often prioritize their own interests, leading to censorship and @@ -128,14 +135,18 @@ references and resources that you need in order to be productive and get that you need.

-

The things that you can do with Blasta are endless

+

+ The things that you can do with Blasta are endless +

Blasta is an open-ended indexing system, and, as such, it provides a versatile platform with which you have the ability to tailor its usage according to your desired preferences. Learn more.

-

The difference from other services

+

+ The difference from other services +

Unlike some so called "social" bookmarking systems, Blasta does not own your information; your bookmarks are @@ -151,7 +162,9 @@ your personal XMPP account under PubSub node urn:xmpp:bibliography:0.

-

Information that is stored by Blasta

+

+ Information that is stored by Blasta +

In order for Blasta to facilitate sharing of information and accessibility to information, Blasta aggregates your own @@ -166,14 +179,18 @@ all of their owners as private and no one else has stored them in a public fashion (i.e. not classified private).

-

Blasta source code

+

+ Blasta source code +

The source code of Blasta is available under the terms of the license AGPL-3.0 at git.xmpp-it.net.

-

Our motives

+

+ Our motives +

We are adopting the attitude towards life and towards death, which was implicit in the old Vikings' and in Schopenhauer's @@ -186,7 +203,9 @@ particular for and through his racial community, which is eternal.

-

About us

+

+ About us +

Blasta was proudly made in the Republic of Ireland, by a group of bible loving, religious, and stylish Irish men, who @@ -200,12 +219,16 @@ proceeding year, and he was the one who has initiated the idea of XMPP PubSub bookmarks.

-

Conclusion

+

+ Conclusion +

Blasta is for you to enjoy, excite, instigate, investigate, learn and research.

-

We hope you would have productive outcomes with Blasta.

+

+ We hope you would have productive outcomes with Blasta. +


“All you can take with you; is that which you have given diff --git a/template/ask.xhtml b/blasta/assets/template/ask.xhtml similarity index 100% rename from template/ask.xhtml rename to blasta/assets/template/ask.xhtml diff --git a/template/atomsub.xhtml b/blasta/assets/template/atomsub.xhtml similarity index 98% rename from template/atomsub.xhtml rename to blasta/assets/template/atomsub.xhtml index f6f9658..509be89 100644 --- a/template/atomsub.xhtml +++ b/blasta/assets/template/atomsub.xhtml @@ -168,7 +168,7 @@ xmpp.org ​  - + libervia.org

@@ -188,13 +188,15 @@ xmpp.org ​  - + movim.eu

-

Of note

+

+ Of note +

These type of technologies are public information for over a couple of decades (i.e. more than 20 years); and people diff --git a/template/browse.atom b/blasta/assets/template/browse.atom similarity index 100% rename from template/browse.atom rename to blasta/assets/template/browse.atom diff --git a/template/browse.xhtml b/blasta/assets/template/browse.xhtml similarity index 100% rename from template/browse.xhtml rename to blasta/assets/template/browse.xhtml diff --git a/template/connect.xhtml b/blasta/assets/template/connect.xhtml similarity index 98% rename from template/connect.xhtml rename to blasta/assets/template/connect.xhtml index 6c9cb19..2499db9 100644 --- a/template/connect.xhtml +++ b/blasta/assets/template/connect.xhtml @@ -69,7 +69,7 @@

- Log in to Blasta with your XMPP account or + Connect to Blasta with your XMPP account or register for an account.

diff --git a/template/contact.xhtml b/blasta/assets/template/contact.xhtml similarity index 100% rename from template/contact.xhtml rename to blasta/assets/template/contact.xhtml diff --git a/template/edit.xhtml b/blasta/assets/template/edit.xhtml similarity index 100% rename from template/edit.xhtml rename to blasta/assets/template/edit.xhtml diff --git a/template/feeds.xhtml b/blasta/assets/template/feeds.xhtml similarity index 100% rename from template/feeds.xhtml rename to blasta/assets/template/feeds.xhtml diff --git a/template/folksonomy.xhtml b/blasta/assets/template/folksonomy.xhtml similarity index 100% rename from template/folksonomy.xhtml rename to blasta/assets/template/folksonomy.xhtml diff --git a/template/help.xhtml b/blasta/assets/template/help.xhtml similarity index 100% rename from template/help.xhtml rename to blasta/assets/template/help.xhtml diff --git a/template/ideas.xhtml b/blasta/assets/template/ideas.xhtml similarity index 100% rename from template/ideas.xhtml rename to blasta/assets/template/ideas.xhtml diff --git a/template/libervia.xhtml b/blasta/assets/template/libervia.xhtml similarity index 100% rename from template/libervia.xhtml rename to blasta/assets/template/libervia.xhtml diff --git a/template/movim.xhtml b/blasta/assets/template/movim.xhtml similarity index 98% rename from template/movim.xhtml rename to blasta/assets/template/movim.xhtml index 25f09f6..6f19945 100644 --- a/template/movim.xhtml +++ b/blasta/assets/template/movim.xhtml @@ -226,7 +226,9 @@


- Blasta was inspired by Movim and Rivista. + “Talent hits a target no one else can hit. + Genius hits a target no one else can see.” + ― Arthur Schopenhauer

diff --git a/template/now.xhtml b/blasta/assets/template/now.xhtml similarity index 100% rename from template/now.xhtml rename to blasta/assets/template/now.xhtml diff --git a/template/people.xhtml b/blasta/assets/template/people.xhtml similarity index 100% rename from template/people.xhtml rename to blasta/assets/template/people.xhtml diff --git a/template/philosophy.xhtml b/blasta/assets/template/philosophy.xhtml similarity index 100% rename from template/philosophy.xhtml rename to blasta/assets/template/philosophy.xhtml diff --git a/template/profile.xhtml b/blasta/assets/template/profile.xhtml similarity index 100% rename from template/profile.xhtml rename to blasta/assets/template/profile.xhtml diff --git a/template/projects.xhtml b/blasta/assets/template/projects.xhtml similarity index 100% rename from template/projects.xhtml rename to blasta/assets/template/projects.xhtml diff --git a/template/pubsub.xhtml b/blasta/assets/template/pubsub.xhtml similarity index 93% rename from template/pubsub.xhtml rename to blasta/assets/template/pubsub.xhtml index 33bfbea..6becb42 100644 --- a/template/pubsub.xhtml +++ b/blasta/assets/template/pubsub.xhtml @@ -278,11 +278,19 @@
-

- “Talent hits a target no one else can hit. - Genius hits a target no one else can see.” - ― Arthur Schopenhauer +

+ “Technology is extremely powerful and has the potential to + change the world; however, it cannot realize its full + potential unless people feel the need to use it. Some + researchers agree, that to ensure the success of new + technology, the focus should be on the people’s perspective + rather than on the technology itself. Designing a new + experience is a process that facilitates the relationship + between technology and people; thus, balanced research + should be conducted from both perspectives.” + ― + DIVA.EXCHANGE +

diff --git a/template/questions.xhtml b/blasta/assets/template/questions.xhtml similarity index 100% rename from template/questions.xhtml rename to blasta/assets/template/questions.xhtml diff --git a/template/register.xhtml b/blasta/assets/template/register.xhtml similarity index 95% rename from template/register.xhtml rename to blasta/assets/template/register.xhtml index a8878e3..5a03118 100644 --- a/template/register.xhtml +++ b/blasta/assets/template/register.xhtml @@ -76,8 +76,9 @@

As with email, you need an account with a service provider - to operate Blasta, so if you already have an XMPP account, - you can connect and start to Blasta. + to utilize Blasta; if you already have an XMPP account, you + can connect and start to utilize + Blasta.

If you do not have an XMPP account, yet, you can use a diff --git a/template/result.xhtml b/blasta/assets/template/result.xhtml similarity index 100% rename from template/result.xhtml rename to blasta/assets/template/result.xhtml diff --git a/template/search.xhtml b/blasta/assets/template/search.xhtml similarity index 100% rename from template/search.xhtml rename to blasta/assets/template/search.xhtml diff --git a/template/software.xhtml b/blasta/assets/template/software.xhtml similarity index 100% rename from template/software.xhtml rename to blasta/assets/template/software.xhtml diff --git a/template/syndication.xhtml b/blasta/assets/template/syndication.xhtml similarity index 100% rename from template/syndication.xhtml rename to blasta/assets/template/syndication.xhtml diff --git a/template/tag.xhtml b/blasta/assets/template/tag.xhtml similarity index 100% rename from template/tag.xhtml rename to blasta/assets/template/tag.xhtml diff --git a/template/thanks.xhtml b/blasta/assets/template/thanks.xhtml similarity index 100% rename from template/thanks.xhtml rename to blasta/assets/template/thanks.xhtml diff --git a/template/utilities.xhtml b/blasta/assets/template/utilities.xhtml similarity index 100% rename from template/utilities.xhtml rename to blasta/assets/template/utilities.xhtml diff --git a/template/xmpp.xhtml b/blasta/assets/template/xmpp.xhtml similarity index 100% rename from template/xmpp.xhtml rename to blasta/assets/template/xmpp.xhtml diff --git a/blasta/config.py b/blasta/config.py new file mode 100644 index 0000000..47d34b3 --- /dev/null +++ b/blasta/config.py @@ -0,0 +1,116 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +""" +Functions get_directory() were taken from project jarun/buku. +By Arun Prakash Jana (jarun) and Dmitry Marakasov (AMDmi3). +""" + +import os +import sys +try: + import tomllib +except: + import tomli as tomllib + +class Settings: + + def get_directory(): + """ + Determine the directory path where setting files be stored. + + * If $XDG_CONFIG_HOME is defined, use it; + * else if $HOME exists, use it; + * else if the platform is Windows, use %APPDATA%; + * else use the current directory. + + Returns + ------- + str + Path to configuration directory. + """ + # config_home = xdg.BaseDirectory.xdg_config_home + config_home = os.environ.get('XDG_CONFIG_HOME') + if config_home is None: + if os.environ.get('HOME') is None: + if sys.platform == 'win32': + config_home = os.environ.get('APPDATA') + if config_home is None: + return os.path.abspath('.') + else: + return os.path.abspath('.') + else: + config_home = os.path.join( + os.environ.get('HOME'), '.config' + ) + return os.path.join(config_home, 'blasta') + + def get_setting(filename, section): + with open(filename, mode="rb") as settings: + result = tomllib.load(settings)[section] + return result + + +class Share: + + def get_directory(): + """ + Determine the directory path where data files be stored. + + * If $XDG_DATA_HOME is defined, use it; + * else if $HOME exists, use it; + * else if the platform is Windows, use %APPDATA%; + * else use the current directory. + + Returns + ------- + str + Path to database file. + """ + # data_home = xdg.BaseDirectory.xdg_data_home + data_home = os.environ.get('XDG_DATA_HOME') + if data_home is None: + if os.environ.get('HOME') is None: + if sys.platform == 'win32': + data_home = os.environ.get('APPDATA') + if data_home is None: + return os.path.abspath('.blasta/data') + else: + return os.path.abspath('.blasta/data') + else: + data_home = os.path.join( + os.environ.get('HOME'), '.local', 'share' + ) + return os.path.join(data_home, 'blasta') + +class Cache: + + def get_directory(): + """ + Determine the directory path where cache files be stored. + + * If $XDG_CACHE_HOME is defined, use it; + * else if $HOME exists, use it; + * else if the platform is Windows, use %APPDATA%; + * else use the current directory. + + Returns + ------- + str + Path to cache directory. + """ + # cache_home = xdg.BaseDirectory.xdg_cache_home + cache_home = os.environ.get('XDG_CACHE_HOME') + if cache_home is None: + if os.environ.get('HOME') is None: + if sys.platform == 'win32': + cache_home = os.environ.get('APPDATA') + if cache_home is None: + return os.path.abspath('.blasta/cache') + else: + return os.path.abspath('.blasta/cache') + else: + cache_home = os.path.join( + os.environ.get('HOME'), '.cache' + ) + return os.path.join(cache_home, 'blasta') diff --git a/blasta/helpers/data.py b/blasta/helpers/data.py new file mode 100644 index 0000000..61e9cfa --- /dev/null +++ b/blasta/helpers/data.py @@ -0,0 +1,257 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from blasta.helpers.utilities import Utilities +from blasta.sqlite import SQLite +from blasta.xml.syndication import Syndication +from blasta.xmpp.pubsub import XmppPubsub +import os +from slixmpp.stanza.iq import Iq +import tomli_w + +try: + import tomllib +except: + import tomli as tomllib + +class Data: + + def cache_items_and_tags_search(directory_cache, entries, jid, query): + """Create a cache file of node items and tags.""" + item_ids = [] + tags = {} + for entry in entries: + entry_tags = entry['tags'] + entry_url_hash = entry['url_hash'] + tags_to_include = [] + if query in ' '.join([entry['title'], entry['link'], entry['summary'], ' '.join(entry_tags)]): + item_ids.append(entry_url_hash) + tags_to_include += entry_tags + for tag_to_include in tags_to_include: + tags[tag_to_include] = tags[tag_to_include]+1 if tag_to_include in tags else 1 + if tags: + tags = dict(sorted(tags.items(), key=lambda item: (-item[1], item[0]))) + tags = dict(list(tags.items())[:30]) + if item_ids: + filename = os.path.join(directory_cache, 'data', jid + '_query.toml') + data = { + 'item_ids' : item_ids, + 'tags' : tags} + Data.save_to_toml(filename, data) + + def cache_items_and_tags_filter(directory_cache, entries, jid, tag): + """Create a cache file of node items and tags.""" + item_ids = [] + tags = {} + for entry in entries: + entry_tags = entry['tags'] + entry_url_hash = entry['url_hash'] + tags_to_include = [] + if tag in entry_tags: + item_ids.append(entry_url_hash) + tags_to_include += entry_tags + for tag_to_include in tags_to_include: + tags[tag_to_include] = tags[tag_to_include]+1 if tag_to_include in tags else 1 + if tags: + tags = dict(sorted(tags.items(), key=lambda item: (-item[1], item[0]))) + tags = dict(list(tags.items())[:30]) + del tags[tag] + if item_ids: + directory = os.path.join(directory_cache, 'data', jid) + if not os.path.exists(directory): + os.mkdir(directory) + filename = os.path.join(directory, tag) + # Add support for search query + #filename = 'data/{}/query:{}.toml'.format(jid, query) + #filename = 'data/{}/tag:{}.toml'.format(jid, tag) + data = { + 'item_ids' : item_ids, + 'tags' : tags} + Data.save_to_toml(filename, data) + + def cache_items_and_tags(directory_cache, entries, jid): + """Create a cache file of node items and tags.""" + item_ids = [] + tags = {} + for entry in entries: + entry_tags = entry['tags'] + entry_url_hash = entry['url_hash'] + tags_to_include = [] + item_ids.append(entry_url_hash) + tags_to_include += entry_tags + for tag_to_include in tags_to_include: + tags[tag_to_include] = tags[tag_to_include]+1 if tag_to_include in tags else 1 + if tags: + tags = dict(sorted(tags.items(), key=lambda item: (-item[1], item[0]))) + tags = dict(list(tags.items())[:30]) + if item_ids: + filename = os.path.join(directory_cache, 'data', jid + '.toml') + data = { + 'item_ids' : item_ids, + 'tags' : tags} + Data.save_to_toml(filename, data) + + def extract_iq_items(iq, jabber_id): + iq_items = iq['pubsub']['items'] + entries = [] + name = jabber_id.split('@')[0] + for iq_item in iq_items: + item_payload = iq_item['payload'] + entry = Syndication.extract_items(item_payload) + entries.append(entry) + # TODO Handle this with XEP-0059 (reverse: bool), instead of reversing it. + entries.reverse() + return entries + + def extract_iq_items_extra(iq, jabber_id, limit=None): + iq_items = iq['pubsub']['items'] + entries = [] + name = jabber_id.split('@')[0] + for iq_item in iq_items: + item_payload = iq_item['payload'] + entry = Syndication.extract_items(item_payload, limit) + url_hash = Utilities.hash_url_to_md5(entry['link']) + iq_item_id = iq_item['id'] + if iq_item_id != url_hash: + logging.error('Item ID does not match MD5. id: {} hash: {}'.format(iq_item_id, url_hash)) + logging.warn('Item ID does not match MD5. id: {} hash: {}'.format(iq_item_id, url_hash)) + db_file = 'main.sqlite' + instances = SQLite.get_entry_instances_by_url_hash(db_file, url_hash) + if entry: + entry['instances'] = instances or 0 + entry['jid'] = jabber_id + entry['name'] = name + entry['url_hash'] = url_hash + entries.append(entry) + # TODO Handle this with XEP-0059 (reverse: bool), instead of reversing it. + entries.reverse() + result = entries + return result + + def open_file_toml(filename: str) -> dict: + with open(filename, mode="rb") as fn: + data = tomllib.load(fn) + return data + + def organize_tags(tags): + tags_organized = [] + tags = tags.split(',') + #tags = sorted(set(tags)) + for tag in tags: + if tag: + tag = tag.lower().strip() + if tag not in tags_organized: + tags_organized.append(tag) + return sorted(tags_organized) + + def remove_item_from_cache(directory_cache, jabber_id, node, url_hash): + filename_items = os.path.join(directory_cache, 'items', jabber_id + '.toml') + entries_cache = Data.open_file_toml(filename_items) + if node in entries_cache: + entries_cache_node = entries_cache[node] + for entry_cache in entries_cache_node: + if entry_cache['url_hash'] == url_hash: + entry_cache_index = entries_cache_node.index(entry_cache) + del entries_cache_node[entry_cache_index] + break + data_items = entries_cache + Data.save_to_toml(filename_items, data_items) + + def save_to_json(filename: str, data) -> None: + with open(filename, 'w') as f: + json.dump(data, f) + + def save_to_toml(filename: str, data: dict) -> None: + with open(filename, 'w') as fn: + data_as_string = tomli_w.dumps(data) + fn.write(data_as_string) + + async def update_cache_and_database(directory_cache, xmpp_instance, jabber_id: str, node_type: str, node_id: str): + # Download identifiers of node items. + iq = await XmppPubsub.get_node_item_ids(xmpp_instance, jabber_id, node_id) + if isinstance(iq, Iq): + iq_items_remote = iq['disco_items'] + + # Cache a list of identifiers of node items to a file. + iq_items_remote_name = [] + for iq_item_remote in iq_items_remote: + iq_item_remote_name = iq_item_remote['name'] + iq_items_remote_name.append(iq_item_remote_name) + + #data_item_ids = {'iq_items' : iq_items_remote_name} + #filename_item_ids = 'item_ids/' + jabber_id + '.toml' + #Data.save_to_toml(filename_item_ids, data_item_ids) + + filename_items = os.path.join(directory_cache, 'items', jabber_id + '.toml') + if not os.path.exists(filename_items) or os.path.getsize(filename_items) in (0, 13): + iq = await XmppPubsub.get_node_items(xmpp_instance, jabber_id, node_id) + if isinstance(iq, Iq): + entries_cache_node = Data.extract_iq_items_extra(iq, jabber_id) + data_items = {node_type : entries_cache_node} + Data.save_to_toml(filename_items, data_items) + return ['fine', iq] # TODO Remove this line + else: + return ['error', iq] + else: + entries_cache = Data.open_file_toml(filename_items) + if not node_type in entries_cache: return ['error', 'Directory "{}" is empty'. format(node_type)] + entries_cache_node = entries_cache[node_type] + db_file = 'main.sqlite' + + # Check whether items still exist on node + for entry in entries_cache_node: + iq_item_remote_exist = False + url_hash = None + for url_hash in iq_items_remote_name: + if url_hash == entry['url_hash']: + iq_item_remote_exist = True + break + if url_hash and not iq_item_remote_exist: + await SQLite.delete_combination_row_by_jid_and_url_hash( + db_file, url_hash, jabber_id) + entry_index = entries_cache_node.index(entry) + del entries_cache_node[entry_index] + + # Check for new items on node + entries_cache_node_new = [] + for url_hash in iq_items_remote_name: + iq_item_local_exist = False + for entry in entries_cache_node: + if url_hash == entry['url_hash']: + iq_item_local_exist = True + break + if not iq_item_local_exist: + iq = await XmppPubsub.get_node_item( + xmpp_instance, jabber_id, node_id, url_hash) + if isinstance(iq, Iq): + entries_iq = Data.extract_iq_items_extra(iq, jabber_id) + entries_cache_node_new += entries_iq + else: + # TODO + # Handle this concern in a different fashion, + # instead of stopping the whole operation. + return ['error', iq] + entries_cache_node += entries_cache_node_new + + if node_type == 'public': + # Fast (low I/O) + if not SQLite.get_jid_id_by_jid(db_file, jabber_id): + await SQLite.set_jid(db_file, jabber_id) + #await SQLite.add_new_entries(db_file, entries) + await SQLite.add_tags(db_file, entries_cache_node) + # Slow (high I/O) + for entry in entries_cache_node: + url_hash = entry['url_hash'] + if not SQLite.get_entry_id_by_url_hash(db_file, url_hash): + await SQLite.add_new_entries(db_file, entries_cache_node) + await SQLite.associate_entries_tags_jids(db_file, entry) + #elif not SQLite.is_jid_associated_with_url_hash(db_file, jabber_id, url_hash): + # await SQLite.associate_entries_tags_jids(db_file, entry) + else: + await SQLite.associate_entries_tags_jids(db_file, entry) + + data_items = entries_cache + Data.save_to_toml(filename_items, data_items) + return ['fine', iq] # TODO Remove this line + else: + return ['error', iq] diff --git a/blasta/helpers/utilities.py b/blasta/helpers/utilities.py new file mode 100644 index 0000000..53f6054 --- /dev/null +++ b/blasta/helpers/utilities.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from datetime import datetime +import hashlib + +class Utilities: + + def convert_iso8601_to_readable(timestamp): + old_date_format = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + new_date_format = old_date_format.strftime("%B %d, %Y") + return new_date_format + + def hash_url_to_md5(url): + url_encoded = url.encode() + url_hashed = hashlib.md5(url_encoded) + url_digest = url_hashed.hexdigest() + return url_digest + + def is_jid_matches_to_session(accounts, sessions, request): + jabber_id = request.cookies.get('jabber_id') + session_key = request.cookies.get('session_key') + if (jabber_id and + jabber_id in accounts and + jabber_id in sessions and + session_key == sessions[jabber_id]): + return jabber_id diff --git a/blasta/helpers/xml.py b/blasta/helpers/xml.py new file mode 100644 index 0000000..4d66663 --- /dev/null +++ b/blasta/helpers/xml.py @@ -0,0 +1,16 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +class Xml: + + def create_setting_entry(xmpp_instance, key : str, value : str): + form = xmpp_instance['xep_0004'].make_form('form', 'Settings') + form['type'] = 'result' + form.add_field(var=key, + value=value) + return form + +# def create_setting_entry(value : str): +# element = ET.Element('value') +# element.text = value +# return element diff --git a/blasta.py b/blasta/http/instance.py similarity index 51% rename from blasta.py rename to blasta/http/instance.py index 17687c0..dff01da 100644 --- a/blasta.py +++ b/blasta/http/instance.py @@ -1,311 +1,58 @@ -#!/usr/bin/env python3 +#!/usr/bin/python # -*- coding: utf-8 -*- -""" - -TODO - -* Delete cookie if session does not match - -* Delete entry/tag/jid combination row upon removal of a tag. - -""" - -import argparse import asyncio -from asyncio import Lock +from blasta.config import Cache, Settings, Share +from blasta.helpers.data import Data +from blasta.helpers.utilities import Utilities +from blasta.sqlite import SQLite +from blasta.xml.syndication import Syndication +from blasta.xmpp.instance import XmppInstance +from blasta.xmpp.pubsub import XmppPubsub from datetime import datetime from fastapi import Cookie, FastAPI, File, Form, HTTPException, Request, Response, UploadFile from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import FileResponse, HTMLResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates -import hashlib -import json -import logging -from os import mkdir -from os.path import getsize, exists +import os import random -import slixmpp -from slixmpp import ClientXMPP -from slixmpp.exceptions import IqError, IqTimeout -import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub -import slixmpp.plugins.xep_0059.rsm as rsm -from sqlite3 import connect, Error, IntegrityError +from slixmpp.stanza.iq import Iq from starlette.responses import RedirectResponse -import sys -import time -import tomli_w -from typing import Optional -import urllib.parse -import uvicorn -import webbrowser import xml.etree.ElementTree as ET - -try: - import tomllib -except: - import tomli as tomllib - -DBLOCK = Lock() - -class Data: - - def cache_items_and_tags_search(entries, jid, query): - """Create a cache file of node items and tags.""" - item_ids = [] - tags = {} - for entry in entries: - entry_tags = entry['tags'] - entry_url_hash = entry['url_hash'] - tags_to_include = [] - if query in ' '.join([entry['title'], entry['link'], entry['summary'], ' '.join(entry_tags)]): - item_ids.append(entry_url_hash) - tags_to_include += entry_tags - for tag_to_include in tags_to_include: - tags[tag_to_include] = tags[tag_to_include]+1 if tag_to_include in tags else 1 - if tags: - tags = dict(sorted(tags.items(), key=lambda item: (-item[1], item[0]))) - tags = dict(list(tags.items())[:30]) - if item_ids: - filename = 'data/{}_query.toml'.format(jid) - data = { - 'item_ids' : item_ids, - 'tags' : tags} - Data.save_to_toml(filename, data) - - def cache_items_and_tags_filter(entries, jid, tag): - """Create a cache file of node items and tags.""" - item_ids = [] - tags = {} - for entry in entries: - entry_tags = entry['tags'] - entry_url_hash = entry['url_hash'] - tags_to_include = [] - if tag in entry_tags: - item_ids.append(entry_url_hash) - tags_to_include += entry_tags - for tag_to_include in tags_to_include: - tags[tag_to_include] = tags[tag_to_include]+1 if tag_to_include in tags else 1 - if tags: - tags = dict(sorted(tags.items(), key=lambda item: (-item[1], item[0]))) - tags = dict(list(tags.items())[:30]) - del tags[tag] - if item_ids: - directory = 'data/{}/'.format(jid) - if not exists(directory): - mkdir(directory) - filename = 'data/{}/{}.toml'.format(jid, tag) - # Add support for search query - #filename = 'data/{}/query:{}.toml'.format(jid, query) - #filename = 'data/{}/tag:{}.toml'.format(jid, tag) - data = { - 'item_ids' : item_ids, - 'tags' : tags} - Data.save_to_toml(filename, data) - - def cache_items_and_tags(entries, jid): - """Create a cache file of node items and tags.""" - item_ids = [] - tags = {} - for entry in entries: - entry_tags = entry['tags'] - entry_url_hash = entry['url_hash'] - tags_to_include = [] - item_ids.append(entry_url_hash) - tags_to_include += entry_tags - for tag_to_include in tags_to_include: - tags[tag_to_include] = tags[tag_to_include]+1 if tag_to_include in tags else 1 - if tags: - tags = dict(sorted(tags.items(), key=lambda item: (-item[1], item[0]))) - tags = dict(list(tags.items())[:30]) - if item_ids: - filename = 'data/{}.toml'.format(jid) - data = { - 'item_ids' : item_ids, - 'tags' : tags} - Data.save_to_toml(filename, data) - - def extract_iq_items(iq, jabber_id): - iq_items = iq['pubsub']['items'] - entries = [] - name = jabber_id.split('@')[0] - for iq_item in iq_items: - item_payload = iq_item['payload'] - entry = Syndication.extract_items(item_payload) - entries.append(entry) - # TODO Handle this with XEP-0059 (reverse: bool), instead of reversing it. - entries.reverse() - return entries - - def extract_iq_items_extra(iq, jabber_id, limit=None): - iq_items = iq['pubsub']['items'] - entries = [] - name = jabber_id.split('@')[0] - for iq_item in iq_items: - item_payload = iq_item['payload'] - entry = Syndication.extract_items(item_payload, limit) - url_hash = Utilities.hash_url_to_md5(entry['link']) - iq_item_id = iq_item['id'] - if iq_item_id != url_hash: - logging.error('Item ID does not match MD5. id: {} hash: {}'.format(iq_item_id, url_hash)) - logging.warn('Item ID does not match MD5. id: {} hash: {}'.format(iq_item_id, url_hash)) - db_file = 'main.sqlite' - instances = SQLite.get_entry_instances_by_url_hash(db_file, url_hash) - if entry: - entry['instances'] = instances or 0 - entry['jid'] = jabber_id - entry['name'] = name - entry['url_hash'] = url_hash - entries.append(entry) - # TODO Handle this with XEP-0059 (reverse: bool), instead of reversing it. - entries.reverse() - result = entries - return result - - def open_file_toml(filename: str) -> dict: - with open(filename, mode="rb") as fn: - data = tomllib.load(fn) - return data - - def organize_tags(tags): - tags_organized = [] - tags = tags.split(',') - #tags = sorted(set(tags)) - for tag in tags: - if tag: - tag = tag.lower().strip() - if tag not in tags_organized: - tags_organized.append(tag) - return sorted(tags_organized) - - def remove_item_from_cache(jabber_id, node, url_hash): - filename_items = 'items/' + jabber_id + '.toml' - entries_cache = Data.open_file_toml(filename_items) - if node in entries_cache: - entries_cache_node = entries_cache[node] - for entry_cache in entries_cache_node: - if entry_cache['url_hash'] == url_hash: - entry_cache_index = entries_cache_node.index(entry_cache) - del entries_cache_node[entry_cache_index] - break - data_items = entries_cache - Data.save_to_toml(filename_items, data_items) - - def save_to_json(filename: str, data) -> None: - with open(filename, 'w') as f: - json.dump(data, f) - - def save_to_toml(filename: str, data: dict) -> None: - with open(filename, 'w') as fn: - data_as_string = tomli_w.dumps(data) - fn.write(data_as_string) - - async def update_cache_and_database(xmpp_instance, jabber_id: str, node_type: str, node_id: str): - # Download identifiers of node items. - iq = await XmppPubsub.get_node_item_ids(xmpp_instance, jabber_id, node_id) - if isinstance(iq, slixmpp.stanza.iq.Iq): - iq_items_remote = iq['disco_items'] - - # Cache a list of identifiers of node items to a file. - iq_items_remote_name = [] - for iq_item_remote in iq_items_remote: - iq_item_remote_name = iq_item_remote['name'] - iq_items_remote_name.append(iq_item_remote_name) - - #data_item_ids = {'iq_items' : iq_items_remote_name} - #filename_item_ids = 'item_ids/' + jabber_id + '.toml' - #Data.save_to_toml(filename_item_ids, data_item_ids) - - filename_items = 'items/' + jabber_id + '.toml' - if not exists(filename_items) or getsize(filename_items) == 13: - iq = await XmppPubsub.get_node_items(xmpp_instance, jabber_id, node_id) - if isinstance(iq, slixmpp.stanza.iq.Iq): - entries_cache_node = Data.extract_iq_items_extra(iq, jabber_id) - data_items = {node_type : entries_cache_node} - Data.save_to_toml(filename_items, data_items) - return ['fine', iq] # TODO Remove this line - else: - return ['error', iq] - else: - entries_cache = Data.open_file_toml(filename_items) - if not node_type in entries_cache: return ['error', 'Directory "{}" is empty'. format(node_type)] - entries_cache_node = entries_cache[node_type] - db_file = 'main.sqlite' - - # Check whether items still exist on node - for entry in entries_cache_node: - iq_item_remote_exist = False - url_hash = None - for url_hash in iq_items_remote_name: - if url_hash == entry['url_hash']: - iq_item_remote_exist = True - break - if url_hash and not iq_item_remote_exist: - await SQLite.delete_combination_row_by_jid_and_url_hash( - db_file, url_hash, jabber_id) - entry_index = entries_cache_node.index(entry) - del entries_cache_node[entry_index] - - # Check for new items on node - entries_cache_node_new = [] - for url_hash in iq_items_remote_name: - iq_item_local_exist = False - for entry in entries_cache_node: - if url_hash == entry['url_hash']: - iq_item_local_exist = True - break - if not iq_item_local_exist: - iq = await XmppPubsub.get_node_item( - xmpp_instance, jabber_id, node_id, url_hash) - if isinstance(iq, slixmpp.stanza.iq.Iq): - entries_iq = Data.extract_iq_items_extra(iq, jabber_id) - entries_cache_node_new += entries_iq - else: - # TODO - # Handle this concern in a different fashion, - # instead of stopping the whole operation. - return ['error', iq] - entries_cache_node += entries_cache_node_new - - if node_type == 'public': - # Fast (low I/O) - if not SQLite.get_jid_id_by_jid(db_file, jabber_id): - await SQLite.set_jid(db_file, jabber_id) - #await SQLite.add_new_entries(db_file, entries) - await SQLite.add_tags(db_file, entries_cache_node) - # Slow (high I/O) - for entry in entries_cache_node: - url_hash = entry['url_hash'] - if not SQLite.get_entry_id_by_url_hash(db_file, url_hash): - await SQLite.add_new_entries(db_file, entries_cache_node) - await SQLite.associate_entries_tags_jids(db_file, entry) - #elif not SQLite.is_jid_associated_with_url_hash(db_file, jabber_id, url_hash): - # await SQLite.associate_entries_tags_jids(db_file, entry) - else: - await SQLite.associate_entries_tags_jids(db_file, entry) - - data_items = entries_cache - Data.save_to_toml(filename_items, data_items) - return ['fine', iq] # TODO Remove this line - else: - return ['error', iq] - class HttpInstance: + def __init__(self, accounts, sessions): + directory_cache = Cache.get_directory() + directory_cache_data = os.path.join(directory_cache, 'data') + directory_cache_export = os.path.join(directory_cache, 'export') + directory_cache_items = os.path.join(directory_cache, 'items') + + self.directory_cache = directory_cache + + directory_data = Share.get_directory() + directory_data_graphic = os.path.join(directory_data, 'graphic') + directory_data_script = os.path.join(directory_data, 'script') + directory_data_stylesheet = os.path.join(directory_data, 'stylesheet') + directory_data_template = os.path.join(directory_data, 'template') + #filename_database = os.path.join(directory_data, 'main.sqlite') + db_file = os.path.join(directory_data, 'main.sqlite') + self.app = FastAPI() - templates = Jinja2Templates(directory='template') + templates = Jinja2Templates(directory=directory_data_template) - self.app.mount('/data', StaticFiles(directory='data'), name='data') - self.app.mount('/export', StaticFiles(directory='export'), name='export') - self.app.mount('/graphic', StaticFiles(directory='graphic'), name='graphic') - self.app.mount('/script', StaticFiles(directory='script'), name='script') - self.app.mount('/stylesheet', StaticFiles(directory='stylesheet'), name='stylesheet') + self.app.mount('/data', StaticFiles(directory=directory_cache_data), name='data') + self.app.mount('/export', StaticFiles(directory=directory_cache_export), name='export') + self.app.mount('/graphic', StaticFiles(directory=directory_data_graphic), name='graphic') + self.app.mount('/script', StaticFiles(directory=directory_data_script), name='script') + self.app.mount('/stylesheet', StaticFiles(directory=directory_data_stylesheet), name='stylesheet') - filename_configuration = 'configuration.toml' - data = Data.open_file_toml(filename_configuration) + directory_settings = Settings.get_directory() + filename_settings = os.path.join(directory_settings, 'settings.toml') + + data = Data.open_file_toml(filename_settings) contacts = data['contacts'] contact_email = contacts['email'] @@ -431,8 +178,15 @@ class HttpInstance: # httponly=False, # True # samesite='lax') + @self.app.exception_handler(401) + def not_authorized_exception_handler(request: Request, exc: HTTPException): + #jabber_id = Utilities.is_jid_matches_to_session(accounts, sessions, request) + message = 'Not Authorized.' + description = 'Not Authorized (401)' + return result_post(request, description, message) + @self.app.exception_handler(403) - def not_found_exception_handler(request: Request, exc: HTTPException): + def access_denied_exception_handler(request: Request, exc: HTTPException): jabber_id = Utilities.is_jid_matches_to_session(accounts, sessions, request) message = 'Blasta system message » Access denied.' description = 'Access denied (403)' @@ -456,7 +210,7 @@ class HttpInstance: return result_post(request, jabber_id, description, message, path) @self.app.exception_handler(500) - def internal_error__exception_handler(request: Request, exc: HTTPException): + def internal_error_exception_handler(request: Request, exc: HTTPException): jabber_id = Utilities.is_jid_matches_to_session(accounts, sessions, request) message = 'Blasta system message » Internal Server Error.' description = 'Internal error (500)' @@ -759,7 +513,7 @@ class HttpInstance: if jabber_id: xmpp_instance = accounts[jabber_id] node_id = nodes[node_type]['name'] - result, reason = await Data.update_cache_and_database(xmpp_instance, jabber_id, node_type, node_id) + result, reason = await Data.update_cache_and_database(directory_cache, xmpp_instance, jabber_id, node_type, node_id) if result == 'error': message = 'XMPP system message » {}.'.format(reason) description = 'IQ Error' @@ -816,7 +570,7 @@ class HttpInstance: xmpp_instance = accounts[jabber_id] # NOTE You need something other than an iterator (XEP-0059). # You need a PubSub key that would hold tags. - filename_items = 'items/' + jabber_id + '.toml' + filename_items = os.path.join(directory_cache, 'items', jabber_id + '.toml') # NOTE Does it work? # It does not seem to actually filter tags. # NOTE Yes. It does work. @@ -825,9 +579,9 @@ class HttpInstance: query = param_query entries_cache = Data.open_file_toml(filename_items) entries_cache_node = entries_cache[node_type] - filename_cache = 'data/{}_query.toml'.format(jid) - Data.cache_items_and_tags_search(entries_cache_node, jid, query) - if exists(filename_cache) and getsize(filename_cache): + filename_cache = os.path.join(directory_cache, 'data', jid + '_query.toml') + Data.cache_items_and_tags_search(directory_cache, entries_cache_node, jid, query) + if os.path.exists(filename_cache) and os.path.getsize(filename_cache): data = Data.open_file_toml(filename_cache) item_ids_all = data['item_ids'] related_tags = data['tags'] @@ -859,9 +613,9 @@ class HttpInstance: tag = param_tags entries_cache = Data.open_file_toml(filename_items) entries_cache_node = entries_cache[node_type] - filename_cache = 'data/{}/{}.toml'.format(jid, tag) - Data.cache_items_and_tags_filter(entries_cache_node, jid, tag) - if exists(filename_cache) and getsize(filename_cache): + filename_cache = os.path.join(directory_cache, 'data', jid, tag + '.toml') + Data.cache_items_and_tags_filter(directory_cache, entries_cache_node, jid, tag) + if os.path.exists(filename_cache) and os.path.getsize(filename_cache): data = Data.open_file_toml(filename_cache) item_ids_all = data['item_ids'] related_tags = data['tags'] @@ -894,10 +648,10 @@ class HttpInstance: name = jabber_id.split('@')[0] entries_cache = Data.open_file_toml(filename_items) entries_cache_node = entries_cache[node_type] - filename_cache = 'data/{}.toml'.format(jid) - #if len(entries_cache_node) and not exists(filename_cache): - Data.cache_items_and_tags(entries_cache_node, jid) - if exists(filename_cache) and getsize(filename_cache): + filename_cache = os.path.join(directory_cache, 'data', jabber_id + '.toml') + #if len(entries_cache_node) and not os.path.exists(filename_cache): + Data.cache_items_and_tags(directory_cache, entries_cache_node, jabber_id) + if os.path.exists(filename_cache) and os.path.getsize(filename_cache): data = Data.open_file_toml(filename_cache) item_ids_all = data['item_ids'] related_tags = data['tags'] @@ -929,7 +683,6 @@ class HttpInstance: # NOTE It might be wiser to use cached items or item identifiers # provided that the viewer is authorized to view items. xmpp_instance = accounts[jabber_id] - db_file = 'main.sqlite' tags_dict = {} if param_query: description = 'Bookmarks from {} with "{}"'.format(jid, param_query) @@ -993,7 +746,7 @@ class HttpInstance: else: # TODO Check permission, so there is no unintended continuing to cached data which is not authorized for. iq = await XmppPubsub.get_node_item_ids(xmpp_instance, jid, node_id_public) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): iq_items_remote = iq['disco_items'] # Cache a list of identifiers of node items to a file. @@ -1165,7 +918,6 @@ class HttpInstance: page_next = 2 page_prev = page - 1 index_first = (page - 1)*10 - db_file = 'main.sqlite' if param_tags or param_tld or param_filetype or param_protocol: entries_count = SQLite.get_entries_count_by_tag(db_file, param_tags) match page_type: @@ -1380,7 +1132,7 @@ class HttpInstance: #configuration_form = await xmpp_instance['xep_0060'].get_node_config(jabber_id, properties['name']) #print(configuration_form) node_id = nodes['public']['name'] - result, reason = await Data.update_cache_and_database(xmpp_instance, jabber_id, 'public', node_id) + result, reason = await Data.update_cache_and_database(directory_cache, xmpp_instance, jabber_id, 'public', node_id) if result == 'error': message = 'XMPP system message » {}.'.format(reason) description = 'IQ Error' @@ -1389,7 +1141,7 @@ class HttpInstance: else: iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, 'xmpp:blasta:configuration:0', 'routine') routine = None - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): payload = iq['pubsub']['items']['item']['payload'] if payload: xmlns = '{jabber:x:data}' @@ -1452,7 +1204,7 @@ class HttpInstance: if jabber_id: xmpp_instance = accounts[jabber_id] node_id = nodes[node_type]['name'] - result, reason = await Data.update_cache_and_database(xmpp_instance, jabber_id, node_type, node_id) + result, reason = await Data.update_cache_and_database(directory_cache, xmpp_instance, jabber_id, node_type, node_id) if result == 'error': message = 'Blasta system message » {}.'.format(reason) description = 'Directory "private" appears to be empty' @@ -1484,7 +1236,7 @@ class HttpInstance: settings = {} for setting in ['enrollment', 'routine']: iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, 'xmpp:blasta:configuration:0', setting) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): payload = iq['pubsub']['items']['item']['payload'] if payload: xmlns = '{jabber:x:data}' @@ -1551,11 +1303,12 @@ class HttpInstance: xmpp_instance = accounts[jabber_id] node_id = nodes[node_type]['name'] iq = await XmppPubsub.get_node_items(xmpp_instance, jabber_id, node_id) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): entries = Data.extract_iq_items(iq, jabber_id) # TODO Append a bookmark or bookmarks of Blasta if entries: - filename = 'export/' + jabber_id + '_' + node_type + '.' + filetype + filename = os.path.join(directory_cache, 'export', jabber_id + '_' + node_type + '.' + filetype) + #filename = 'export/' + jabber_id + '_' + node_type + '.' + filetype #filename = 'export/{}_{}.{}'.format(jabber_id, node_type, filetype) #filename = 'export_' + node_type + '/' + jabber_id + '_' + '.' + filetype #filename = 'export_{}/{}.{}'.format(node_type, jabber_id, filetype) @@ -1613,7 +1366,6 @@ class HttpInstance: name = jabber_id.split('@')[0] # timestamp = datetime.now().isoformat() - db_file = 'main.sqlite' counter = 0 for entry_type in entries: @@ -1665,11 +1417,11 @@ class HttpInstance: node_id = nodes[node_type]['name'] iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, node_id, url_hash) #if len(iq['pubsub']['items']): - if (isinstance(iq, slixmpp.stanza.iq.Iq) and + if (isinstance(iq, Iq) and url_hash == iq['pubsub']['items']['item']['id']): return RedirectResponse(url='/url/' + url_hash + '/edit') iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, 'xmpp:blasta:configuration:0', 'routine') - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): payload = iq['pubsub']['items']['item']['payload'] if payload: xmlns = '{jabber:x:data}' @@ -1726,7 +1478,7 @@ class HttpInstance: node_id = nodes[node_type]['name'] iq = await XmppPubsub.get_node_item( xmpp_instance, jabber_id, node_id, url_hash) - if (isinstance(iq, slixmpp.stanza.iq.Iq) and + if (isinstance(iq, Iq) and url_hash == iq['pubsub']['items']['item']['id']): return RedirectResponse(url='/url/' + url_hash + '/edit') description = 'Confirm properties of a bookmark' @@ -1765,7 +1517,7 @@ class HttpInstance: if jabber_id: xmpp_instance = accounts[jabber_id] node_id = nodes[node_type]['name'] - result, reason = await Data.update_cache_and_database(xmpp_instance, jabber_id, node_type, node_id) + result, reason = await Data.update_cache_and_database(directory_cache, xmpp_instance, jabber_id, node_type, node_id) if result == 'error': message = 'Blasta system message » {}.'.format(reason) description = 'Directory "read" appears to be empty' @@ -1915,7 +1667,6 @@ class HttpInstance: @self.app.get('/tag') def tag_get(request: Request): jabber_id = Utilities.is_jid_matches_to_session(accounts, sessions, request) - db_file = 'main.sqlite' tag_list = SQLite.get_tags_500(db_file) message = 'Common 500 tags sorted by name and sized by commonality.' description = 'Common tags' @@ -1937,7 +1688,6 @@ class HttpInstance: # NOTE Consider retrieval of tags from cache file. # This is relevant to private and read nodes. #if jabber_id == jid or node_type in ('private', 'read'): - db_file = 'main.sqlite' tag_list = SQLite.get_500_tags_by_jid_sorted_by_name(db_file, jid) message = 'Common 500 tags sorted by name and sized by commonality.' description = 'Common tags of {}'.format(jid) @@ -1965,7 +1715,6 @@ class HttpInstance: node_id = 'hash:{}'.format(url_hash) param_hash = url_hash syndicate = path = 'url' - db_file = 'main.sqlite' entries = [] exist = False if len(url_hash) == 32: @@ -1974,7 +1723,7 @@ class HttpInstance: for node in nodes: node_id = nodes[node]['name'] iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, node_id, url_hash) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): # TODO If URL exist in visitor's bookmarks, display its properties (summary, tags title etc.) before data of others. iq_item = iq['pubsub']['items']['item'] item_payload = iq_item['payload'] @@ -2083,6 +1832,7 @@ class HttpInstance: path = 'error' return result_post(request, jabber_id, description, message, path) message = 'Information for URI {}'.format(entries[0]['link']) # entry[2] + if not instances: instances = 0 if instances > 1: description = 'Discover new resources and see who shares them' template_file = 'people.xhtml' @@ -2138,7 +1888,6 @@ class HttpInstance: jabber_id = Utilities.is_jid_matches_to_session(accounts, sessions, request) if jabber_id: name = jabber_id.split('@')[0] - db_file = 'main.sqlite' instances = SQLite.get_entry_instances_by_url_hash(db_file, url_hash) timestamp = datetime.now().isoformat() tags_new = Data.organize_tags(tags) if tags else '' @@ -2169,21 +1918,21 @@ class HttpInstance: #iq = await XmppPubsub.publish_node_item_private( # xmpp_instance, node_id_private, url_hash, iq) await XmppPubsub.del_node_item(xmpp_instance, jabber_id, node_id_public, url_hash) - Data.remove_item_from_cache(jabber_id, 'public', url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, 'public', url_hash) await XmppPubsub.del_node_item(xmpp_instance, jabber_id, node_id_read, url_hash) - Data.remove_item_from_cache(jabber_id, 'read', url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, 'read', url_hash) case 'public': await XmppPubsub.del_node_item(xmpp_instance, jabber_id, node_id_private, url_hash) - Data.remove_item_from_cache(jabber_id, 'private', url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, 'private', url_hash) await XmppPubsub.del_node_item(xmpp_instance, jabber_id, node_id_read, url_hash) - Data.remove_item_from_cache(jabber_id, 'read', url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, 'read', url_hash) case 'read': #iq = await XmppPubsub.publish_node_item_private( # xmpp_instance, node_id_read, url_hash, iq) await XmppPubsub.del_node_item(xmpp_instance, jabber_id, node_id_public, url_hash) - Data.remove_item_from_cache(jabber_id, 'public', url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, 'public', url_hash) await XmppPubsub.del_node_item(xmpp_instance, jabber_id, node_id_private, url_hash) - Data.remove_item_from_cache(jabber_id, 'private', url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, 'private', url_hash) if isinstance(iq, str): description = 'Could not save bookmark' message = 'XMPP system message » {}.'.format(iq) @@ -2191,7 +1940,7 @@ class HttpInstance: return result_post(request, jabber_id, description, message, path) #await iq.send(timeout=15) # Save changes to cache file - entries_cache_filename = 'items/' + jabber_id + '.toml' + entries_cache_filename = os.path.join(directory_cache, 'items', jabber_id + '.toml') entries_cache = Data.open_file_toml(entries_cache_filename) entries_cache_node = entries_cache[node] if node in entries_cache else [] entries_cache_mod = [] @@ -2294,7 +2043,7 @@ class HttpInstance: for node in nodes: node_id = nodes[node]['name'] iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, node_id, url_hash) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): # TODO If URL exist in visitor's bookmarks, display its properties (summary, tags title etc.) before data of others. iq_item = iq['pubsub']['items']['item'] item_payload = iq_item['payload'] @@ -2313,7 +2062,6 @@ class HttpInstance: # TODO Add a check: if iq_item['id'] == url_hash: entries = [] entry = Syndication.extract_items(item_payload) - db_file = 'main.sqlite' instances = SQLite.get_entry_instances_by_url_hash(db_file, url_hash) entry['instances'] = instances entry['jid'] = jabber_id @@ -2367,7 +2115,7 @@ class HttpInstance: for node_type in nodes: node_id = nodes[node_type]['name'] iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, node_id, url_hash) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): # TODO If URL exist in visitor's bookmarks, display its properties (summary, tags title etc.) before data of others. iq_item = iq['pubsub']['items']['item'] item_payload = iq_item['payload'] @@ -2386,7 +2134,6 @@ class HttpInstance: # TODO Add a check: if iq_item['id'] == url_hash: entries = [] entry = Syndication.extract_items(item_payload) - db_file = 'main.sqlite' instances = SQLite.get_entry_instances_by_url_hash(db_file, url_hash) entry['instances'] = instances entry['jid'] = jabber_id @@ -2416,7 +2163,7 @@ class HttpInstance: #await SQLite.delete_combination_row_by_url_hash_and_tag_and_jid(db_file, url_hash, entry['tags'], jabber_id) # Remove the item from cache - Data.remove_item_from_cache(jabber_id, node_type, url_hash) + Data.remove_item_from_cache(directory_cache, jabber_id, node_type, url_hash) template_file = 'browse.xhtml' template_dict = { @@ -2461,12 +2208,11 @@ class HttpInstance: for node in nodes: node_id = nodes[node]['name'] iq = await XmppPubsub.get_node_item(xmpp_instance, jabber_id, node_id, url_hash) - if isinstance(iq, slixmpp.stanza.iq.Iq): + if isinstance(iq, Iq): name = jabber_id.split('@')[0] iq_item = iq['pubsub']['items']['item'] # TODO Add a check: if iq_item['id'] == url_hash: # Is this valid entry['url_hash'] = iq['id'] or should it be iq_item['id'] - db_file = 'main.sqlite' entry = None item_payload = iq_item['payload'] if item_payload: @@ -2548,2898 +2294,3 @@ class HttpInstance: path = 'error' return result_post(request, jabber_id, description, message, path) return response - -class SQLite: - - #from slixfeed.log import Logger - #from slixfeed.utilities import DateAndTime, Url - - # DBLOCK = Lock() - - #logger = Logger(__name__) - - def create_connection(db_file): - """ - Create a database connection to the SQLite database - specified by db_file. - - Parameters - ---------- - db_file : str - Path to database file. - - Returns - ------- - conn : object - Connection object or None. - """ - time_begin = time.time() - function_name = sys._getframe().f_code.co_name -# message_log = '{}' -# logger.debug(message_log.format(function_name)) - conn = None - try: - conn = connect(db_file) - conn.execute("PRAGMA foreign_keys = ON") - # return conn - except Error as e: - print(e) -# logger.warning('Error creating a connection to database {}.'.format(db_file)) -# logger.error(e) - time_end = time.time() - difference = time_end - time_begin - if difference > 1: logger.warning('{} (time: {})'.format(function_name, - difference)) - return conn - - - def create_tables(db_file): - """ - Create SQLite tables. - - Parameters - ---------- - db_file : str - Path to database file. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - with SQLite.create_connection(db_file) as conn: - sql_table_main_entries = ( - """ - CREATE TABLE IF NOT EXISTS main_entries ( - id INTEGER NOT NULL, - url_hash TEXT NOT NULL UNIQUE, - url TEXT NOT NULL UNIQUE, - title TEXT NOT NULL, - summary TEXT, - jid_id TEXT NOT NULL, - date_first TEXT NOT NULL, - date_last TEXT NOT NULL, - instances INTEGER NOT NULL DEFAULT 1, - PRIMARY KEY ("id") - ); - """ - ) - sql_table_main_jids = ( - """ - CREATE TABLE IF NOT EXISTS main_jids ( - id INTEGER NOT NULL, - jid TEXT NOT NULL UNIQUE, - opt_in INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY ("id") - ); - """ - ) - sql_table_main_tags = ( - """ - CREATE TABLE IF NOT EXISTS main_tags ( - id INTEGER NOT NULL, - tag TEXT NOT NULL UNIQUE, - instances INTEGER NOT NULL DEFAULT 1, - PRIMARY KEY ("id") - ); - """ - ) - sql_table_main_statistics = ( - """ - CREATE TABLE IF NOT EXISTS main_statistics ( - id INTEGER NOT NULL, - type TEXT NOT NULL UNIQUE, - count INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY ("id") - ); - """ - ) - sql_table_combination_entries_tags_jids = ( - """ - CREATE TABLE IF NOT EXISTS combination_entries_tags_jids ( - id INTEGER NOT NULL, - entry_id INTEGER NOT NULL, - tag_id INTEGER NOT NULL, - jid_id INTEGER NOT NULL, - FOREIGN KEY ("entry_id") REFERENCES "main_entries" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY ("tag_id") REFERENCES "main_tags" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY ("jid_id") REFERENCES "main_jids" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY ("id") - ); - """ - ) - # NOTE Digit for JID which is authorized; - # Zero (0) for private; - # Empty (no row) for public. - sql_table_authorization_entries_jids = ( - """ - CREATE TABLE IF NOT EXISTS authorization_entries_jids ( - id INTEGER NOT NULL, - entry_id INTEGER NOT NULL, - jid_id INTEGER NOT NULL, - authorization INTEGER NOT NULL, - FOREIGN KEY ("entry_id") REFERENCES "main_entries" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - FOREIGN KEY ("jid_id") REFERENCES "main_jids" ("id") - ON UPDATE CASCADE - ON DELETE CASCADE, - PRIMARY KEY ("id") - ); - """ - ) - sql_table_report_entries = ( - """ - CREATE TABLE IF NOT EXISTS report_entries ( - id INTEGER NOT NULL, - url_hash_subject TEXT NOT NULL, - jid_reporter TEXT NOT NULL, - type TEXT, - comment TEXT, - PRIMARY KEY ("id") - ); - """ - ) - sql_table_report_jids = ( - """ - CREATE TABLE IF NOT EXISTS report_jids ( - id INTEGER NOT NULL, - jid_subject TEXT NOT NULL, - jid_reporter TEXT NOT NULL, - type TEXT, - comment TEXT, - PRIMARY KEY ("id") - ); - """ - ) - sql_trigger_instances_entry_decrease = ( - """ - CREATE TRIGGER instances_entry_decrease - AFTER DELETE ON combination_entries_tags_jids - FOR EACH ROW - BEGIN - UPDATE main_entries - SET instances = ( - SELECT COUNT(DISTINCT jid_id) - FROM combination_entries_tags_jids - WHERE entry_id = OLD.entry_id - ) - WHERE id = OLD.entry_id; - END; - """ - ) - sql_trigger_instances_entry_increase = ( - """ - CREATE TRIGGER instances_entry_increase - AFTER INSERT ON combination_entries_tags_jids - FOR EACH ROW - BEGIN - UPDATE main_entries - SET instances = ( - SELECT COUNT(DISTINCT jid_id) - FROM combination_entries_tags_jids - WHERE entry_id = NEW.entry_id - ) - WHERE id = NEW.entry_id; - END; - """ - ) - sql_trigger_instances_entry_update = ( - """ - CREATE TRIGGER instances_entry_update - AFTER UPDATE ON combination_entries_tags_jids - FOR EACH ROW - BEGIN - -- Decrease instances for the old tag_id - UPDATE main_entries - SET instances = ( - SELECT COUNT(DISTINCT jid_id) - FROM combination_entries_tags_jids - WHERE entry_id = OLD.entry_id - ) - WHERE id = OLD.entry_id; - - -- Increase instances for the new tag_id - UPDATE main_entries - SET instances = ( - SELECT COUNT(DISTINCT jid_id) - FROM combination_entries_tags_jids - WHERE entry_id = NEW.entry_id - ) - WHERE id = NEW.entry_id; - END; - """ - ) - sql_trigger_instances_tag_decrease = ( - """ - CREATE TRIGGER instances_tag_decrease - AFTER DELETE ON combination_entries_tags_jids - FOR EACH ROW - BEGIN - UPDATE main_tags - SET instances = ( - SELECT COUNT(*) - FROM combination_entries_tags_jids - WHERE tag_id = OLD.tag_id - ) - WHERE id = OLD.tag_id; - END; - """ - ) - sql_trigger_instances_tag_increase = ( - """ - CREATE TRIGGER instances_tag_increase - AFTER INSERT ON combination_entries_tags_jids - FOR EACH ROW - BEGIN - UPDATE main_tags - SET instances = ( - SELECT COUNT(*) - FROM combination_entries_tags_jids - WHERE tag_id = NEW.tag_id - ) - WHERE id = NEW.tag_id; - END; - """ - ) - sql_trigger_instances_tag_update = ( - """ - CREATE TRIGGER instances_tag_update - AFTER UPDATE ON combination_entries_tags_jids - FOR EACH ROW - BEGIN - -- Decrease instances for the old tag_id - UPDATE main_tags - SET instances = ( - SELECT COUNT(*) - FROM combination_entries_tags_jids - WHERE tag_id = OLD.tag_id - ) - WHERE id = OLD.tag_id; - - -- Increase instances for the new tag_id - UPDATE main_tags - SET instances = ( - SELECT COUNT(*) - FROM combination_entries_tags_jids - WHERE tag_id = NEW.tag_id - ) - WHERE id = NEW.tag_id; - END; - """ - ) - sql_trigger_entry_count_increase = ( - """ - CREATE TRIGGER entry_count_increase - AFTER INSERT ON main_entries - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_entries - ) - WHERE type = 'entries'; - END; - """ - ) - sql_trigger_entry_count_decrease = ( - """ - CREATE TRIGGER entry_count_decrease - AFTER DELETE ON main_entries - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_entries - ) - WHERE type = 'entries'; - END; - """ - ) - sql_trigger_entry_count_update = ( - """ - CREATE TRIGGER entry_count_update - AFTER UPDATE ON main_entries - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_entries - ) - WHERE type = 'entries'; - END; - """ - ) - sql_trigger_entry_remove = ( - """ - CREATE TRIGGER entry_remove - AFTER UPDATE ON main_entries - FOR EACH ROW - WHEN NEW.instances < 1 - BEGIN - DELETE FROM main_entries WHERE id = OLD.id; - END; - """ - ) - sql_trigger_jid_count_increase = ( - """ - CREATE TRIGGER jid_count_increase - AFTER INSERT ON main_jids - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_jids - ) - WHERE type = 'jids'; - END; - """ - ) - sql_trigger_jid_count_decrease = ( - """ - CREATE TRIGGER jid_count_decrease - AFTER DELETE ON main_jids - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_jids - ) - WHERE type = 'jids'; - END; - """ - ) - sql_trigger_jid_count_update = ( - """ - CREATE TRIGGER jid_count_update - AFTER UPDATE ON main_jids - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_jids - ) - WHERE type = 'jids'; - END; - """ - ) - sql_trigger_tag_count_increase = ( - """ - CREATE TRIGGER tag_count_increase - AFTER INSERT ON main_tags - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_tags - ) - WHERE type = 'tags'; - END; - """ - ) - sql_trigger_tag_count_decrease = ( - """ - CREATE TRIGGER tag_count_decrease - AFTER DELETE ON main_tags - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_tags - ) - WHERE type = 'tags'; - END; - """ - ) - sql_trigger_tag_count_update = ( - """ - CREATE TRIGGER tag_count_update - AFTER UPDATE ON main_tags - BEGIN - UPDATE main_statistics - SET count = ( - SELECT COUNT(*) - FROM main_tags - ) - WHERE type = 'tags'; - END; - """ - ) - sql_trigger_tag_remove = ( - """ - CREATE TRIGGER tag_remove - AFTER UPDATE ON main_tags - FOR EACH ROW - WHEN NEW.instances < 1 - BEGIN - DELETE FROM main_tags WHERE id = OLD.id; - END; - """ - ) - cur = conn.cursor() - cur.execute(sql_table_main_entries) - cur.execute(sql_table_main_jids) - cur.execute(sql_table_main_tags) - cur.execute(sql_table_main_statistics) - cur.execute(sql_table_combination_entries_tags_jids) - cur.execute(sql_table_authorization_entries_jids) - cur.execute(sql_table_report_entries) - cur.execute(sql_table_report_jids) - cur.execute(sql_trigger_instances_entry_decrease) - cur.execute(sql_trigger_instances_entry_increase) - cur.execute(sql_trigger_instances_entry_update) - cur.execute(sql_trigger_instances_tag_decrease) - cur.execute(sql_trigger_instances_tag_increase) - cur.execute(sql_trigger_instances_tag_update) - cur.execute(sql_trigger_entry_count_increase) - cur.execute(sql_trigger_entry_count_decrease) - cur.execute(sql_trigger_entry_count_update) - cur.execute(sql_trigger_entry_remove) - cur.execute(sql_trigger_jid_count_increase) - cur.execute(sql_trigger_jid_count_decrease) - cur.execute(sql_trigger_jid_count_update) - cur.execute(sql_trigger_tag_count_increase) - cur.execute(sql_trigger_tag_count_decrease) - cur.execute(sql_trigger_tag_count_update) - cur.execute(sql_trigger_tag_remove) - - def add_statistics(db_file): - """ - Batch insertion of tags. - - Parameters - ---------- - db_file : str - Path to database file. - entries : list - Set of entries. - - Returns - ------- - None. - - Note - ---- - This function is executed immediately after the creation of the database - and, therefore, the directive "async with DBLOCK:" is not necessary. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - sql = ( - """ - INSERT - INTO main_statistics( - type) - VALUES ('entries'), - ('jids'), - ('tags'); - """ - ) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - try: - cur.execute(sql) - except IntegrityError as e: - print(e) - - async def associate_entries_tags_jids(db_file, entry): - async with DBLOCK: - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - jid = entry['jid'] - url_hash = entry['url_hash'] - entry_id = SQLite.get_entry_id_by_url_hash(db_file, url_hash) - jid_id = SQLite.get_jid_id_by_jid(db_file, jid) - if entry_id: - for tag in entry['tags']: - tag_id = SQLite.get_tag_id_by_tag(db_file, tag) - cet_id = SQLite.get_combination_id_by_entry_id_tag_id_jid_id(db_file, entry_id, tag_id, jid_id) - if not cet_id: - sql = ( - """ - INSERT - INTO combination_entries_tags_jids ( - entry_id, tag_id, jid_id) - VALUES ( - ?, ?, ?); - """ - ) - par = (entry_id, tag_id, jid_id) - try: - cur.execute(sql, par) - except IntegrityError as e: - print('associate_entries_tags_jids') - print(e) - - async def add_tags(db_file, entries): - """ - Batch insertion of tags. - - Parameters - ---------- - db_file : str - Path to database file. - entries : list - Set of entries. - - Returns - ------- - None. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - async with DBLOCK: - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - for entry in entries: - tags = entry['tags'] - for tag in tags: -# sql = ( -# """ -# INSERT OR IGNORE INTO main_tags(tag) VALUES (?); -# """ -# ) - if not SQLite.get_tag_id_by_tag(db_file, tag): - sql = ( - """ - INSERT INTO main_tags(tag) VALUES(?); - """ - ) - par = (tag,) - try: - cur.execute(sql, par) - except IntegrityError as e: - print(e) - - async def add_new_entries(db_file, entries): - """ - Batch insert of new entries into table entries. - - Parameters - ---------- - db_file : str - Path to database file. - entries : list - Set of entries. - - Returns - ------- - None. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - async with DBLOCK: - - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - - for entry in entries: - url_hash = entry['url_hash'] - url = entry['link'] - title = entry['title'] - summary = entry['summary'] - jid = entry['jid'] - date_first = entry['published'] - date_last = entry['published'] - # instances = entry['instances'] - - # Import entries - jid_id = SQLite.get_jid_id_by_jid(db_file, jid) - sql = ( - """ - INSERT - INTO main_entries( - url_hash, url, title, summary, jid_id, date_first, date_last) - VALUES( - ?, ?, ?, ?, ?, ?, ?); - """ - ) - par = (url_hash, url, title, summary, jid_id, date_first, date_last) - - try: - cur.execute(sql, par) - except IntegrityError as e: - print(e) - print(jid_id) - print(entry) -# logger.warning("Skipping: " + str(url)) -# logger.error(e) - - # TODO An additional function to ssociate jid_id (jid) with entry_id (hash_url) - async def set_jid(db_file, jid): - """ - Add a JID to database. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - A Jabber ID. - - Returns - ------- - None. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid: {}' -# .format(function_name, db_file, jid)) - sql = ( - """ - INSERT - INTO main_jids( - jid) - VALUES( - ?); - """ - ) - par = (jid, ) - async with DBLOCK: - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - try: - cur.execute(sql, par) - except IntegrityError as e: - print(e) -# logger.warning("Skipping: " + str(url)) -# logger.error(e) - - def get_entries_count(db_file): - """ - Get entries count. - - Parameters - ---------- - db_file : str - Path to database file. - - Returns - ------- - result : tuple - Number. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - sql = ( - """ - SELECT count - FROM main_statistics - WHERE type = "entries"; - """ - ) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_combination_id_by_entry_id_tag_id_jid_id(db_file, entry_id, tag_id, jid_id): - """ - Get ID by a given Entry ID and a given Tag ID and a given Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - entry_id : str - Entry ID. - tag_id : str - Tag ID. - jid_id : str - Jabber ID. - - Returns - ------- - result : tuple - ID. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} entry_id: {} tag_id: {} jid_id: {}' -# .format(function_name, db_file, entry_id, tag_id, jid_id)) - sql = ( - """ - SELECT id - FROM combination_entries_tags_jids - WHERE entry_id = :entry_id AND tag_id = :tag_id AND jid_id = :jid_id; - """ - ) - par = { - "entry_id": entry_id, - "tag_id": tag_id, - "jid_id": jid_id - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - async def delete_combination_row_by_url_hash_and_tag_and_jid(db_file, url_hash, tags, jid): - """ - Delete a row by a given entry ID and a given Jabber ID and given tags. - - Parameters - ---------- - db_file : str - Path to database file. - url_hash : str - URL hash. - tags : list - Tags. - jid : str - Jabber ID. - - Returns - ------- - None. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} url_hash: {} tag_id: {} jid_id: {}' -# .format(function_name, db_file, url_hash, tag_id, jid_id)) - sql = ( - """ - DELETE - FROM combination_entries_tags_jids - WHERE - entry_id = (SELECT id FROM main_entries WHERE url_hash = :url_hash) AND - tag_id = (SELECT id FROM main_tags WHERE tag = :tag) AND - jid_id = (SELECT id FROM main_jids WHERE jid = :jid); - """ - ) - async with DBLOCK: - with SQLite.create_connection(db_file) as conn: - for tag in tags: - par = { - "url_hash": url_hash, - "tag": tag, - "jid": jid - } - cur = conn.cursor() - cur.execute(sql, par) - - def get_tag_id_and_instances_by_tag(db_file, tag): - """ - Get a tag ID and instances by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - Tag. - - Returns - ------- - result : tuple - Tag ID. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {}' -# .format(function_name, db_file, tag)) - sql = ( - """ - SELECT id, instances - FROM main_tags - WHERE tag = ?; - """ - ) - par = (tag,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() -# return result[0] if result else None, None - if not result: result = None, None - return result - - def get_tags_and_instances_by_url_hash(db_file, url_hash): - """ - Get tags and instances by a given URL hash. - - Parameters - ---------- - db_file : str - Path to database file. - url_hash : str - A hash of a URL. - - Returns - ------- - result : tuple - Tags and instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT mt.tag, mt.instances - FROM main_tags AS mt - INNER JOIN combination_entries_tags_jids AS co ON mt.id = co.tag_id - INNER JOIN main_entries AS me ON me.id = co.entry_id - WHERE me.url_hash = ? - ORDER BY mt.instances DESC; - """ - ) - par = (url_hash,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_tags_and_instances_by_entry_id(db_file, entry_id): - """ - Get tags and instances by a given ID entry. - - Parameters - ---------- - db_file : str - Path to database file. - entry_id : str - An ID of an entry. - - Returns - ------- - result : tuple - Tags and instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT main_tags.tag, main_tags.instances - FROM main_tags - INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id - WHERE combination_entries_tags_jids.entry_id = ? - ORDER BY main_tags.instances DESC; - """ - ) - par = (entry_id,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_jids_and_tags_by_entry_id(db_file, entry_id): - """ - Get JIDs and tags by a given ID entry. - - Parameters - ---------- - db_file : str - Path to database file. - entry_id : str - An ID of an entry. - - Returns - ------- - result : tuple - JIDs and tags. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT main_jids.jid, main_tags.tag - FROM main_tags - INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id - INNER JOIN main_jids ON main_jids.id = combination_entries_tags_jids.jid_id - WHERE combination_entries_tags_jids.entry_id = ? - ORDER BY main_tags.instances DESC; - """ - ) - par = (entry_id,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_jids_and_tags_by_url_hash(db_file, url_hash): - """ - Get JIDs and tags by a given URI hash. - - Parameters - ---------- - db_file : str - Path to database file. - url_hash : str - A URL hash of an entry. - - Returns - ------- - result : tuple - JIDs and tags. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT main_jids.jid, main_tags.tag - FROM main_tags - INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id - INNER JOIN main_jids ON main_jids.id = combination_entries_tags_jids.jid_id - INNER JOIN main_entries ON main_entries.id = combination_entries_tags_jids.entry_id - WHERE main_entries.url_hash = ? - ORDER BY main_tags.instances DESC; - """ - ) - par = (url_hash,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_tag_id_by_tag(db_file, tag): - """ - Get a tag ID by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - Tag. - - Returns - ------- - result : tuple - Tag ID. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {}' -# .format(function_name, db_file, tag)) - sql = ( - """ - SELECT id - FROM main_tags - WHERE tag = ?; - """ - ) - par = (tag,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entry_id_by_url_hash(db_file, url_hash): - """ - Get an entry ID by a given URL hash. - - Parameters - ---------- - db_file : str - Path to database file. - url_hash : str - MD5 hash of URL. - - Returns - ------- - result : tuple - Entry ID. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} url_hash: {}' -# .format(function_name, db_file, url_hash)) - sql = ( - """ - SELECT id - FROM main_entries - WHERE url_hash = ?; - """ - ) - par = (url_hash,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entry_instances_by_url_hash(db_file, url_hash): - """ - Get value of entry instances by a given URL hash. - - Parameters - ---------- - db_file : str - Path to database file. - url_hash : str - MD5 hash of URL. - - Returns - ------- - result : tuple - Value of entry instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} url_hash: {}' -# .format(function_name, db_file, url_hash)) - sql = ( - """ - SELECT instances - FROM main_entries - WHERE url_hash = ?; - """ - ) - par = (url_hash,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entry_by_url_hash(db_file, url_hash): - """ - Get entry of a given URL hash. - - Parameters - ---------- - db_file : str - Path to database file. - url_hash : str - MD5 hash of URL. - - Returns - ------- - result : tuple - Entry properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} url_hash: {}' -# .format(function_name, db_file, url_hash)) - sql = ( - """ - SELECT * - FROM main_entries - WHERE url_hash = ?; - """ - ) - par = (url_hash,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entries_new(db_file, index_first): - """ - Get new entries. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT * - FROM main_entries - ORDER BY date_first DESC - LIMIT 10 - OFFSET ?; - """ - ) - par = (index_first,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_popular(db_file, index_first): - """ - Get popular entries. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT * - FROM main_entries - ORDER BY instances DESC - LIMIT 10 - OFFSET ?; - """ - ) - par = (index_first,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_recent(db_file, index_first): - """ - Get recent entries. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT * - FROM main_entries - ORDER BY date_last DESC - LIMIT 10 - OFFSET ?; - """ - ) - par = (index_first,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_by_query(db_file, query, index_first): - """ - Get entries by a query. - - Parameters - ---------- - db_file : str - Path to database file. - query : str - Search query. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT * - FROM main_entries - WHERE title LIKE :query OR url LIKE :query OR summary LIKE :query - ORDER BY instances DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "query": f'%{query}%', - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_count_by_query(db_file, query): - """ - Get entries count by a query. - - Parameters - ---------- - db_file : str - Path to database file. - query : str - Search query. - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - # NOTE Consider date_first - sql = ( - """ - SELECT COUNT(id) - FROM main_entries - WHERE title LIKE :query OR url LIKE :query OR summary LIKE :query - ORDER BY instances DESC; - """ - ) - par = { - "query": f'%{query}%', - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entries_by_jid_and_tag(db_file, jid, tag, index_first): - """ - Get entries by a tag and a Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - Tag. - jid : str - Jabber ID. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {} jid: {} index_first: {}' -# .format(function_name, db_file, tag, jid, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT DISTINCT me.* - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE mj.jid = :jid AND mt.tag = :tag - ORDER BY instances DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "jid": jid, - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_count_by_jid_and_tag(db_file, jid, tag): - """ - Get entries count by a tag and a Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - Tag. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {} jid: {}' -# .format(function_name, db_file, tag, jid)) - # NOTE Consider date_first - sql = ( - """ - SELECT COUNT(DISTINCT me.id) - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE mj.jid = :jid AND mt.tag = :tag; - """ - ) - par = { - "jid": jid, - "tag": tag - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entries_by_jid_and_query(db_file, jid, query, index_first): - """ - Get entries by a query and a Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - query : str - Search query. - jid : str - Jabber ID. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} query: {} jid: {} index_first: {}' -# .format(function_name, db_file, query, jid, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT DISTINCT me.* - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - WHERE mj.jid = :jid AND (title LIKE :query OR url LIKE :query OR summary LIKE :query) - ORDER BY instances DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "jid": jid, - "query": f'%{query}%', - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_count_by_jid_and_query(db_file, jid, query): - """ - Get entries count by a query and a Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - query : str - Search query. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} query: {} jid: {}' -# .format(function_name, db_file, query, jid)) - # NOTE Consider date_first - sql = ( - """ - SELECT COUNT(DISTINCT me.id) - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - WHERE mj.jid = :jid AND (title LIKE :query OR url LIKE :query OR summary LIKE :query); - """ - ) - par = { - "jid": jid, - "query": f'%{query}%' - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entries_by_jid(db_file, jid, index_first): - """ - Get entries by a Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - index_first : str - . - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid: {} index_first: {}' -# .format(function_name, db_file, jid, index_first)) - # NOTE Consider date_first - sql = ( - """ - SELECT DISTINCT me.* - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - WHERE mj.jid = :jid - ORDER BY instances DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "jid": jid, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_count_by_jid(db_file, jid): - """ - Get entries count by a Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - Entries properties. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid: {}' -# .format(function_name, db_file, jid)) - # NOTE Consider date_first - sql = ( - """ - SELECT COUNT(DISTINCT me.id) - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - WHERE mj.jid = :jid; - """ - ) - par = { - "jid": jid - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entries_count_by_tag(db_file, tag): - """ - Get entries count by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - A tag. - - Returns - ------- - result : tuple - Entries. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {}' -# .format(function_name, db_file, tag)) - sql = ( - """ - SELECT COUNT(DISTINCT entries.id) - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag; - """ - ) - par = (tag,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_entries_popular_by_tag(db_file, tag, index_first): - """ - Get popular entries by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - A tag. - index_first : str - . - - Returns - ------- - result : tuple - Entries. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {}' -# .format(function_name, db_file, tag)) - sql = ( - """ - SELECT DISTINCT entries.* - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag - ORDER BY entries.instances DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_recent_by_tag(db_file, tag, index_first): - """ - Get recent entries by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - A tag. - index_first : str - . - - Returns - ------- - result : tuple - Entries. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {}' -# .format(function_name, db_file, tag)) - sql = ( - """ - SELECT DISTINCT entries.* - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag - ORDER BY date_last DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_entries_new_by_tag(db_file, tag, index_first): - """ - Get new entries by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - tag : str - A tag. - index_first : str - . - - Returns - ------- - result : tuple - Entries. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} tag: {}' -# .format(function_name, db_file, tag)) - sql = ( - """ - SELECT DISTINCT entries.* - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag - ORDER BY date_first DESC - LIMIT 10 - OFFSET :index_first; - """ - ) - par = { - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_tags_30(db_file): - """ - Get 30 tags. - - Parameters - ---------- - db_file : str - Path to database file. - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT tag, instances - FROM main_tags - ORDER BY instances DESC - LIMIT 30; - """ - ) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql).fetchall() - return result - - def get_30_tags_by_entries_popular(db_file, index_first): - """ - Get 30 tags by currently viewed popular entries. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT id - FROM main_entries - ORDER BY instances DESC - LIMIT 10 - OFFSET ? - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = (index_first,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_entries_new_by_tag(db_file, tag, index_first): - """ - Get 30 tags by currently viewed new entries by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT DISTINCT entries.id - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag - ORDER BY date_first DESC - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_entries_popular_by_tag(db_file, tag, index_first): - """ - Get 30 tags by currently viewed popular entries by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT DISTINCT entries.id - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag - ORDER BY entries.instances DESC - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_entries_recent_by_tag(db_file, tag, index_first): - """ - Get 30 tags by currently viewed recent entries by a given tag. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT DISTINCT entries.id - FROM main_entries AS entries - INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id - INNER JOIN main_tags AS tags ON tags.id = co.tag_id - WHERE tags.tag = :tag - ORDER BY date_last DESC - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_entries_new(db_file, index_first): - """ - Get 30 tags by currently viewed new entries. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT id - FROM main_entries - ORDER BY date_first DESC - LIMIT 10 - OFFSET ? - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = (index_first,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_entries_recent(db_file, index_first): - """ - Get 30 tags by currently viewed recent entries. - - Parameters - ---------- - db_file : str - Path to database file. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT id - FROM main_entries - ORDER BY date_last DESC - LIMIT 10 - OFFSET ? - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = (index_first,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_entries_by_query_recent(db_file, query, index_first): - """ - Get 30 tags by currently viewed entries by query. - - Parameters - ---------- - db_file : str - Path to database file. - query : str - A search query. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT id - FROM main_entries - WHERE title LIKE :query OR url LIKE :query OR summary LIKE :query - ORDER BY instances DESC - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "query": f'%{query}%', - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_jid_and_tag(db_file, jid, tag, index_first): - """ - Get 30 tags by Jabber ID and tags. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - tag : str - A tag. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT co.entry_id - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE mj.jid = :jid AND mt.tag = :tag - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "jid": jid, - "tag": tag, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_jid_and_query(db_file, jid, query, index_first): - """ - Get 30 tags by Jabber ID and query. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - query : str - A search query. - index_first : str - . - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT co.entry_id - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE mj.jid = :jid AND (title LIKE :query OR url LIKE :query OR summary LIKE :query) - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "jid": jid, - "query": f'%{query}%', - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_30_tags_by_jid(db_file, jid, index_first): - """ - Get 30 tags by Jabber ID. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT mt.tag, mt.instances - FROM combination_entries_tags_jids AS co - INNER JOIN main_tags AS mt ON mt.id = co.tag_id - WHERE co.entry_id IN ( - SELECT DISTINCT me.id - FROM main_entries AS me - INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id - INNER JOIN main_jids AS mj ON mj.id = co.jid_id - WHERE mj.jid = :jid - ORDER BY instances DESC - LIMIT 10 - OFFSET :index_first - ) - ORDER BY mt.instances DESC - LIMIT 30; - """ - ) - par = { - "jid": jid, - "index_first": index_first - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_tags_500(db_file): - """ - Get 500 tags. - - Parameters - ---------- - db_file : str - Path to database file. - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {}' -# .format(function_name, db_file)) - sql = ( - """ - WITH Common500Tags AS ( - SELECT tag, instances - FROM main_tags - ORDER BY instances DESC - LIMIT 500 - ) - SELECT tag, instances - FROM Common500Tags - ORDER BY tag ASC; - """ - ) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql).fetchall() - return result - - def get_500_tags_by_jid_sorted_by_name(db_file, jid): - """ - Get 500 tags by Jabber ID, sorted by name. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT mt.tag, COUNT(*) AS instances - FROM main_tags mt - JOIN combination_entries_tags_jids combination ON mt.id = combination.tag_id - JOIN main_jids mj ON combination.jid_id = mj.id - WHERE mj.jid = :jid - GROUP BY mt.tag - LIMIT 500; - """ - ) - par = { - "jid": jid - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_500_tags_by_jid_sorted_by_instance(db_file, jid): - """ - Get 500 tags by Jabber ID, sorted by instance. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - Tags and number of instances. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT mt.tag, COUNT(*) AS instances - FROM main_tags mt - JOIN combination_entries_tags_jids combination ON mt.id = combination.tag_id - JOIN main_jids mj ON combination.jid_id = mj.id - WHERE mj.jid = :jid - GROUP BY mt.tag - ORDER BY instances DESC - LIMIT 500; - """ - ) - par = { - "jid": jid - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - # FIXME It appear that the wrong table is fetched - # The table to be fetched is combination_entries_tags_jids - def is_jid_associated_with_url_hash(db_file, jid, url_hash): - """ - Check whether a given Jabber ID is associated with a given URL hash. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - A Jabber ID. - url_hash : str - An MD5 checksuum of a URL. - - Returns - ------- - result : tuple - Tags. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid: {} url_hash: {}' -# .format(function_name, db_file, jid, url_hash)) - sql = ( - """ - SELECT mj.jid, me.url_hash - FROM main_jids AS mj - INNER JOIN combination_entries_tags_jids AS co ON mj.id = co.jid_id - INNER JOIN main_entries AS me ON me.id = co.entry_id - WHERE mj.jid = :jid AND me.url_hash = :url_hash; - """ - ) - par = { - "jid": jid, - "url_hash": url_hash - } - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - #deassociate_entry_from_jid - async def delete_combination_row_by_jid_and_url_hash(db_file, url_hash, jid): - """ - Remove association of a given Jabber ID and a given URL hash. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - A Jabber ID. - url_hash : str - An MD5 checksuum of a URL. - - Returns - ------- - None. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid: {} url_hash: {}' -# .format(function_name, db_file, jid, url_hash)) - sql = ( - """ - DELETE FROM combination_entries_tags_jids - WHERE id IN ( - SELECT co.id - FROM combination_entries_tags_jids co - JOIN main_entries me ON co.entry_id = me.id - JOIN main_jids mj ON co.jid_id = mj.id - WHERE me.url_hash = :url_hash AND mj.jid = :jid - ); - """ - ) - par = { - "jid": jid, - "url_hash": url_hash - } - async with DBLOCK: - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - cur.execute(sql, par) - - # NOTE The result was ordered by number of instances - # ORDER BY main_tags.instances DESC - # And has been changed to order of alphabet - # ORDER BY main_tags.tag ASC - def get_tags_by_entry_id(db_file, entry_id): - """ - Get tags by an ID entry. - - Parameters - ---------- - db_file : str - Path to database file. - entry_id : str - An ID of an entry. - - Returns - ------- - result : tuple - Tags. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} index_first: {}' -# .format(function_name, db_file, index_first)) - sql = ( - """ - SELECT DISTINCT main_tags.tag - FROM main_tags - INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id - WHERE combination_entries_tags_jids.entry_id = ? - ORDER BY main_tags.tag ASC - LIMIT 5; - """ - ) - par = (entry_id,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchall() - return result - - def get_jid_id_by_jid(db_file, jid): - """ - Get id of a given jid. - - Parameters - ---------- - db_file : str - Path to database file. - jid : str - Jabber ID. - - Returns - ------- - result : tuple - ID. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid: {}' -# .format(function_name, db_file, jid)) - sql = ( - """ - SELECT id - FROM main_jids - WHERE jid = ?; - """ - ) - par = (jid,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - - def get_jid_by_jid_id(db_file, jid_id): - """ - Get jid of a given jid_id. - - Parameters - ---------- - db_file : str - Path to database file. - jid_id : str - ID of Jabber ID. - - Returns - ------- - result : tuple - ID. - """ - function_name = sys._getframe().f_code.co_name -# logger.debug('{}: db_file: {} jid_id: {}' -# .format(function_name, db_file, jid_id)) - sql = ( - """ - SELECT jid - FROM main_jids - WHERE id = ?; - """ - ) - par = (jid_id,) - with SQLite.create_connection(db_file) as conn: - cur = conn.cursor() - result = cur.execute(sql, par).fetchone() - return result[0] if result and len(result) == 1 else result - -class Syndication: - - def create_rfc4287_entry(feed_entry): - node_entry = ET.Element('entry') - node_entry.set('xmlns', 'http://www.w3.org/2005/Atom') - # Title - title = ET.SubElement(node_entry, 'title') - title.set('type', 'text') - title.text = feed_entry['title'] - # Summary - summary = ET.SubElement(node_entry, 'summary') # TODO Try 'content' - summary.set('type', 'text') - #summary.set('lang', feed_entry['summary_lang']) - summary.text = feed_entry['summary'] - # Tags - if feed_entry['tags']: - for term in feed_entry['tags']: - tag = ET.SubElement(node_entry, 'category') - tag.set('term', term) - # Link - link = ET.SubElement(node_entry, "link") - link.set('href', feed_entry['link']) - # Links -# for feed_entry_link in feed_entry['links']: -# link = ET.SubElement(node_entry, "link") -# link.set('href', feed_entry_link['url']) -# link.set('type', feed_entry_link['type']) -# link.set('rel', feed_entry_link['rel']) - # Date saved - if 'published' in feed_entry and feed_entry['published']: - published = ET.SubElement(node_entry, 'published') - published.text = feed_entry['published'] - # Date edited - if 'updated' in feed_entry and feed_entry['updated']: - updated = ET.SubElement(node_entry, 'updated') - updated.text = feed_entry['updated'] - return node_entry - - def extract_items(item_payload, limit=False): - namespace = '{http://www.w3.org/2005/Atom}' - title = item_payload.find(namespace + 'title') - links = item_payload.find(namespace + 'link') - if (not isinstance(title, ET.Element) and - not isinstance(links, ET.Element)): return None - title_text = '' if title == None else title.text - if isinstance(links, ET.Element): - for link in item_payload.findall(namespace + 'link'): - link_href = link.attrib['href'] if 'href' in link.attrib else '' - if link_href: break - contents = item_payload.find(namespace + 'summary') - summary_text = '' - if isinstance(contents, ET.Element): - for summary in item_payload.findall(namespace + 'summary'): - summary_text = summary.text or '' - if summary_text: break - published = item_payload.find(namespace + 'published') - published_text = '' if published == None else published.text - categories = item_payload.find(namespace + 'category') - tags = [] - if isinstance(categories, ET.Element): - for category in item_payload.findall(namespace + 'category'): - if 'term' in category.attrib and category.attrib['term']: - category_term = category.attrib['term'] - if len(category_term) < 20: - tags.append(category_term) - elif len(category_term) < 50: - tags.append(category_term) - if limit and len(tags) > 4: break - - - identifier = item_payload.find(namespace + 'id') - if identifier and identifier.attrib: print(identifier.attrib) - identifier_text = '' if identifier == None else identifier.text - - instances = '' # TODO Check the Blasta database for instances. - - entry = {'title' : title_text, - 'link' : link_href, - 'summary' : summary_text, - 'published' : published_text, - 'updated' : published_text, # TODO "Updated" is missing - 'tags' : tags} - return entry - -class Utilities: - - def convert_iso8601_to_readable(timestamp): - old_date_format = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) - new_date_format = old_date_format.strftime("%B %d, %Y") - return new_date_format - - def hash_url_to_md5(url): - url_encoded = url.encode() - url_hashed = hashlib.md5(url_encoded) - url_digest = url_hashed.hexdigest() - return url_digest - - def is_jid_matches_to_session(accounts, sessions, request): - jabber_id = request.cookies.get('jabber_id') - session_key = request.cookies.get('session_key') - if (jabber_id and - jabber_id in accounts and - jabber_id in sessions and - session_key == sessions[jabber_id]): - return jabber_id - -class Xml: - - def create_setting_entry(xmpp_instance, key : str, value : str): - form = xmpp_instance['xep_0004'].make_form('form', 'Settings') - form['type'] = 'result' - form.add_field(var=key, - value=value) - return form - -# def create_setting_entry(value : str): -# element = ET.Element('value') -# element.text = value -# return element - -class Configuration: - - def instantiate_database(db_file): -# db_dir = get_default_data_directory() -# if not os.path.isdir(db_dir): -# os.mkdir(db_dir) -# if not os.path.isdir(db_dir + "/sqlite"): -# os.mkdir(db_dir + "/sqlite") -# db_file = os.path.join(db_dir, "sqlite", r"{}.db".format(jid_file)) - SQLite.create_tables(db_file) - SQLite.add_statistics(db_file) - return db_file - -class XmppInstance(ClientXMPP): - def __init__(self, jid, password): - super().__init__(jid, password) - #self.add_event_handler("connection_failed", self.on_connection_failed) - #self.add_event_handler("failed_auth", self.on_failed_auth) - self.add_event_handler("session_start", self.on_session_start) - self.register_plugin('xep_0004') # XEP-0004: Data Forms - self.register_plugin('xep_0030') # XEP-0030: Service Discovery - self.register_plugin('xep_0059') # XEP-0059: Result Set Management - self.register_plugin('xep_0060') # XEP-0060: Publish-Subscribe - self.register_plugin('xep_0078') # XEP-0078: Non-SASL Authentication - self.register_plugin('xep_0163') # XEP-0163: Personal Eventing Protocol - self.register_plugin('xep_0223') # XEP-0223: Persistent Storage of Private Data via PubSub - self.connect() - # self.process(forever=False) - - self.connection_accepted = False - -# def on_connection_failed(self, event): -# self.connection_accepted = False - -# def on_failed_auth(self, event): -# self.connection_accepted = False - - def on_session_start(self, event): - self.connection_accepted = True - -class XmppMessage: - - def send(self, jid, message_body): - jid_from = str(self.boundjid) if self.is_component else None - self.send_message( - mto=jid, - mfrom=jid_from, - mbody=message_body, - mtype='chat') - - # NOTE It appears to not work. - def send_headline(self, jid, message_subject, message_body): - jid_from = str(self.boundjid) if self.is_component else None - self.send_message( - mto=jid, - mfrom=jid_from, - msubject=message_subject, - mbody=message_body, - mtype='headline') - -class XmppPubsub: - - # TODO max-items might be limited (CanChat: 255), so iterate from a bigger number to a smaller. - # NOTE This function was copied from atomtopubsub - def create_node_atom(xmpp_instance, jid, node, title, subtitle, access_model): - jid_from = str(xmpp_instance.boundjid) if xmpp_instance.is_component else None - iq = xmpp_instance.Iq(stype='set', - sto=jid, - sfrom=jid_from) - iq['pubsub']['create']['node'] = node - form = iq['pubsub']['configure']['form'] - form['type'] = 'submit' - form.addField('pubsub#access_model', - ftype='list-single', - value=access_model) - form.addField('pubsub#deliver_payloads', - ftype='boolean', - value=0) - form.addField('pubsub#description', - ftype='text-single', - value=subtitle) - form.addField('pubsub#max_items', - ftype='text-single', - value='255') - form.addField('pubsub#notify_retract', - ftype='boolean', - value=1) - form.addField('pubsub#persist_items', - ftype='boolean', - value=1) - form.addField('pubsub#send_last_published_item', - ftype='text-single', - value='never') - form.addField('pubsub#title', - ftype='text-single', - value=title) - form.addField('pubsub#type', - ftype='text-single', - value='http://www.w3.org/2005/Atom') - return iq - - def create_node_config(xmpp_instance, jid): - jid_from = str(xmpp_instance.boundjid) if xmpp_instance.is_component else None - iq = xmpp_instance.Iq(stype='set', - sto=jid, - sfrom=jid_from) - iq['pubsub']['create']['node'] = 'xmpp:blasta:configuration:0' - form = iq['pubsub']['configure']['form'] - form['type'] = 'submit' - form.addField('pubsub#access_model', - ftype='list-single', - value='whitelist') - form.addField('pubsub#deliver_payloads', - ftype='boolean', - value=0) - form.addField('pubsub#description', - ftype='text-single', - value='Settings of the Blasta PubSub bookmarks system') - form.addField('pubsub#max_items', - ftype='text-single', - value='30') - form.addField('pubsub#notify_retract', - ftype='boolean', - value=1) - form.addField('pubsub#persist_items', - ftype='boolean', - value=1) - form.addField('pubsub#send_last_published_item', - ftype='text-single', - value='never') - form.addField('pubsub#title', - ftype='text-single', - value='Blasta Settings') - form.addField('pubsub#type', - ftype='text-single', - value='settings') - return iq - - async def del_node_item(xmpp_instance, pubsub, node, item_id): - try: - iq = await xmpp_instance.plugin['xep_0060'].retract( - pubsub, node, item_id, timeout=5, notify=None) - result = iq - except IqError as e: - result = e.iq['error']['text'] - print(e) - except IqTimeout as e: - result = 'Timeout' - print(e) - print(result) - return result - - def get_iterator(xmpp_instance, pubsub, node, max_items, iterator): - iterator = xmpp_instance.plugin['xep_0060'].get_items( - pubsub, node, timeout=5, max_items=max_items, iterator=iterator) - return iterator - - async def get_node_configuration(xmpp_instance, pubsub, node): - try: - iq = await xmpp_instance.plugin['xep_0060'].get_node_config( - pubsub, node) - return iq - except (IqError, IqTimeout) as e: - print(e) - - async def get_node_item(xmpp_instance, pubsub, node, item_id): - try: - iq = await xmpp_instance.plugin['xep_0060'].get_item( - pubsub, node, item_id, timeout=5) - result = iq - except IqError as e: - result = e.iq['error']['text'] - print(e) - except IqTimeout as e: - result = 'Timeout' - print(e) - return result - - async def get_node_item_ids(xmpp_instance, pubsub, node): - try: - iq = await xmpp_instance.plugin['xep_0030'].get_items( - pubsub, node) - # Broken. See https://codeberg.org/poezio/slixmpp/issues/3548 - #iq = await xmpp_instance.plugin['xep_0060'].get_item_ids( - # pubsub, node, timeout=5) - result = iq - except IqError as e: - if e.iq['error']['text'] == 'Node not found': - result = 'Node not found' - elif e.iq['error']['condition'] == 'item-not-found': - result = 'Item not found' - else: - result = None - print(e) - except IqTimeout as e: - result = 'Timeout' - print(e) - return result - - async def get_node_item_private(xmpp_instance, node, item_id): - try: - iq = await xmpp_instance.plugin['xep_0223'].retrieve( - node, item_id, timeout=5) - result = iq - except IqError as e: - result = e.iq['error']['text'] - print(e) - except IqTimeout as e: - result = 'Timeout' - print(e) - return result - - async def get_node_items(xmpp_instance, pubsub, node, item_ids=None, max_items=None): - try: - if max_items: - iq = await xmpp_instance.plugin['xep_0060'].get_items( - pubsub, node, timeout=5) - it = xmpp_instance.plugin['xep_0060'].get_items( - pubsub, node, timeout=5, max_items=max_items, iterator=True) - q = rsm.Iq() - q['to'] = pubsub - q['disco_items']['node'] = node - async for item in rsm.ResultIterator(q, 'disco_items', '10'): - print(item['disco_items']['items']) - - else: - iq = await xmpp_instance.plugin['xep_0060'].get_items( - pubsub, node, timeout=5, item_ids=item_ids) - result = iq - except IqError as e: - if e.iq['error']['text'] == 'Node not found': - result = 'Node not found' - elif e.iq['error']['condition'] == 'item-not-found': - result = 'Item not found' - else: - result = None - print(e) - except IqTimeout as e: - result = 'Timeout' - print(e) - return result - - async def get_nodes(xmpp_instance): - try: - iq = await xmpp_instance.plugin['xep_0060'].get_nodes() - return iq - except (IqError, IqTimeout) as e: - print(e) - - async def is_node_exist(xmpp_instance, node_name): - iq = await XmppPubsub.get_nodes(xmpp_instance) - nodes = iq['disco_items']['items'] - for node in nodes: - if node[1] == node_name: - return True - - async def publish_node_item(xmpp_instance, jid, node, item_id, payload): - try: - iq = await xmpp_instance.plugin['xep_0060'].publish( - jid, node, id=item_id, payload=payload) - print(iq) - return iq - except (IqError, IqTimeout) as e: - print(e) - - async def publish_node_item_private(xmpp_instance, node, item_id, stanza): - try: - iq = await xmpp_instance.plugin['xep_0223'].store( - stanza, node, item_id) - print(iq) - return iq - except (IqError, IqTimeout) as e: - print(e) - if e.iq['error']['text'] == 'Field does not match: access_model': - return 'Error: Could not set private bookmark due to Access Model mismatch' - - async def set_node_private(xmpp_instance, node): - try: - iq = await xmpp_instance.plugin['xep_0223'].configure(node) - print(iq) - return iq - except (IqError, IqTimeout) as e: - print(e) - -def main(): - if not exists('main.sqlite') or not getsize('main.sqlite'): - Configuration.instantiate_database('main.sqlite') - accounts = {} - sessions = {} - http_instance = HttpInstance(accounts, sessions) - return http_instance.app - -app = main() - -# FIXME -if __name__ == '__main__': - parser = argparse.ArgumentParser( - prog='blasta', - description='Blasta - A collaborative annotation system.', - usage='%(prog)s [OPTION]...') - parser.add_argument('-v', '--version', help='print version', - action='version', version='0.1') - parser.add_argument('-p', '--port', help='port number', dest='port') - parser.add_argument('-o', '--open', help='open an html browser', action='store_const', const=True, dest='open') - args = parser.parse_args() - port = args.port if args.port else 8000 - uvicorn.run(app, host='localhost', port=port, reload=True) - if args.open: - # TODO Check first time - webbrowser.open('http://localhost:{}/help/about'.format(port)) - webbrowser.open_new_tab('http://localhost:{}'.format(port)) - diff --git a/screenshot/browse.png b/blasta/screenshot/browse.png similarity index 100% rename from screenshot/browse.png rename to blasta/screenshot/browse.png diff --git a/screenshot/tag.png b/blasta/screenshot/tag.png similarity index 100% rename from screenshot/tag.png rename to blasta/screenshot/tag.png diff --git a/blasta/sqlite.py b/blasta/sqlite.py new file mode 100644 index 0000000..eef77ae --- /dev/null +++ b/blasta/sqlite.py @@ -0,0 +1,2482 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from asyncio import Lock +from sqlite3 import connect, Error, IntegrityError +import sys +import time + +DBLOCK = Lock() + +class SQLite: + + def instantiate_database(db_file): +# db_dir = get_default_data_directory() +# if not os.path.isdir(db_dir): +# os.mkdir(db_dir) +# if not os.path.isdir(db_dir + "/sqlite"): +# os.mkdir(db_dir + "/sqlite") +# db_file = os.path.join(db_dir, "sqlite", r"{}.db".format(jid_file)) + SQLite.create_tables(db_file) + SQLite.add_statistics(db_file) + return db_file + + #from slixfeed.log import Logger + #from slixfeed.utilities import DateAndTime, Url + + # DBLOCK = Lock() + + #logger = Logger(__name__) + + def create_connection(db_file): + """ + Create a database connection to the SQLite database + specified by db_file. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + conn : object + Connection object or None. + """ + time_begin = time.time() + function_name = sys._getframe().f_code.co_name +# message_log = '{}' +# logger.debug(message_log.format(function_name)) + conn = None + try: + conn = connect(db_file) + conn.execute("PRAGMA foreign_keys = ON") + # return conn + except Error as e: + print(e) +# logger.warning('Error creating a connection to database {}.'.format(db_file)) +# logger.error(e) + time_end = time.time() + difference = time_end - time_begin + if difference > 1: logger.warning('{} (time: {})'.format(function_name, + difference)) + return conn + + + def create_tables(db_file): + """ + Create SQLite tables. + + Parameters + ---------- + db_file : str + Path to database file. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + with SQLite.create_connection(db_file) as conn: + sql_table_main_entries = ( + """ + CREATE TABLE IF NOT EXISTS main_entries ( + id INTEGER NOT NULL, + url_hash TEXT NOT NULL UNIQUE, + url TEXT NOT NULL UNIQUE, + title TEXT NOT NULL, + summary TEXT, + jid_id TEXT NOT NULL, + date_first TEXT NOT NULL, + date_last TEXT NOT NULL, + instances INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY ("id") + ); + """ + ) + sql_table_main_jids = ( + """ + CREATE TABLE IF NOT EXISTS main_jids ( + id INTEGER NOT NULL, + jid TEXT NOT NULL UNIQUE, + opt_in INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY ("id") + ); + """ + ) + sql_table_main_tags = ( + """ + CREATE TABLE IF NOT EXISTS main_tags ( + id INTEGER NOT NULL, + tag TEXT NOT NULL UNIQUE, + instances INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY ("id") + ); + """ + ) + sql_table_main_statistics = ( + """ + CREATE TABLE IF NOT EXISTS main_statistics ( + id INTEGER NOT NULL, + type TEXT NOT NULL UNIQUE, + count INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY ("id") + ); + """ + ) + sql_table_combination_entries_tags_jids = ( + """ + CREATE TABLE IF NOT EXISTS combination_entries_tags_jids ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + jid_id INTEGER NOT NULL, + FOREIGN KEY ("entry_id") REFERENCES "main_entries" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY ("tag_id") REFERENCES "main_tags" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY ("jid_id") REFERENCES "main_jids" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + # NOTE Digit for JID which is authorized; + # Zero (0) for private; + # Empty (no row) for public. + sql_table_authorization_entries_jids = ( + """ + CREATE TABLE IF NOT EXISTS authorization_entries_jids ( + id INTEGER NOT NULL, + entry_id INTEGER NOT NULL, + jid_id INTEGER NOT NULL, + authorization INTEGER NOT NULL, + FOREIGN KEY ("entry_id") REFERENCES "main_entries" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + FOREIGN KEY ("jid_id") REFERENCES "main_jids" ("id") + ON UPDATE CASCADE + ON DELETE CASCADE, + PRIMARY KEY ("id") + ); + """ + ) + sql_table_report_entries = ( + """ + CREATE TABLE IF NOT EXISTS report_entries ( + id INTEGER NOT NULL, + url_hash_subject TEXT NOT NULL, + jid_reporter TEXT NOT NULL, + type TEXT, + comment TEXT, + PRIMARY KEY ("id") + ); + """ + ) + sql_table_report_jids = ( + """ + CREATE TABLE IF NOT EXISTS report_jids ( + id INTEGER NOT NULL, + jid_subject TEXT NOT NULL, + jid_reporter TEXT NOT NULL, + type TEXT, + comment TEXT, + PRIMARY KEY ("id") + ); + """ + ) + sql_trigger_instances_entry_decrease = ( + """ + CREATE TRIGGER instances_entry_decrease + AFTER DELETE ON combination_entries_tags_jids + FOR EACH ROW + BEGIN + UPDATE main_entries + SET instances = ( + SELECT COUNT(DISTINCT jid_id) + FROM combination_entries_tags_jids + WHERE entry_id = OLD.entry_id + ) + WHERE id = OLD.entry_id; + END; + """ + ) + sql_trigger_instances_entry_increase = ( + """ + CREATE TRIGGER instances_entry_increase + AFTER INSERT ON combination_entries_tags_jids + FOR EACH ROW + BEGIN + UPDATE main_entries + SET instances = ( + SELECT COUNT(DISTINCT jid_id) + FROM combination_entries_tags_jids + WHERE entry_id = NEW.entry_id + ) + WHERE id = NEW.entry_id; + END; + """ + ) + sql_trigger_instances_entry_update = ( + """ + CREATE TRIGGER instances_entry_update + AFTER UPDATE ON combination_entries_tags_jids + FOR EACH ROW + BEGIN + -- Decrease instances for the old tag_id + UPDATE main_entries + SET instances = ( + SELECT COUNT(DISTINCT jid_id) + FROM combination_entries_tags_jids + WHERE entry_id = OLD.entry_id + ) + WHERE id = OLD.entry_id; + + -- Increase instances for the new tag_id + UPDATE main_entries + SET instances = ( + SELECT COUNT(DISTINCT jid_id) + FROM combination_entries_tags_jids + WHERE entry_id = NEW.entry_id + ) + WHERE id = NEW.entry_id; + END; + """ + ) + sql_trigger_instances_tag_decrease = ( + """ + CREATE TRIGGER instances_tag_decrease + AFTER DELETE ON combination_entries_tags_jids + FOR EACH ROW + BEGIN + UPDATE main_tags + SET instances = ( + SELECT COUNT(*) + FROM combination_entries_tags_jids + WHERE tag_id = OLD.tag_id + ) + WHERE id = OLD.tag_id; + END; + """ + ) + sql_trigger_instances_tag_increase = ( + """ + CREATE TRIGGER instances_tag_increase + AFTER INSERT ON combination_entries_tags_jids + FOR EACH ROW + BEGIN + UPDATE main_tags + SET instances = ( + SELECT COUNT(*) + FROM combination_entries_tags_jids + WHERE tag_id = NEW.tag_id + ) + WHERE id = NEW.tag_id; + END; + """ + ) + sql_trigger_instances_tag_update = ( + """ + CREATE TRIGGER instances_tag_update + AFTER UPDATE ON combination_entries_tags_jids + FOR EACH ROW + BEGIN + -- Decrease instances for the old tag_id + UPDATE main_tags + SET instances = ( + SELECT COUNT(*) + FROM combination_entries_tags_jids + WHERE tag_id = OLD.tag_id + ) + WHERE id = OLD.tag_id; + + -- Increase instances for the new tag_id + UPDATE main_tags + SET instances = ( + SELECT COUNT(*) + FROM combination_entries_tags_jids + WHERE tag_id = NEW.tag_id + ) + WHERE id = NEW.tag_id; + END; + """ + ) + sql_trigger_entry_count_increase = ( + """ + CREATE TRIGGER entry_count_increase + AFTER INSERT ON main_entries + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_entries + ) + WHERE type = 'entries'; + END; + """ + ) + sql_trigger_entry_count_decrease = ( + """ + CREATE TRIGGER entry_count_decrease + AFTER DELETE ON main_entries + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_entries + ) + WHERE type = 'entries'; + END; + """ + ) + sql_trigger_entry_count_update = ( + """ + CREATE TRIGGER entry_count_update + AFTER UPDATE ON main_entries + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_entries + ) + WHERE type = 'entries'; + END; + """ + ) + sql_trigger_entry_remove = ( + """ + CREATE TRIGGER entry_remove + AFTER UPDATE ON main_entries + FOR EACH ROW + WHEN NEW.instances < 1 + BEGIN + DELETE FROM main_entries WHERE id = OLD.id; + END; + """ + ) + sql_trigger_jid_count_increase = ( + """ + CREATE TRIGGER jid_count_increase + AFTER INSERT ON main_jids + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_jids + ) + WHERE type = 'jids'; + END; + """ + ) + sql_trigger_jid_count_decrease = ( + """ + CREATE TRIGGER jid_count_decrease + AFTER DELETE ON main_jids + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_jids + ) + WHERE type = 'jids'; + END; + """ + ) + sql_trigger_jid_count_update = ( + """ + CREATE TRIGGER jid_count_update + AFTER UPDATE ON main_jids + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_jids + ) + WHERE type = 'jids'; + END; + """ + ) + sql_trigger_tag_count_increase = ( + """ + CREATE TRIGGER tag_count_increase + AFTER INSERT ON main_tags + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_tags + ) + WHERE type = 'tags'; + END; + """ + ) + sql_trigger_tag_count_decrease = ( + """ + CREATE TRIGGER tag_count_decrease + AFTER DELETE ON main_tags + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_tags + ) + WHERE type = 'tags'; + END; + """ + ) + sql_trigger_tag_count_update = ( + """ + CREATE TRIGGER tag_count_update + AFTER UPDATE ON main_tags + BEGIN + UPDATE main_statistics + SET count = ( + SELECT COUNT(*) + FROM main_tags + ) + WHERE type = 'tags'; + END; + """ + ) + sql_trigger_tag_remove = ( + """ + CREATE TRIGGER tag_remove + AFTER UPDATE ON main_tags + FOR EACH ROW + WHEN NEW.instances < 1 + BEGIN + DELETE FROM main_tags WHERE id = OLD.id; + END; + """ + ) + cur = conn.cursor() + cur.execute(sql_table_main_entries) + cur.execute(sql_table_main_jids) + cur.execute(sql_table_main_tags) + cur.execute(sql_table_main_statistics) + cur.execute(sql_table_combination_entries_tags_jids) + cur.execute(sql_table_authorization_entries_jids) + cur.execute(sql_table_report_entries) + cur.execute(sql_table_report_jids) + cur.execute(sql_trigger_instances_entry_decrease) + cur.execute(sql_trigger_instances_entry_increase) + cur.execute(sql_trigger_instances_entry_update) + cur.execute(sql_trigger_instances_tag_decrease) + cur.execute(sql_trigger_instances_tag_increase) + cur.execute(sql_trigger_instances_tag_update) + cur.execute(sql_trigger_entry_count_increase) + cur.execute(sql_trigger_entry_count_decrease) + cur.execute(sql_trigger_entry_count_update) + cur.execute(sql_trigger_entry_remove) + cur.execute(sql_trigger_jid_count_increase) + cur.execute(sql_trigger_jid_count_decrease) + cur.execute(sql_trigger_jid_count_update) + cur.execute(sql_trigger_tag_count_increase) + cur.execute(sql_trigger_tag_count_decrease) + cur.execute(sql_trigger_tag_count_update) + cur.execute(sql_trigger_tag_remove) + + def add_statistics(db_file): + """ + Batch insertion of tags. + + Parameters + ---------- + db_file : str + Path to database file. + entries : list + Set of entries. + + Returns + ------- + None. + + Note + ---- + This function is executed immediately after the creation of the database + and, therefore, the directive "async with DBLOCK:" is not necessary. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + sql = ( + """ + INSERT + INTO main_statistics( + type) + VALUES ('entries'), + ('jids'), + ('tags'); + """ + ) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + try: + cur.execute(sql) + except IntegrityError as e: + print(e) + + async def associate_entries_tags_jids(db_file, entry): + async with DBLOCK: + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + jid = entry['jid'] + url_hash = entry['url_hash'] + entry_id = SQLite.get_entry_id_by_url_hash(db_file, url_hash) + jid_id = SQLite.get_jid_id_by_jid(db_file, jid) + if entry_id: + for tag in entry['tags']: + tag_id = SQLite.get_tag_id_by_tag(db_file, tag) + cet_id = SQLite.get_combination_id_by_entry_id_tag_id_jid_id(db_file, entry_id, tag_id, jid_id) + if not cet_id: + sql = ( + """ + INSERT + INTO combination_entries_tags_jids ( + entry_id, tag_id, jid_id) + VALUES ( + ?, ?, ?); + """ + ) + par = (entry_id, tag_id, jid_id) + try: + cur.execute(sql, par) + except IntegrityError as e: + print('associate_entries_tags_jids') + print(e) + + async def add_tags(db_file, entries): + """ + Batch insertion of tags. + + Parameters + ---------- + db_file : str + Path to database file. + entries : list + Set of entries. + + Returns + ------- + None. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + async with DBLOCK: + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + for entry in entries: + tags = entry['tags'] + for tag in tags: +# sql = ( +# """ +# INSERT OR IGNORE INTO main_tags(tag) VALUES (?); +# """ +# ) + if not SQLite.get_tag_id_by_tag(db_file, tag): + sql = ( + """ + INSERT INTO main_tags(tag) VALUES(?); + """ + ) + par = (tag,) + try: + cur.execute(sql, par) + except IntegrityError as e: + print(e) + + async def add_new_entries(db_file, entries): + """ + Batch insert of new entries into table entries. + + Parameters + ---------- + db_file : str + Path to database file. + entries : list + Set of entries. + + Returns + ------- + None. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + async with DBLOCK: + + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + + for entry in entries: + url_hash = entry['url_hash'] + url = entry['link'] + title = entry['title'] + summary = entry['summary'] + jid = entry['jid'] + date_first = entry['published'] + date_last = entry['published'] + # instances = entry['instances'] + + # Import entries + jid_id = SQLite.get_jid_id_by_jid(db_file, jid) + sql = ( + """ + INSERT + INTO main_entries( + url_hash, url, title, summary, jid_id, date_first, date_last) + VALUES( + ?, ?, ?, ?, ?, ?, ?); + """ + ) + par = (url_hash, url, title, summary, jid_id, date_first, date_last) + + try: + cur.execute(sql, par) + except IntegrityError as e: + print(e) + print(jid_id) + print(entry) +# logger.warning("Skipping: " + str(url)) +# logger.error(e) + + # TODO An additional function to ssociate jid_id (jid) with entry_id (hash_url) + async def set_jid(db_file, jid): + """ + Add a JID to database. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + A Jabber ID. + + Returns + ------- + None. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid: {}' +# .format(function_name, db_file, jid)) + sql = ( + """ + INSERT + INTO main_jids( + jid) + VALUES( + ?); + """ + ) + par = (jid, ) + async with DBLOCK: + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + try: + cur.execute(sql, par) + except IntegrityError as e: + print(e) +# logger.warning("Skipping: " + str(url)) +# logger.error(e) + + def get_entries_count(db_file): + """ + Get entries count. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + result : tuple + Number. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + sql = ( + """ + SELECT count + FROM main_statistics + WHERE type = "entries"; + """ + ) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_combination_id_by_entry_id_tag_id_jid_id(db_file, entry_id, tag_id, jid_id): + """ + Get ID by a given Entry ID and a given Tag ID and a given Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + entry_id : str + Entry ID. + tag_id : str + Tag ID. + jid_id : str + Jabber ID. + + Returns + ------- + result : tuple + ID. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} entry_id: {} tag_id: {} jid_id: {}' +# .format(function_name, db_file, entry_id, tag_id, jid_id)) + sql = ( + """ + SELECT id + FROM combination_entries_tags_jids + WHERE entry_id = :entry_id AND tag_id = :tag_id AND jid_id = :jid_id; + """ + ) + par = { + "entry_id": entry_id, + "tag_id": tag_id, + "jid_id": jid_id + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + async def delete_combination_row_by_url_hash_and_tag_and_jid(db_file, url_hash, tags, jid): + """ + Delete a row by a given entry ID and a given Jabber ID and given tags. + + Parameters + ---------- + db_file : str + Path to database file. + url_hash : str + URL hash. + tags : list + Tags. + jid : str + Jabber ID. + + Returns + ------- + None. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} url_hash: {} tag_id: {} jid_id: {}' +# .format(function_name, db_file, url_hash, tag_id, jid_id)) + sql = ( + """ + DELETE + FROM combination_entries_tags_jids + WHERE + entry_id = (SELECT id FROM main_entries WHERE url_hash = :url_hash) AND + tag_id = (SELECT id FROM main_tags WHERE tag = :tag) AND + jid_id = (SELECT id FROM main_jids WHERE jid = :jid); + """ + ) + async with DBLOCK: + with SQLite.create_connection(db_file) as conn: + for tag in tags: + par = { + "url_hash": url_hash, + "tag": tag, + "jid": jid + } + cur = conn.cursor() + cur.execute(sql, par) + + def get_tag_id_and_instances_by_tag(db_file, tag): + """ + Get a tag ID and instances by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + Tag. + + Returns + ------- + result : tuple + Tag ID. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {}' +# .format(function_name, db_file, tag)) + sql = ( + """ + SELECT id, instances + FROM main_tags + WHERE tag = ?; + """ + ) + par = (tag,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() +# return result[0] if result else None, None + if not result: result = None, None + return result + + def get_tags_and_instances_by_url_hash(db_file, url_hash): + """ + Get tags and instances by a given URL hash. + + Parameters + ---------- + db_file : str + Path to database file. + url_hash : str + A hash of a URL. + + Returns + ------- + result : tuple + Tags and instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT mt.tag, mt.instances + FROM main_tags AS mt + INNER JOIN combination_entries_tags_jids AS co ON mt.id = co.tag_id + INNER JOIN main_entries AS me ON me.id = co.entry_id + WHERE me.url_hash = ? + ORDER BY mt.instances DESC; + """ + ) + par = (url_hash,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_tags_and_instances_by_entry_id(db_file, entry_id): + """ + Get tags and instances by a given ID entry. + + Parameters + ---------- + db_file : str + Path to database file. + entry_id : str + An ID of an entry. + + Returns + ------- + result : tuple + Tags and instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT main_tags.tag, main_tags.instances + FROM main_tags + INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id + WHERE combination_entries_tags_jids.entry_id = ? + ORDER BY main_tags.instances DESC; + """ + ) + par = (entry_id,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_jids_and_tags_by_entry_id(db_file, entry_id): + """ + Get JIDs and tags by a given ID entry. + + Parameters + ---------- + db_file : str + Path to database file. + entry_id : str + An ID of an entry. + + Returns + ------- + result : tuple + JIDs and tags. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT main_jids.jid, main_tags.tag + FROM main_tags + INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id + INNER JOIN main_jids ON main_jids.id = combination_entries_tags_jids.jid_id + WHERE combination_entries_tags_jids.entry_id = ? + ORDER BY main_tags.instances DESC; + """ + ) + par = (entry_id,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_jids_and_tags_by_url_hash(db_file, url_hash): + """ + Get JIDs and tags by a given URI hash. + + Parameters + ---------- + db_file : str + Path to database file. + url_hash : str + A URL hash of an entry. + + Returns + ------- + result : tuple + JIDs and tags. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT main_jids.jid, main_tags.tag + FROM main_tags + INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id + INNER JOIN main_jids ON main_jids.id = combination_entries_tags_jids.jid_id + INNER JOIN main_entries ON main_entries.id = combination_entries_tags_jids.entry_id + WHERE main_entries.url_hash = ? + ORDER BY main_tags.instances DESC; + """ + ) + par = (url_hash,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_tag_id_by_tag(db_file, tag): + """ + Get a tag ID by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + Tag. + + Returns + ------- + result : tuple + Tag ID. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {}' +# .format(function_name, db_file, tag)) + sql = ( + """ + SELECT id + FROM main_tags + WHERE tag = ?; + """ + ) + par = (tag,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entry_id_by_url_hash(db_file, url_hash): + """ + Get an entry ID by a given URL hash. + + Parameters + ---------- + db_file : str + Path to database file. + url_hash : str + MD5 hash of URL. + + Returns + ------- + result : tuple + Entry ID. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} url_hash: {}' +# .format(function_name, db_file, url_hash)) + sql = ( + """ + SELECT id + FROM main_entries + WHERE url_hash = ?; + """ + ) + par = (url_hash,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entry_instances_by_url_hash(db_file, url_hash): + """ + Get value of entry instances by a given URL hash. + + Parameters + ---------- + db_file : str + Path to database file. + url_hash : str + MD5 hash of URL. + + Returns + ------- + result : tuple + Value of entry instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} url_hash: {}' +# .format(function_name, db_file, url_hash)) + sql = ( + """ + SELECT instances + FROM main_entries + WHERE url_hash = ?; + """ + ) + par = (url_hash,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entry_by_url_hash(db_file, url_hash): + """ + Get entry of a given URL hash. + + Parameters + ---------- + db_file : str + Path to database file. + url_hash : str + MD5 hash of URL. + + Returns + ------- + result : tuple + Entry properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} url_hash: {}' +# .format(function_name, db_file, url_hash)) + sql = ( + """ + SELECT * + FROM main_entries + WHERE url_hash = ?; + """ + ) + par = (url_hash,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entries_new(db_file, index_first): + """ + Get new entries. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT * + FROM main_entries + ORDER BY date_first DESC + LIMIT 10 + OFFSET ?; + """ + ) + par = (index_first,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_popular(db_file, index_first): + """ + Get popular entries. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT * + FROM main_entries + ORDER BY instances DESC + LIMIT 10 + OFFSET ?; + """ + ) + par = (index_first,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_recent(db_file, index_first): + """ + Get recent entries. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT * + FROM main_entries + ORDER BY date_last DESC + LIMIT 10 + OFFSET ?; + """ + ) + par = (index_first,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_by_query(db_file, query, index_first): + """ + Get entries by a query. + + Parameters + ---------- + db_file : str + Path to database file. + query : str + Search query. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT * + FROM main_entries + WHERE title LIKE :query OR url LIKE :query OR summary LIKE :query + ORDER BY instances DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "query": f'%{query}%', + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_count_by_query(db_file, query): + """ + Get entries count by a query. + + Parameters + ---------- + db_file : str + Path to database file. + query : str + Search query. + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + # NOTE Consider date_first + sql = ( + """ + SELECT COUNT(id) + FROM main_entries + WHERE title LIKE :query OR url LIKE :query OR summary LIKE :query + ORDER BY instances DESC; + """ + ) + par = { + "query": f'%{query}%', + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entries_by_jid_and_tag(db_file, jid, tag, index_first): + """ + Get entries by a tag and a Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + Tag. + jid : str + Jabber ID. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {} jid: {} index_first: {}' +# .format(function_name, db_file, tag, jid, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT DISTINCT me.* + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE mj.jid = :jid AND mt.tag = :tag + ORDER BY instances DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "jid": jid, + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_count_by_jid_and_tag(db_file, jid, tag): + """ + Get entries count by a tag and a Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + Tag. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {} jid: {}' +# .format(function_name, db_file, tag, jid)) + # NOTE Consider date_first + sql = ( + """ + SELECT COUNT(DISTINCT me.id) + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE mj.jid = :jid AND mt.tag = :tag; + """ + ) + par = { + "jid": jid, + "tag": tag + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entries_by_jid_and_query(db_file, jid, query, index_first): + """ + Get entries by a query and a Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + query : str + Search query. + jid : str + Jabber ID. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} query: {} jid: {} index_first: {}' +# .format(function_name, db_file, query, jid, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT DISTINCT me.* + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + WHERE mj.jid = :jid AND (title LIKE :query OR url LIKE :query OR summary LIKE :query) + ORDER BY instances DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "jid": jid, + "query": f'%{query}%', + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_count_by_jid_and_query(db_file, jid, query): + """ + Get entries count by a query and a Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + query : str + Search query. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} query: {} jid: {}' +# .format(function_name, db_file, query, jid)) + # NOTE Consider date_first + sql = ( + """ + SELECT COUNT(DISTINCT me.id) + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + WHERE mj.jid = :jid AND (title LIKE :query OR url LIKE :query OR summary LIKE :query); + """ + ) + par = { + "jid": jid, + "query": f'%{query}%' + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entries_by_jid(db_file, jid, index_first): + """ + Get entries by a Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + index_first : str + . + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid: {} index_first: {}' +# .format(function_name, db_file, jid, index_first)) + # NOTE Consider date_first + sql = ( + """ + SELECT DISTINCT me.* + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + WHERE mj.jid = :jid + ORDER BY instances DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "jid": jid, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_count_by_jid(db_file, jid): + """ + Get entries count by a Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + Entries properties. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid: {}' +# .format(function_name, db_file, jid)) + # NOTE Consider date_first + sql = ( + """ + SELECT COUNT(DISTINCT me.id) + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + WHERE mj.jid = :jid; + """ + ) + par = { + "jid": jid + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entries_count_by_tag(db_file, tag): + """ + Get entries count by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + A tag. + + Returns + ------- + result : tuple + Entries. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {}' +# .format(function_name, db_file, tag)) + sql = ( + """ + SELECT COUNT(DISTINCT entries.id) + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag; + """ + ) + par = (tag,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_entries_popular_by_tag(db_file, tag, index_first): + """ + Get popular entries by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + A tag. + index_first : str + . + + Returns + ------- + result : tuple + Entries. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {}' +# .format(function_name, db_file, tag)) + sql = ( + """ + SELECT DISTINCT entries.* + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag + ORDER BY entries.instances DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_recent_by_tag(db_file, tag, index_first): + """ + Get recent entries by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + A tag. + index_first : str + . + + Returns + ------- + result : tuple + Entries. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {}' +# .format(function_name, db_file, tag)) + sql = ( + """ + SELECT DISTINCT entries.* + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag + ORDER BY date_last DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_entries_new_by_tag(db_file, tag, index_first): + """ + Get new entries by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + tag : str + A tag. + index_first : str + . + + Returns + ------- + result : tuple + Entries. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} tag: {}' +# .format(function_name, db_file, tag)) + sql = ( + """ + SELECT DISTINCT entries.* + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag + ORDER BY date_first DESC + LIMIT 10 + OFFSET :index_first; + """ + ) + par = { + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_tags_30(db_file): + """ + Get 30 tags. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT tag, instances + FROM main_tags + ORDER BY instances DESC + LIMIT 30; + """ + ) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql).fetchall() + return result + + def get_30_tags_by_entries_popular(db_file, index_first): + """ + Get 30 tags by currently viewed popular entries. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT id + FROM main_entries + ORDER BY instances DESC + LIMIT 10 + OFFSET ? + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = (index_first,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_entries_new_by_tag(db_file, tag, index_first): + """ + Get 30 tags by currently viewed new entries by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT DISTINCT entries.id + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag + ORDER BY date_first DESC + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_entries_popular_by_tag(db_file, tag, index_first): + """ + Get 30 tags by currently viewed popular entries by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT DISTINCT entries.id + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag + ORDER BY entries.instances DESC + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_entries_recent_by_tag(db_file, tag, index_first): + """ + Get 30 tags by currently viewed recent entries by a given tag. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT DISTINCT entries.id + FROM main_entries AS entries + INNER JOIN combination_entries_tags_jids AS co ON entries.id = co.entry_id + INNER JOIN main_tags AS tags ON tags.id = co.tag_id + WHERE tags.tag = :tag + ORDER BY date_last DESC + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_entries_new(db_file, index_first): + """ + Get 30 tags by currently viewed new entries. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT id + FROM main_entries + ORDER BY date_first DESC + LIMIT 10 + OFFSET ? + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = (index_first,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_entries_recent(db_file, index_first): + """ + Get 30 tags by currently viewed recent entries. + + Parameters + ---------- + db_file : str + Path to database file. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT id + FROM main_entries + ORDER BY date_last DESC + LIMIT 10 + OFFSET ? + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = (index_first,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_entries_by_query_recent(db_file, query, index_first): + """ + Get 30 tags by currently viewed entries by query. + + Parameters + ---------- + db_file : str + Path to database file. + query : str + A search query. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT id + FROM main_entries + WHERE title LIKE :query OR url LIKE :query OR summary LIKE :query + ORDER BY instances DESC + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "query": f'%{query}%', + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_jid_and_tag(db_file, jid, tag, index_first): + """ + Get 30 tags by Jabber ID and tags. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + tag : str + A tag. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT co.entry_id + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE mj.jid = :jid AND mt.tag = :tag + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "jid": jid, + "tag": tag, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_jid_and_query(db_file, jid, query, index_first): + """ + Get 30 tags by Jabber ID and query. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + query : str + A search query. + index_first : str + . + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT co.entry_id + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE mj.jid = :jid AND (title LIKE :query OR url LIKE :query OR summary LIKE :query) + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "jid": jid, + "query": f'%{query}%', + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_30_tags_by_jid(db_file, jid, index_first): + """ + Get 30 tags by Jabber ID. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT mt.tag, mt.instances + FROM combination_entries_tags_jids AS co + INNER JOIN main_tags AS mt ON mt.id = co.tag_id + WHERE co.entry_id IN ( + SELECT DISTINCT me.id + FROM main_entries AS me + INNER JOIN combination_entries_tags_jids AS co ON co.entry_id = me.id + INNER JOIN main_jids AS mj ON mj.id = co.jid_id + WHERE mj.jid = :jid + ORDER BY instances DESC + LIMIT 10 + OFFSET :index_first + ) + ORDER BY mt.instances DESC + LIMIT 30; + """ + ) + par = { + "jid": jid, + "index_first": index_first + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_tags_500(db_file): + """ + Get 500 tags. + + Parameters + ---------- + db_file : str + Path to database file. + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {}' +# .format(function_name, db_file)) + sql = ( + """ + WITH Common500Tags AS ( + SELECT tag, instances + FROM main_tags + ORDER BY instances DESC + LIMIT 500 + ) + SELECT tag, instances + FROM Common500Tags + ORDER BY tag ASC; + """ + ) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql).fetchall() + return result + + def get_500_tags_by_jid_sorted_by_name(db_file, jid): + """ + Get 500 tags by Jabber ID, sorted by name. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT mt.tag, COUNT(*) AS instances + FROM main_tags mt + JOIN combination_entries_tags_jids combination ON mt.id = combination.tag_id + JOIN main_jids mj ON combination.jid_id = mj.id + WHERE mj.jid = :jid + GROUP BY mt.tag + LIMIT 500; + """ + ) + par = { + "jid": jid + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_500_tags_by_jid_sorted_by_instance(db_file, jid): + """ + Get 500 tags by Jabber ID, sorted by instance. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + Tags and number of instances. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT mt.tag, COUNT(*) AS instances + FROM main_tags mt + JOIN combination_entries_tags_jids combination ON mt.id = combination.tag_id + JOIN main_jids mj ON combination.jid_id = mj.id + WHERE mj.jid = :jid + GROUP BY mt.tag + ORDER BY instances DESC + LIMIT 500; + """ + ) + par = { + "jid": jid + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + # FIXME It appear that the wrong table is fetched + # The table to be fetched is combination_entries_tags_jids + def is_jid_associated_with_url_hash(db_file, jid, url_hash): + """ + Check whether a given Jabber ID is associated with a given URL hash. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + A Jabber ID. + url_hash : str + An MD5 checksuum of a URL. + + Returns + ------- + result : tuple + Tags. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid: {} url_hash: {}' +# .format(function_name, db_file, jid, url_hash)) + sql = ( + """ + SELECT mj.jid, me.url_hash + FROM main_jids AS mj + INNER JOIN combination_entries_tags_jids AS co ON mj.id = co.jid_id + INNER JOIN main_entries AS me ON me.id = co.entry_id + WHERE mj.jid = :jid AND me.url_hash = :url_hash; + """ + ) + par = { + "jid": jid, + "url_hash": url_hash + } + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + #deassociate_entry_from_jid + async def delete_combination_row_by_jid_and_url_hash(db_file, url_hash, jid): + """ + Remove association of a given Jabber ID and a given URL hash. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + A Jabber ID. + url_hash : str + An MD5 checksuum of a URL. + + Returns + ------- + None. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid: {} url_hash: {}' +# .format(function_name, db_file, jid, url_hash)) + sql = ( + """ + DELETE FROM combination_entries_tags_jids + WHERE id IN ( + SELECT co.id + FROM combination_entries_tags_jids co + JOIN main_entries me ON co.entry_id = me.id + JOIN main_jids mj ON co.jid_id = mj.id + WHERE me.url_hash = :url_hash AND mj.jid = :jid + ); + """ + ) + par = { + "jid": jid, + "url_hash": url_hash + } + async with DBLOCK: + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + cur.execute(sql, par) + + # NOTE The result was ordered by number of instances + # ORDER BY main_tags.instances DESC + # And has been changed to order of alphabet + # ORDER BY main_tags.tag ASC + def get_tags_by_entry_id(db_file, entry_id): + """ + Get tags by an ID entry. + + Parameters + ---------- + db_file : str + Path to database file. + entry_id : str + An ID of an entry. + + Returns + ------- + result : tuple + Tags. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} index_first: {}' +# .format(function_name, db_file, index_first)) + sql = ( + """ + SELECT DISTINCT main_tags.tag + FROM main_tags + INNER JOIN combination_entries_tags_jids ON main_tags.id = combination_entries_tags_jids.tag_id + WHERE combination_entries_tags_jids.entry_id = ? + ORDER BY main_tags.tag ASC + LIMIT 5; + """ + ) + par = (entry_id,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchall() + return result + + def get_jid_id_by_jid(db_file, jid): + """ + Get id of a given jid. + + Parameters + ---------- + db_file : str + Path to database file. + jid : str + Jabber ID. + + Returns + ------- + result : tuple + ID. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid: {}' +# .format(function_name, db_file, jid)) + sql = ( + """ + SELECT id + FROM main_jids + WHERE jid = ?; + """ + ) + par = (jid,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result + + def get_jid_by_jid_id(db_file, jid_id): + """ + Get jid of a given jid_id. + + Parameters + ---------- + db_file : str + Path to database file. + jid_id : str + ID of Jabber ID. + + Returns + ------- + result : tuple + ID. + """ + function_name = sys._getframe().f_code.co_name +# logger.debug('{}: db_file: {} jid_id: {}' +# .format(function_name, db_file, jid_id)) + sql = ( + """ + SELECT jid + FROM main_jids + WHERE id = ?; + """ + ) + par = (jid_id,) + with SQLite.create_connection(db_file) as conn: + cur = conn.cursor() + result = cur.execute(sql, par).fetchone() + return result[0] if result and len(result) == 1 else result diff --git a/blasta/version.py b/blasta/version.py new file mode 100644 index 0000000..0e742f9 --- /dev/null +++ b/blasta/version.py @@ -0,0 +1,2 @@ +__version__ = '0.1' +__version_info__ = (0, 1) diff --git a/blasta/xml/syndication.py b/blasta/xml/syndication.py new file mode 100644 index 0000000..fcd48fe --- /dev/null +++ b/blasta/xml/syndication.py @@ -0,0 +1,88 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import xml.etree.ElementTree as ET + +class Syndication: + + def create_rfc4287_entry(feed_entry): + node_entry = ET.Element('entry') + node_entry.set('xmlns', 'http://www.w3.org/2005/Atom') + # Title + title = ET.SubElement(node_entry, 'title') + title.set('type', 'text') + title.text = feed_entry['title'] + # Summary + summary = ET.SubElement(node_entry, 'summary') # TODO Try 'content' + summary.set('type', 'text') + #summary.set('lang', feed_entry['summary_lang']) + summary.text = feed_entry['summary'] + # Tags + if feed_entry['tags']: + for term in feed_entry['tags']: + tag = ET.SubElement(node_entry, 'category') + tag.set('term', term) + # Link + link = ET.SubElement(node_entry, "link") + link.set('href', feed_entry['link']) + # Links +# for feed_entry_link in feed_entry['links']: +# link = ET.SubElement(node_entry, "link") +# link.set('href', feed_entry_link['url']) +# link.set('type', feed_entry_link['type']) +# link.set('rel', feed_entry_link['rel']) + # Date saved + if 'published' in feed_entry and feed_entry['published']: + published = ET.SubElement(node_entry, 'published') + published.text = feed_entry['published'] + # Date edited + if 'updated' in feed_entry and feed_entry['updated']: + updated = ET.SubElement(node_entry, 'updated') + updated.text = feed_entry['updated'] + return node_entry + + def extract_items(item_payload, limit=False): + namespace = '{http://www.w3.org/2005/Atom}' + title = item_payload.find(namespace + 'title') + links = item_payload.find(namespace + 'link') + if (not isinstance(title, ET.Element) and + not isinstance(links, ET.Element)): return None + title_text = '' if title == None else title.text + if isinstance(links, ET.Element): + for link in item_payload.findall(namespace + 'link'): + link_href = link.attrib['href'] if 'href' in link.attrib else '' + if link_href: break + contents = item_payload.find(namespace + 'summary') + summary_text = '' + if isinstance(contents, ET.Element): + for summary in item_payload.findall(namespace + 'summary'): + summary_text = summary.text or '' + if summary_text: break + published = item_payload.find(namespace + 'published') + published_text = '' if published == None else published.text + categories = item_payload.find(namespace + 'category') + tags = [] + if isinstance(categories, ET.Element): + for category in item_payload.findall(namespace + 'category'): + if 'term' in category.attrib and category.attrib['term']: + category_term = category.attrib['term'] + if len(category_term) < 20: + tags.append(category_term) + elif len(category_term) < 50: + tags.append(category_term) + if limit and len(tags) > 4: break + + + identifier = item_payload.find(namespace + 'id') + if identifier and identifier.attrib: print(identifier.attrib) + identifier_text = '' if identifier == None else identifier.text + + instances = '' # TODO Check the Blasta database for instances. + + entry = {'title' : title_text, + 'link' : link_href, + 'summary' : summary_text, + 'published' : published_text, + 'updated' : published_text, # TODO "Updated" is missing + 'tags' : tags} + return entry diff --git a/blasta/xmpp/instance.py b/blasta/xmpp/instance.py new file mode 100644 index 0000000..47f7820 --- /dev/null +++ b/blasta/xmpp/instance.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +from slixmpp import ClientXMPP + +class XmppInstance(ClientXMPP): + def __init__(self, jid, password): + super().__init__(jid, password) + #self.add_event_handler("connection_failed", self.on_connection_failed) + #self.add_event_handler("failed_auth", self.on_failed_auth) + self.add_event_handler("session_start", self.on_session_start) + self.register_plugin('xep_0004') # XEP-0004: Data Forms + self.register_plugin('xep_0030') # XEP-0030: Service Discovery + self.register_plugin('xep_0059') # XEP-0059: Result Set Management + self.register_plugin('xep_0060') # XEP-0060: Publish-Subscribe + self.register_plugin('xep_0078') # XEP-0078: Non-SASL Authentication + self.register_plugin('xep_0163') # XEP-0163: Personal Eventing Protocol + self.register_plugin('xep_0223') # XEP-0223: Persistent Storage of Private Data via PubSub + self.connect() + # self.process(forever=False) + + self.connection_accepted = False + +# def on_connection_failed(self, event): +# self.connection_accepted = False + +# def on_failed_auth(self, event): +# self.connection_accepted = False + + def on_session_start(self, event): + self.connection_accepted = True diff --git a/blasta/xmpp/message.py b/blasta/xmpp/message.py new file mode 100644 index 0000000..3e52ac6 --- /dev/null +++ b/blasta/xmpp/message.py @@ -0,0 +1,22 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +class XmppMessage: + + def send(self, jid, message_body): + jid_from = str(self.boundjid) if self.is_component else None + self.send_message( + mto=jid, + mfrom=jid_from, + mbody=message_body, + mtype='chat') + + # NOTE It appears to not work. + def send_headline(self, jid, message_subject, message_body): + jid_from = str(self.boundjid) if self.is_component else None + self.send_message( + mto=jid, + mfrom=jid_from, + msubject=message_subject, + mbody=message_body, + mtype='headline') diff --git a/blasta/xmpp/pubsub.py b/blasta/xmpp/pubsub.py new file mode 100644 index 0000000..6fef346 --- /dev/null +++ b/blasta/xmpp/pubsub.py @@ -0,0 +1,231 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import slixmpp +from slixmpp.exceptions import IqError, IqTimeout +#import slixmpp.plugins.xep_0060.stanza.pubsub as pubsub +import slixmpp.plugins.xep_0059.rsm as rsm + +class XmppPubsub: + + # TODO max-items might be limited (CanChat: 255), so iterate from a bigger number to a smaller. + # NOTE This function was copied from atomtopubsub + def create_node_atom(xmpp_instance, jid, node, title, subtitle, access_model): + jid_from = str(xmpp_instance.boundjid) if xmpp_instance.is_component else None + iq = xmpp_instance.Iq(stype='set', + sto=jid, + sfrom=jid_from) + iq['pubsub']['create']['node'] = node + form = iq['pubsub']['configure']['form'] + form['type'] = 'submit' + form.addField('pubsub#access_model', + ftype='list-single', + value=access_model) + form.addField('pubsub#deliver_payloads', + ftype='boolean', + value=0) + form.addField('pubsub#description', + ftype='text-single', + value=subtitle) + form.addField('pubsub#max_items', + ftype='text-single', + value='255') + form.addField('pubsub#notify_retract', + ftype='boolean', + value=1) + form.addField('pubsub#persist_items', + ftype='boolean', + value=1) + form.addField('pubsub#send_last_published_item', + ftype='text-single', + value='never') + form.addField('pubsub#title', + ftype='text-single', + value=title) + form.addField('pubsub#type', + ftype='text-single', + value='http://www.w3.org/2005/Atom') + return iq + + def create_node_config(xmpp_instance, jid): + jid_from = str(xmpp_instance.boundjid) if xmpp_instance.is_component else None + iq = xmpp_instance.Iq(stype='set', + sto=jid, + sfrom=jid_from) + iq['pubsub']['create']['node'] = 'xmpp:blasta:configuration:0' + form = iq['pubsub']['configure']['form'] + form['type'] = 'submit' + form.addField('pubsub#access_model', + ftype='list-single', + value='whitelist') + form.addField('pubsub#deliver_payloads', + ftype='boolean', + value=0) + form.addField('pubsub#description', + ftype='text-single', + value='Settings of the Blasta PubSub bookmarks system') + form.addField('pubsub#max_items', + ftype='text-single', + value='30') + form.addField('pubsub#notify_retract', + ftype='boolean', + value=1) + form.addField('pubsub#persist_items', + ftype='boolean', + value=1) + form.addField('pubsub#send_last_published_item', + ftype='text-single', + value='never') + form.addField('pubsub#title', + ftype='text-single', + value='Blasta Settings') + form.addField('pubsub#type', + ftype='text-single', + value='settings') + return iq + + async def del_node_item(xmpp_instance, pubsub, node, item_id): + try: + iq = await xmpp_instance.plugin['xep_0060'].retract( + pubsub, node, item_id, timeout=5, notify=None) + result = iq + except IqError as e: + result = e.iq['error']['text'] + print(e) + except IqTimeout as e: + result = 'Timeout' + print(e) + print(result) + return result + + def get_iterator(xmpp_instance, pubsub, node, max_items, iterator): + iterator = xmpp_instance.plugin['xep_0060'].get_items( + pubsub, node, timeout=5, max_items=max_items, iterator=iterator) + return iterator + + async def get_node_configuration(xmpp_instance, pubsub, node): + try: + iq = await xmpp_instance.plugin['xep_0060'].get_node_config( + pubsub, node) + return iq + except (IqError, IqTimeout) as e: + print(e) + + async def get_node_item(xmpp_instance, pubsub, node, item_id): + try: + iq = await xmpp_instance.plugin['xep_0060'].get_item( + pubsub, node, item_id, timeout=5) + result = iq + except IqError as e: + result = e.iq['error']['text'] + print(e) + except IqTimeout as e: + result = 'Timeout' + print(e) + return result + + async def get_node_item_ids(xmpp_instance, pubsub, node): + try: + iq = await xmpp_instance.plugin['xep_0030'].get_items( + pubsub, node) + # Broken. See https://codeberg.org/poezio/slixmpp/issues/3548 + #iq = await xmpp_instance.plugin['xep_0060'].get_item_ids( + # pubsub, node, timeout=5) + result = iq + except IqError as e: + if e.iq['error']['text'] == 'Node not found': + result = 'Node not found' + elif e.iq['error']['condition'] == 'item-not-found': + result = 'Item not found' + else: + result = None + print(e) + except IqTimeout as e: + result = 'Timeout' + print(e) + return result + + async def get_node_item_private(xmpp_instance, node, item_id): + try: + iq = await xmpp_instance.plugin['xep_0223'].retrieve( + node, item_id, timeout=5) + result = iq + except IqError as e: + result = e.iq['error']['text'] + print(e) + except IqTimeout as e: + result = 'Timeout' + print(e) + return result + + async def get_node_items(xmpp_instance, pubsub, node, item_ids=None, max_items=None): + try: + if max_items: + iq = await xmpp_instance.plugin['xep_0060'].get_items( + pubsub, node, timeout=5) + it = xmpp_instance.plugin['xep_0060'].get_items( + pubsub, node, timeout=5, max_items=max_items, iterator=True) + q = rsm.Iq() + q['to'] = pubsub + q['disco_items']['node'] = node + async for item in rsm.ResultIterator(q, 'disco_items', '10'): + print(item['disco_items']['items']) + + else: + iq = await xmpp_instance.plugin['xep_0060'].get_items( + pubsub, node, timeout=5, item_ids=item_ids) + result = iq + except IqError as e: + if e.iq['error']['text'] == 'Node not found': + result = 'Node not found' + elif e.iq['error']['condition'] == 'item-not-found': + result = 'Item not found' + else: + result = None + print(e) + except IqTimeout as e: + result = 'Timeout' + print(e) + return result + + async def get_nodes(xmpp_instance): + try: + iq = await xmpp_instance.plugin['xep_0060'].get_nodes() + return iq + except (IqError, IqTimeout) as e: + print(e) + + async def is_node_exist(xmpp_instance, node_name): + iq = await XmppPubsub.get_nodes(xmpp_instance) + nodes = iq['disco_items']['items'] + for node in nodes: + if node[1] == node_name: + return True + + async def publish_node_item(xmpp_instance, jid, node, item_id, payload): + try: + iq = await xmpp_instance.plugin['xep_0060'].publish( + jid, node, id=item_id, payload=payload) + print(iq) + return iq + except (IqError, IqTimeout) as e: + print(e) + + async def publish_node_item_private(xmpp_instance, node, item_id, stanza): + try: + iq = await xmpp_instance.plugin['xep_0223'].store( + stanza, node, item_id) + print(iq) + return iq + except (IqError, IqTimeout) as e: + print(e) + if e.iq['error']['text'] == 'Field does not match: access_model': + return 'Error: Could not set private bookmark due to Access Model mismatch' + + async def set_node_private(xmpp_instance, node): + try: + iq = await xmpp_instance.plugin['xep_0223'].configure(node) + print(iq) + return iq + except (IqError, IqTimeout) as e: + print(e) diff --git a/data/README.txt b/data/README.txt deleted file mode 100644 index 1946376..0000000 --- a/data/README.txt +++ /dev/null @@ -1 +0,0 @@ -This directory is meant to store hashes and tags per JID as TOML. diff --git a/export/README.txt b/export/README.txt deleted file mode 100644 index 14ac63a..0000000 --- a/export/README.txt +++ /dev/null @@ -1 +0,0 @@ -This directory is contains exported nodes. diff --git a/graphic/syndicate.svg b/graphic/syndicate.svg deleted file mode 100644 index b325149..0000000 --- a/graphic/syndicate.svg +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/items/README.txt b/items/README.txt deleted file mode 100644 index f706a25..0000000 --- a/items/README.txt +++ /dev/null @@ -1 +0,0 @@ -This directory is meant to cache nodes per JID as TOML. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fc27fff --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = ["setuptools>=61.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "Blasta" +version = "1.0" +description = "A collaborative annotation management system for XMPP" +authors = [{name = "Schimon Zachary", email = "sch@fedora.email"}] +license = {text = "AGPL-3.0"} +classifiers = [ + "Framework :: slixmpp", + "Intended Audience :: End Users/Desktop", + "License :: OSI Approved :: AGPL-3.0 License", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 3.10", + "Topic :: Internet :: Extensible Messaging and Presence Protocol (XMPP)", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary", + "Topic :: Internet :: XMPP", + "Topic :: Office/Business :: News/Diary", +] +keywords = [ + "atom", + "bookmark", + "collaboration", + "gemini", + "index", + "jabber", + "journal", + "news", + "social", + "syndication", + "xml", + "xmpp", +] + +dependencies = [ + "fastapi", + "lxml", + "python-dateutil", + "python-multipart", + "slixmpp", + "tomli", # Python 3.10 + "uvicorn", +] + +[project.urls] +Homepage = "https://schapps.woodpeckersnest.eu/blasta/" +Repository = "https://git.xmpp-it.net/sch/Blasta" +Issues = "https://git.xmpp-it.net/sch/Blasta/issues" + +[project.scripts] +blasta = "blasta.__main__:main" + +[tool.setuptools] +platforms = ["any"] + +[tool.setuptools.package-data] +"*" = ["*.toml"] diff --git a/configuration.toml b/settings.toml similarity index 77% rename from configuration.toml rename to settings.toml index cdf86ba..045a22b 100644 --- a/configuration.toml +++ b/settings.toml @@ -13,22 +13,22 @@ journal = "" pubsub = "" # Bibliography -node_id = "urn:xmpp:bibliography:0" +node_id = "blasta:annotation:0" node_title = "Blasta" -node_subtitle = "Bibliography" +node_subtitle = "Annotation" # Private bibliography -node_id_private = "xmpp:bibliography:private:0" +node_id_private = "blasta:annotation:private:0" node_title_private = "Blasta (Private)" -node_subtitle_private = "Private bibliography" +node_subtitle_private = "Private annotation" # Reading list -node_id_read = "xmpp:bibliography:read:0" +node_id_read = "blasta:annotation:read:0" node_title_read = "Blasta (Read)" node_subtitle_read = "Reading list" # Settings node -node_settings = "xmpp:blasta:settings:0" +node_settings = "blasta:settings:0" # Acceptable protocol types that would be aggregated to the Blasta database schemes = [