From 75ccfbe521da6c2f0d4a2423ea9cb1658eab1e67 Mon Sep 17 00:00:00 2001 From: Schimon Jehudah Date: Fri, 1 Dec 2023 13:22:03 +0000 Subject: [PATCH] Various of modifications --- datasets/activity/game.csv | 2 + datasets/activity/internet.csv | 2 + datasets/activity/operating_system.csv | 2 + datasets/activity/xmpp.csv | 4 + .../{audiobooks.csv => audiobook.csv} | 0 datasets/catalogues/{books.csv => book.csv} | 2 +- datasets/catalogues/computer.csv | 2 + datasets/catalogues/electronics.csv | 2 + datasets/catalogues/film.csv | 2 + datasets/catalogues/game.csv | 2 + datasets/catalogues/music.csv | 1 + datasets/catalogues/radio.csv | 2 + datasets/catalogues/travel.csv | 1 + datasets/corporations/automotive.csv | 2 + datasets/corporations/computing.csv | 4 + datasets/corporations/hardware.csv | 2 + datasets/corporations/music.csv | 2 + datasets/forums/computer.csv | 3 + datasets/forums/hacking.csv | 2 +- datasets/forums/intrenet.csv | 1 + datasets/forums/linux.csv | 3 + datasets/news/art.csv | 1 + datasets/news/bsd.csv | 1 + datasets/news/business.csv | 2 + datasets/news/christianity.csv | 2 + datasets/news/code.csv | 4 + datasets/news/computer.csv | 17 +++ datasets/news/culture.csv | 12 +- datasets/news/cybersecurity.csv | 1 + datasets/news/data.csv | 1 + datasets/news/design.csv | 2 + datasets/news/electronics.csv | 2 + datasets/news/english.csv | 2 + datasets/news/family.csv | 2 + datasets/news/germany.csv | 4 +- datasets/news/government.csv | 2 + datasets/news/grammar.csv | 2 + datasets/news/hacking.csv | 1 + datasets/news/hardware.csv | 2 + datasets/news/history.csv | 4 + datasets/news/internet.csv | 1 + datasets/news/law.csv | 2 + datasets/news/linux.csv | 8 ++ datasets/news/literature.csv | 4 + datasets/news/misc.csv | 1 + datasets/news/people.csv | 12 ++ datasets/news/privacy.csv | 2 + datasets/news/russia.csv | 1 + datasets/news/surveillance.csv | 1 + datasets/news/technology.csv | 6 + datasets/news/telecom.csv | 1 + datasets/news/united_states.csv | 1 + datasets/news/unix.csv | 3 + datasets/news/war.csv | 2 + datasets/news/xmpp.csv | 12 ++ datasets/organizations/germany.csv | 2 + datasets/organizations/united_states.csv | 4 + datasets/podcasts/computer.csv | 3 + datasets/podcasts/law.csv | 2 + datasets/podcasts/technology.csv | 1 + datasets/projects/chat.csv | 6 + datasets/projects/cloud.csv | 2 + datasets/projects/desktop.csv | 3 + datasets/projects/email.csv | 3 + datasets/projects/fediverse.csv | 2 + datasets/projects/internet.csv | 7 + datasets/projects/mix_network.csv | 2 + datasets/projects/operating_system.csv | 4 +- datasets/projects/website.csv | 2 + datasets/projects/xmpp.csv | 7 + datasets/tutorials/code.csv | 1 + datasets/tutorials/python.csv | 1 + datasets/videos/general.csv | 1 + setup.py | 46 ++++++- slixfeed/confighandler.py | 10 +- slixfeed/datahandler.py | 55 ++++---- slixfeed/main.py.bak | 124 ++++++++++++++++++ slixfeed/sqlitehandler.py | 61 ++++++++- slixfeed/taskhandler.py | 4 +- slixfeed/xmpphandler.py | 84 ++++++++++-- 80 files changed, 538 insertions(+), 63 deletions(-) create mode 100644 datasets/activity/game.csv create mode 100644 datasets/activity/internet.csv create mode 100644 datasets/activity/operating_system.csv create mode 100644 datasets/activity/xmpp.csv rename datasets/catalogues/{audiobooks.csv => audiobook.csv} (100%) rename datasets/catalogues/{books.csv => book.csv} (78%) create mode 100644 datasets/catalogues/computer.csv create mode 100644 datasets/catalogues/electronics.csv create mode 100644 datasets/catalogues/film.csv create mode 100644 datasets/catalogues/game.csv create mode 100644 datasets/catalogues/radio.csv create mode 100644 datasets/corporations/automotive.csv create mode 100644 datasets/corporations/hardware.csv create mode 100644 datasets/corporations/music.csv create mode 100644 datasets/forums/linux.csv create mode 100644 datasets/news/english.csv create mode 100644 datasets/news/grammar.csv create mode 100644 datasets/news/hardware.csv create mode 100644 datasets/news/history.csv create mode 100644 datasets/podcasts/law.csv create mode 100644 datasets/projects/cloud.csv create mode 100644 datasets/projects/desktop.csv create mode 100644 datasets/projects/email.csv create mode 100644 datasets/projects/fediverse.csv create mode 100644 datasets/projects/mix_network.csv create mode 100644 datasets/projects/xmpp.csv create mode 100644 slixfeed/main.py.bak diff --git a/datasets/activity/game.csv b/datasets/activity/game.csv new file mode 100644 index 0000000..908fd3f --- /dev/null +++ b/datasets/activity/game.csv @@ -0,0 +1,2 @@ +name, url, language, country +Feed of "MintApps Team", https://codeberg.org/MintApps.rss, en-US, Germany diff --git a/datasets/activity/internet.csv b/datasets/activity/internet.csv new file mode 100644 index 0000000..bfc9c3b --- /dev/null +++ b/datasets/activity/internet.csv @@ -0,0 +1,2 @@ +name, url, language, country +DokuWiki, https://www.dokuwiki.org/feed.php, en-US, Worldwide diff --git a/datasets/activity/operating_system.csv b/datasets/activity/operating_system.csv new file mode 100644 index 0000000..eaecd2d --- /dev/null +++ b/datasets/activity/operating_system.csv @@ -0,0 +1,2 @@ +name, url, language, country +Replicant: Issues, https://redmine.replicant.us/projects/replicant/issues.atom, en-US, Worldwide diff --git a/datasets/activity/xmpp.csv b/datasets/activity/xmpp.csv new file mode 100644 index 0000000..479726f --- /dev/null +++ b/datasets/activity/xmpp.csv @@ -0,0 +1,4 @@ +name, url, language, country +gajim activity, https://dev.gajim.org/gajim/gajim.atom, en-US, France +Bugs in qutim, http://feeds.launchpad.net/qutim/latest-bugs.atom, en-US, Russia +Feed of "poezio/slixmpp", https://codeberg.org/poezio/slixmpp.rss, en-US, France diff --git a/datasets/catalogues/audiobooks.csv b/datasets/catalogues/audiobook.csv similarity index 100% rename from datasets/catalogues/audiobooks.csv rename to datasets/catalogues/audiobook.csv diff --git a/datasets/catalogues/books.csv b/datasets/catalogues/book.csv similarity index 78% rename from datasets/catalogues/books.csv rename to datasets/catalogues/book.csv index e69db23..c7b78bc 100644 --- a/datasets/catalogues/books.csv +++ b/datasets/catalogues/book.csv @@ -1,2 +1,2 @@ name, url, language, country -New Online Books, https://onlinebooks.library.upenn.edu/newrss.xml, en-US, USA +New Online Books, https://onlinebooks.library.upenn.edu/newrss.xml, en-US, United States diff --git a/datasets/catalogues/computer.csv b/datasets/catalogues/computer.csv new file mode 100644 index 0000000..af7fe6a --- /dev/null +++ b/datasets/catalogues/computer.csv @@ -0,0 +1,2 @@ +name, url, language, country +Vikings Shop, https://shop.vikings.net/feed/, en-US, Germany diff --git a/datasets/catalogues/electronics.csv b/datasets/catalogues/electronics.csv new file mode 100644 index 0000000..b8cab7f --- /dev/null +++ b/datasets/catalogues/electronics.csv @@ -0,0 +1,2 @@ +name, url, language, country +BaoFeng Radios, https://baofengtech.com/feed/, en-US, South Dakota diff --git a/datasets/catalogues/film.csv b/datasets/catalogues/film.csv new file mode 100644 index 0000000..1d6eafe --- /dev/null +++ b/datasets/catalogues/film.csv @@ -0,0 +1,2 @@ +name, url, language, country +Movies Subtitle, https://qsubtitles.com/feed/atom/, en-US, Worldwide diff --git a/datasets/catalogues/game.csv b/datasets/catalogues/game.csv new file mode 100644 index 0000000..c8030de --- /dev/null +++ b/datasets/catalogues/game.csv @@ -0,0 +1,2 @@ +name, url, language, country +Open Source Game Clones: New/Updated Games, https://osgameclones.com/feed.xml, en-US, Worldwide diff --git a/datasets/catalogues/music.csv b/datasets/catalogues/music.csv index 9fbd422..4c0ed57 100644 --- a/datasets/catalogues/music.csv +++ b/datasets/catalogues/music.csv @@ -1,3 +1,4 @@ name, url, language, country MetaBrainz Blog, https://blog.metabrainz.org/feed/, en-US, Worldwide MP3 NFO Database, https://nfodb.ru/rss.php, en-US, Worldwide +Tubesonix – The Future of Royalty Free Licensed Music, https://www.tubesonix.com/feed/, en-US, Worldwide diff --git a/datasets/catalogues/radio.csv b/datasets/catalogues/radio.csv new file mode 100644 index 0000000..b8cab7f --- /dev/null +++ b/datasets/catalogues/radio.csv @@ -0,0 +1,2 @@ +name, url, language, country +BaoFeng Radios, https://baofengtech.com/feed/, en-US, South Dakota diff --git a/datasets/catalogues/travel.csv b/datasets/catalogues/travel.csv index f9e9b25..0685900 100644 --- a/datasets/catalogues/travel.csv +++ b/datasets/catalogues/travel.csv @@ -1,2 +1,3 @@ name, url, language, country Trip101, https://trip101.com/feed, en-US, United States +Travdo Hotels, https://travdo-hotels.de/feed/, de-DE, Germany diff --git a/datasets/corporations/automotive.csv b/datasets/corporations/automotive.csv new file mode 100644 index 0000000..b85fd99 --- /dev/null +++ b/datasets/corporations/automotive.csv @@ -0,0 +1,2 @@ +name, url, language, country +Seafarix, https://seafarix.com/feed/, en-US, Finland diff --git a/datasets/corporations/computing.csv b/datasets/corporations/computing.csv index 6008848..f25c727 100644 --- a/datasets/corporations/computing.csv +++ b/datasets/corporations/computing.csv @@ -2,3 +2,7 @@ name, url, language, country Panic Blog, https://www.panic.com/blog/feed/, en-US, United States Necuno Solutions, https://necunos.com/feed.xml, en-US, Finland Debricked, https://debricked.com/blog/feed/, en-US, Europe +Linaro - Arm-based solutions, https://www.linaro.org/feed.xml, en-US, United States +Jolla, https://jolla.com/feed/, en-US, Finland +Seafarix, https://seafarix.com/feed/, en-US, Finland +Weingärtner IT Services, https://weingaertner-it.de/index.php/feed/, de-DE, Germany diff --git a/datasets/corporations/hardware.csv b/datasets/corporations/hardware.csv new file mode 100644 index 0000000..64506a9 --- /dev/null +++ b/datasets/corporations/hardware.csv @@ -0,0 +1,2 @@ +name, url, language, country +Linaro - Arm-based solutions, https://www.linaro.org/feed.xml, en-US, United States diff --git a/datasets/corporations/music.csv b/datasets/corporations/music.csv new file mode 100644 index 0000000..7eef8fd --- /dev/null +++ b/datasets/corporations/music.csv @@ -0,0 +1,2 @@ +name, url, language, country +DopeSONIX VSTi VST AU Hip Hop Beatmaking Production Software, https://www.dopesonix.com/feed/, en-UK, England diff --git a/datasets/forums/computer.csv b/datasets/forums/computer.csv index 6a238be..f4349d5 100644 --- a/datasets/forums/computer.csv +++ b/datasets/forums/computer.csv @@ -1,2 +1,5 @@ name, url, language, country eGPU.io, https://egpu.io/feed/, en-US, Worldwide +Arch Linux ARM, https://archlinuxarm.org/forum/feed.php, en-US, Worldwide +Arch Linux, https://bbs.archlinux.org/extern.php?action=feed&type=atom, en-US, Worldwide +Replicant: Messages, https://redmine.replicant.us/projects/replicant/activity.atom?show_messages=1, en-US, Worldwide diff --git a/datasets/forums/hacking.csv b/datasets/forums/hacking.csv index fdc52d8..0b2eb8d 100644 --- a/datasets/forums/hacking.csv +++ b/datasets/forums/hacking.csv @@ -1,3 +1,3 @@ name, url, language, country Leak Zone - Leaking & Cracking Forum, https://leakzone.net/syndication.php?type=atom1.0, en-US, Worldwide -Pale Moon, https://forum.palemoon.org/app.php/feed/forum/1, en-US, Worldwide + diff --git a/datasets/forums/intrenet.csv b/datasets/forums/intrenet.csv index 8990ab5..b4c3e5b 100644 --- a/datasets/forums/intrenet.csv +++ b/datasets/forums/intrenet.csv @@ -1,3 +1,4 @@ name, url, language, country Pale Moon, https://forum.palemoon.org/app.php/feed, en-US, Worldwide Transmission BT, https://forum.transmissionbt.com/app.php/feed, en-US, Worldwide +ikiwiki, http://ikiwiki.info/forum/index.rss, en-US, Worldwide diff --git a/datasets/forums/linux.csv b/datasets/forums/linux.csv new file mode 100644 index 0000000..cb1183b --- /dev/null +++ b/datasets/forums/linux.csv @@ -0,0 +1,3 @@ +name, url, language, country +Arch Linux ARM, https://archlinuxarm.org/forum/feed.php, en-US, Worldwide +Arch Linux, https://bbs.archlinux.org/extern.php?action=feed&type=atom, en-US, Worldwide diff --git a/datasets/news/art.csv b/datasets/news/art.csv index 053081e..63abc71 100644 --- a/datasets/news/art.csv +++ b/datasets/news/art.csv @@ -4,3 +4,4 @@ třiapůl česky, https://triapul.cz/feed/cesky.xml, cz-CZ, Czech triapul.cz, http://triapul.cz/feed/english.xml, en-US, Czech techno-mage in:, https://bsd.network/@prahou.rss, en-US, United States Behance Featured Projects, https://www.behance.net/feeds/projects, en-US, United States +Mansfield-Devine, https://mansfield-devine.com/feed/, en-US, Worldwide diff --git a/datasets/news/bsd.csv b/datasets/news/bsd.csv index 8a03840..22095ca 100644 --- a/datasets/news/bsd.csv +++ b/datasets/news/bsd.csv @@ -3,3 +3,4 @@ drkhsh, https://drkhsh.at/atom.xml, en-US, Austria OpenBSD Webzine, https://webzine.puffy.cafe/atom.xml, en-US, United States Solene'%, https://dataswamp.org/~solene/rss.xml, en-US, United States Solene'%, https://dataswamp.org/~solene/rss-html.xml, en-US, United States +Michael W Lucas, https://mwl.io/feed, en-US, United States diff --git a/datasets/news/business.csv b/datasets/news/business.csv index 35d1976..6cfff7a 100644 --- a/datasets/news/business.csv +++ b/datasets/news/business.csv @@ -14,3 +14,5 @@ ynet - כלכלה, https://www.ynet.co.il/Integration/StoryRss6.xml, he-IL, Isra דה מרקר - פרשנויות היום, https://www.themarker.com/srv/tm-opinions, he-IL, Israel שוק ההון והשקעות, https://www.globes.co.il/webservice/rss/rssfeeder.asmx/FeederNode?iID=585, he-IL, Israel StockNews, https://stocknews.com/feed/, en-US, United States +Phoronix, https://www.phoronix.com/rss.php, en-US, United States +Business Writing Info Blog, https://www.instructionalsolutions.com/blog/rss.xml, en-US, United States diff --git a/datasets/news/christianity.csv b/datasets/news/christianity.csv index dbb6a93..09efa44 100644 --- a/datasets/news/christianity.csv +++ b/datasets/news/christianity.csv @@ -3,3 +3,5 @@ Evangelical Endtimemachine, https://www.evangelicalendtimemachine.com/feed/, ce- Orthodox Christianity, https://orthochristian.com/xml/rss.xml, en-US, USA Legio Christi, https://legiochristi.com/rss/, en-US, USA Jerm Warfare, https://jermwarfare.com/feed, en-US, USA +Mom on a Mission, https://www.mom-on-a-mission.blog/all-posts?format=rss, en-US, United States +iMOM, https://www.imom.com/feed/, en-US, United States diff --git a/datasets/news/code.csv b/datasets/news/code.csv index b05d209..038df01 100644 --- a/datasets/news/code.csv +++ b/datasets/news/code.csv @@ -5,5 +5,9 @@ Newest Python PEPs, https://peps.python.org/peps.rss/, en-US, Worldwide Victor Stinner blog 3, https://vstinner.github.io/feeds/all.atom.xml, en-US, Worldwide Victor Stinner 🐍, https://mamot.fr/@vstinner.rss, en-US, Worldwide camen design, https://camendesign.com/rss, en-US, United States +Hugo Barrera, https://whynothugo.nl/posts.xml, en-US, Netherlands LZone | Devops, https://lzone.de/feed/devops.xml, en-US, Germany Leonora Tindall on Nora Codes, https://nora.codes/index.xml, en-US, United States +code monk | hacking habits, https://drj11.wordpress.com/feed/, en-US, United States +Machina Speculatrix, https://mansfield-devine.com/speculatrix/feed/, en-US, Worldwide +Lainblog, https://blog.soykaf.com/index.xml, en-US, Worldwide diff --git a/datasets/news/computer.csv b/datasets/news/computer.csv index 967df12..85cdff8 100644 --- a/datasets/news/computer.csv +++ b/datasets/news/computer.csv @@ -1,12 +1,26 @@ name, url, language, country +Michael W Lucas, https://mwl.io/feed, en-US, United States +Jan Wagemakers, https://www.janwagemakers.be/jekyll/feed.xml, en-US, Belgium +Remko Tronçon (mko.re), https://el-tramo.be/feed.xml, en-US, Germany +Hacking around oppressions, https://bouah.net/index.xml, en-US, France +Tatsumoto Ren, https://tatsumoto.neocities.org/blog/feed.rss, en-US, Japan +Hugo Barrera, https://whynothugo.nl/posts.xml, en-US, Netherlands Liliputing, https://liliputing.com/feed/, en-US, Worldwide +Mario Sabatino Gemini capsule, https://gemini.sabatino.cloud/rss.xml, it-IT, Italy +Lainblog, https://blog.soykaf.com/index.xml, en-US, Worldwide camen design, https://camendesign.com/rss, en-US, United States computers are bad, https://computer.rip/rss.xml, en-US, Worldwide +Machina Speculatrix, https://mansfield-devine.com/speculatrix/feed/, en-US, Worldwide Jacob McCormick, https://mccor.xyz/rss.xml, en-US, United States +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, en-US, Germany +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, de-DE, Germany OpenNews.opennet.ru: Общая лента новостей, https://www.opennet.ru/opennews/opennews_all_utf.rss, ru-RU, Russia ju_hnny5, https://blog.jbriault.fr/rss/, fr-FR, France +This Week in GNOME, https://thisweek.gnome.org/index.xml, en-US, Worldwide Lazy Reading | The Cyber Vanguard, https://cyber.dabamos.de/blog/feed.rss, en-US, United States +Christof Meerwald, https://cmeerw.org/blog.atom, en-US, Austria XXIIVV, https://wiki.xxiivv.com/links/rss.xml, en-US, Canada +Phoronix, https://www.phoronix.com/rss.php, en-US, United States drkhsh, https://drkhsh.at/atom.xml, en-US, Austria Codemadness, https://codemadness.org/atom.xml, en-US, United States Leonora Tindall on Nora Codes, https://nora.codes/index.xml, en-US, United States @@ -17,6 +31,7 @@ schollz, https://schollz.com/index.xml, en-US, United States LZone | Devops, https://lzone.de/feed/devops.xml, en-US, Germany The Logs, https://www.flu0r1ne.net/logs/rss.xml, en-US, United States Newest Python PEPs, https://peps.python.org/peps.rss/, en-US, Worldwide +monocles, https://monocles.social/@monocles.rss, en-US, Germany Ctrl blog: Mobile Apps, https://feed.ctrl.blog/topic/mobile-apps.atom, en-US, United States Chiportal, https://chiportal.co.il/feed/, he-IL, Israel Hackaday, https://hackaday.com/feed/, en-US, United States @@ -45,6 +60,7 @@ TuxPhones, https://tuxphones.com/rss/, en-US, United States OpenSource.net, https://opensource.net/feed/, en-US, United States Collabora Office and Collabora Online, https://www.collaboraoffice.com/feed/, en-US, United States LGUG2Z, https://lgug2z.com/index.xml, en-US, United States +code monk | hacking habits, https://drj11.wordpress.com/feed/, en-US, United States Karl Bartel's Website, https://www.karl.berlin/atom.xml, en-US, Germany Victor Stinner blog 3, https://vstinner.github.io/feeds/all.atom.xml, en-US, Worldwide Victor Stinner 🐍, https://mamot.fr/@vstinner.rss, en-US, Worldwide @@ -67,3 +83,4 @@ Enterprise Storage, http://feeds.fosketts.net/StephenFoskettPackRat_EnterpriseSt Personal, http://feeds.fosketts.net/StephenFoskettPackRat_Personal, en-US, United States Terabyte Home, http://feeds.fosketts.net/StephenFoskettPackRat_TerabyteHome, en-US, United States Virtual Storage, http://feeds.fosketts.net/StephenFoskettPackRat_VirtualStorage, en-US, United States +Jolla, https://jolla.com/feed/, en-US, Finland diff --git a/datasets/news/culture.csv b/datasets/news/culture.csv index 0b39dbc..dec8ac1 100644 --- a/datasets/news/culture.csv +++ b/datasets/news/culture.csv @@ -1,13 +1,17 @@ name, url, language, country -Jerm Warfare, https://jermwarfare.com/feed, en-US, USA -The Organic Prepper, https://www.theorganicprepper.com/feed/, en-US, USA -The Public Domain Review, https://publicdomainreview.org/rss.xml, en-US, USA +Jerm Warfare, https://jermwarfare.com/feed, en-US, United States +The Organic Prepper, https://www.theorganicprepper.com/feed/, en-US, United States +The Public Domain Review, https://publicdomainreview.org/rss.xml, en-US, United States ynet - תרבות, https://www.ynet.co.il/Integration/StoryRss538.xml, he-IL, Israel ynet - Culture, https://www.ynet.co.il/Integration/StoryRss3086.xml, en-IL, Israel ynet - חדר משלך, https://www.ynet.co.il/Integration/StoryRss4111.xml, he-IL, Israel -Tampa Free Press, https://www.tampafp.com/feed/, en-US, USA +Tampa Free Press, https://www.tampafp.com/feed/, en-US, United States Deutsche Welle: DW.com KULTUR & LEBEN, http://rss.dw.com/atom/rss-de-cul, de-DE, Germany Deutsche Welle: DW.com KULTUR & LEBEN: BÜCHER, http://rss.dw.com/atom/rss-de-cul-buch, de-DE, Germany Deutsche Welle: DW.com KULTUR & LEBEN: FILM, http://rss.dw.com/atom/rss-de-cul-film, de-DE, Germany Deutsche Welle: DW.com KULTUR & LEBEN: MUSIK, http://rss.dw.com/atom/rss-de-cul-musik, de-DE, Germany Deutsche Welle: DW.com DEUTSCHLAND ENTDECKEN, http://rss.dw.com/atom/rss-de-deutschlandentdecken, de-DE, Germany +Kopimi, https://kopimi.com/feed/, en-US, Sweden +Mashable, https://mashable.com/feeds/rss/all, en-US, United States +За вашу и нашу Свободу!, https://lev-sharansky2.livejournal.com/data/atom, ru-RU, Russia + diff --git a/datasets/news/cybersecurity.csv b/datasets/news/cybersecurity.csv index c1c2419..e7aeb65 100644 --- a/datasets/news/cybersecurity.csv +++ b/datasets/news/cybersecurity.csv @@ -1,4 +1,5 @@ name, url, language, country +Hacking around oppressions, https://bouah.net/index.xml, en-US, France Michael W Lucas, https://mwl.io/feed, en-US, United States Necuno Solutions, https://necunos.com/feed.xml, en-US, Finland Flipper Blog, https://blog.flipper.net/rss/, en-US, United States diff --git a/datasets/news/data.csv b/datasets/news/data.csv index 61cdfd6..b0395f5 100644 --- a/datasets/news/data.csv +++ b/datasets/news/data.csv @@ -1,2 +1,3 @@ name, url, language, country Postgres Professional, https://postgrespro.ru/rss, ru-RU, Russia +Datenschutz, https://www.datenschutz.org/feed/, de-DE, Germany diff --git a/datasets/news/design.csv b/datasets/news/design.csv index 080a471..daa187b 100644 --- a/datasets/news/design.csv +++ b/datasets/news/design.csv @@ -1,3 +1,5 @@ name, url, language, country Crumina, https://crumina.net/feed/, en-US, Ukraine Behance Featured Projects, https://www.behance.net/feeds/projects, en-US, United States +Dezeen, https://www.dezeen.com/feed/, en-US, Worldwide +designboom | architecture & design magazine, https://www.designboom.com/feed/, en-US, Worldwide diff --git a/datasets/news/electronics.csv b/datasets/news/electronics.csv index 047b52e..f668ee2 100644 --- a/datasets/news/electronics.csv +++ b/datasets/news/electronics.csv @@ -1,4 +1,5 @@ name, url, language, country +Jan Wagemakers, https://www.janwagemakers.be/jekyll/feed.xml, en-US, Belgium The Logs, https://www.flu0r1ne.net/logs/rss.xml, en-US, United States Flipper Blog, https://blog.flipper.net/rss/, en-US, United States ollieparanoid, https://ollieparanoid.github.io/feed.xml, en-US, United States @@ -10,3 +11,4 @@ c't Magazin, https://social.heise.de/@ct_Magazin.rss, de-DE, Germany electro·pizza, https://electro.pizza/feed.xml, en-US, United States Ηρακλής Παπαθεοδώρου • heracl.es, https://heracl.es/el/feed.xml, el-GR, Greece Heracles Papatheodorou • heracl.es, https://heracl.es/feed.xml, en-US, Greece +Machina Speculatrix, https://mansfield-devine.com/speculatrix/feed/, en-US, Worldwide diff --git a/datasets/news/english.csv b/datasets/news/english.csv new file mode 100644 index 0000000..0347b44 --- /dev/null +++ b/datasets/news/english.csv @@ -0,0 +1,2 @@ +name, url, language, country +Business Writing Info Blog, https://www.instructionalsolutions.com/blog/rss.xml, en-US, United States diff --git a/datasets/news/family.csv b/datasets/news/family.csv index 68ce299..1353a68 100644 --- a/datasets/news/family.csv +++ b/datasets/news/family.csv @@ -1,3 +1,5 @@ name, url, language, country Mom on a Mission, https://www.mom-on-a-mission.blog/all-posts?format=rss, en-US, United States ynet - הורים, https://www.ynet.co.il/Integration/StoryRss3052.xml, he-IL, Israel +iMOM, https://www.imom.com/feed/, en-US, United States +Mom on a Mission, https://momonamission.me/feed/, en-US, United States diff --git a/datasets/news/germany.csv b/datasets/news/germany.csv index e8d2d4b..1e21bd9 100644 --- a/datasets/news/germany.csv +++ b/datasets/news/germany.csv @@ -6,4 +6,6 @@ Deutsche Welle: DW.com WISSENSCHAFT, http://rss.dw.com/atom/rss-de-wissenschaft, Deutsche Welle: DW.com DEUTSCHLAND ENTDECKEN, http://rss.dw.com/atom/rss-de-deutschlandentdecken, de-DE, Germany Deutsche Welle: DW.com PRESSE, http://rss.dw.com/atom/presse, de-DE, Germany "ZEIT ONLINE | Nachrichten, News, Hintergründe und Debatten", https://newsfeed.zeit.de/index, de-DE, Germany -strukturart.com latest articles,https://strukturart.com/feed, de-DE, Germany +strukturart.com latest articles,https://strukturart.com/feed, de-DE, Germany +classless Kulla, https://www.classless.org/feed/atom/, de-DE, Germany +classless Kulla, https://www.classless.org/feed/atom/, en-US, Germany diff --git a/datasets/news/government.csv b/datasets/news/government.csv index 32cfcaa..bddb403 100644 --- a/datasets/news/government.csv +++ b/datasets/news/government.csv @@ -11,3 +11,5 @@ Tampa Free Press, https://www.tampafp.com/feed/, en-US, United States Deutsche Welle: DW.com POLITIK, http://rss.dw.com/atom/rss_de_politik, de-DE, Germany Real Liberty Media, https://www.reallibertymedia.com/feed/, en-US, United States Robert Reich, https://robertreich.substack.com/feed, en-US, United States +Historic.ly, https://www.historicly.net/feed, en-US, United States +Reclaim The Net, https://reclaimthenet.org/feed, en-US, United States diff --git a/datasets/news/grammar.csv b/datasets/news/grammar.csv new file mode 100644 index 0000000..0347b44 --- /dev/null +++ b/datasets/news/grammar.csv @@ -0,0 +1,2 @@ +name, url, language, country +Business Writing Info Blog, https://www.instructionalsolutions.com/blog/rss.xml, en-US, United States diff --git a/datasets/news/hacking.csv b/datasets/news/hacking.csv index 3e06d33..3612966 100644 --- a/datasets/news/hacking.csv +++ b/datasets/news/hacking.csv @@ -1,2 +1,3 @@ name, url, language, country Flipper Blog, https://blog.flipper.net/rss/, en-US, United States +code monk | hacking habits, https://drj11.wordpress.com/feed/, en-US, United States diff --git a/datasets/news/hardware.csv b/datasets/news/hardware.csv new file mode 100644 index 0000000..51676f8 --- /dev/null +++ b/datasets/news/hardware.csv @@ -0,0 +1,2 @@ +name, url, language, country +Phoronix, https://www.phoronix.com/rss.php, en-US, United States diff --git a/datasets/news/history.csv b/datasets/news/history.csv new file mode 100644 index 0000000..031c6fa --- /dev/null +++ b/datasets/news/history.csv @@ -0,0 +1,4 @@ +name, url, language, country +Historic.ly, https://www.historicly.net/feed, en-US, United States +classless Kulla, https://www.classless.org/feed/atom/, de-DE, Germany +classless Kulla, https://www.classless.org/feed/atom/, en-US, Germany diff --git a/datasets/news/internet.csv b/datasets/news/internet.csv index e056e51..88acfb6 100644 --- a/datasets/news/internet.csv +++ b/datasets/news/internet.csv @@ -6,6 +6,7 @@ WordPress.com News, https://wordpress.com/blog/feed/, en-US, United States Micro.blog News, https://news.micro.blog/feed.xml, en-US, United States EFF Action Center, https://act.eff.org/action.atom, en-US, United States Leonora Tindall on Nora Codes, https://nora.codes/index.xml, en-US, United States +Reclaim The Net, https://reclaimthenet.org/feed, en-US, United States Melon's Thoughts - Guidebooks, https://thoughts.melonking.net/atom/?section=guides, en-US, United States thecozy.cat, https://thecozy.cat/feed/, en-US, United States OpenNews.opennet.ru: Общая лента новостей, https://www.opennet.ru/opennews/opennews_all_utf.rss, ru-RU, Russia diff --git a/datasets/news/law.csv b/datasets/news/law.csv index 0f8ec96..c240e5e 100644 --- a/datasets/news/law.csv +++ b/datasets/news/law.csv @@ -1,2 +1,4 @@ name, url, language, country North Carolina Lawyers Weekly, https://nclawyersweekly.com/feed/, en-US, United States +Avvocato a Roma | Studio Legale Sabatino | Famiglia e civile, https://www.studiosabatino.it/feed/, it-IT, Italy +La Pagina Giuridica, https://www.lapaginagiuridica.it/?feed=rss2, it-IT, Italy diff --git a/datasets/news/linux.csv b/datasets/news/linux.csv index 410a50a..ab18e92 100644 --- a/datasets/news/linux.csv +++ b/datasets/news/linux.csv @@ -1,13 +1,21 @@ name, url, language, country +Michael W Lucas, https://mwl.io/feed, en-US, United States +Jan Wagemakers, https://www.janwagemakers.be/jekyll/feed.xml, en-US, Belgium +code monk | hacking habits, https://drj11.wordpress.com/feed/, en-US, United States +Remko Tronçon (mko.re), https://el-tramo.be/feed.xml, en-US, Germany Linux Professional Institute (LPI), https://www.lpi.org/feed/, en-US, Canada LinuxConfig, https://linuxconfig.org/feed, en-US, Worldwide The Logs, https://www.flu0r1ne.net/logs/rss.xml, en-US, United States +Hugo Barrera, https://whynothugo.nl/posts.xml, en-US, Netherlands Ctrl blog: Fedora Linux, https://feed.ctrl.blog/topic/fedora-linux.atom, en-US, United States Ctrl blog: Linux, https://feed.ctrl.blog/topic/linux.atom, en-US, United States Anjan Momi, https://momi.ca/feed.xml, en-US, United States +This Week in GNOME, https://thisweek.gnome.org/index.xml, en-US, Worldwide +Phoronix, https://www.phoronix.com/rss.php, en-US, United States postmarketOS, https://postmarketos.org/blog/feed.atom, en-US, United States PureTryOut (Alpine Linux, postmarketOS), https://mastodon.fam-ribbers.com/@bart.rss, en-US, Netherlands Blog on Bart Ribbers - PureTryOut (Alpine Linux, postmarketOS), https://fam-ribbers.com/blog/index.xml, en-US, Netherlands +Christof Meerwald, https://cmeerw.org/blog.atom, en-US, Austria LZone | Devops, https://lzone.de/feed/devops.xml, en-US, Germany Necuno Solutions, https://necunos.com/feed.xml, en-US, Finland Laslo Hunhold, https://laslo.hunhold.de/index.xml, en-US, Germany diff --git a/datasets/news/literature.csv b/datasets/news/literature.csv index fb63ed9..c33e6f4 100644 --- a/datasets/news/literature.csv +++ b/datasets/news/literature.csv @@ -3,3 +3,7 @@ Stories by Williams, https://storiesbywilliams.com/feed/, en-US, United States Lili Saintcrow, https://www.lilithsaintcrow.com/feed/, en-US, United States Shannon Kay, https://blog.shannonkay.me/feed.xml, en-US, United States V.K. Dixon, https://vkdixon.substack.com/feed, en-US, United States +Business Writing Info Blog, https://www.instructionalsolutions.com/blog/rss.xml, en-US, United States +Michael W Lucas, https://mwl.io/feed, en-US, United States +classless Kulla, https://www.classless.org/feed/atom/, de-DE, Germany +classless Kulla, https://www.classless.org/feed/atom/, en-US, Germany diff --git a/datasets/news/misc.csv b/datasets/news/misc.csv index f51fcaa..b828987 100644 --- a/datasets/news/misc.csv +++ b/datasets/news/misc.csv @@ -4,3 +4,4 @@ Kasparov, https://www.kasparov.com/feed/, en-US, United States Ctrl blog: Metadata, https://feed.ctrl.blog/topic/metadata.atom, en-US, United States Ctrl blog: Compression, https://feed.ctrl.blog/topic/compression.atom, en-US, United States Ctrl blog: Backup, https://feed.ctrl.blog/topic/backup.atom, en-US, United States +Kopimi, https://kopimi.com/feed/, en-US, Sweden diff --git a/datasets/news/people.csv b/datasets/news/people.csv index b39c5c2..a59c98a 100644 --- a/datasets/news/people.csv +++ b/datasets/news/people.csv @@ -1,10 +1,22 @@ name, url, language, country +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, en-US, Germany +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, de-DE, Germany +Jan Wagemakers, https://www.janwagemakers.be/jekyll/feed.xml, en-US, Belgium +immanuel60, https://immanuel60.hu/feed/, hu-HU, Hungary +Hugo Barrera, https://whynothugo.nl/posts.xml, en-US, Netherlands +Remko Tronçon (mko.re), https://el-tramo.be/feed.xml, en-US, Germany +Hacking around oppressions, https://bouah.net/index.xml, en-US, France +iMOM, https://www.imom.com/feed/, en-US, United States +Mario Sabatino Gemini capsule, https://gemini.sabatino.cloud/rss.xml, it-IT, Italy +Mom on a Mission, https://www.mom-on-a-mission.blog/all-posts?format=rss, en-US, United States Michael W Lucas, https://mwl.io/feed, en-US, United States Alixander Court, https://alixandercourt.com/feed/, en-US, United States Anjan Momi, https://momi.ca/feed.xml, en-US, United States Amethyst Reese, https://noswap.com/feed.xml, en-US, United States Dan d'Auge, https://fedi.thechangebook.org/feed/dandauge, fr-FR, France Dan d'Auge, https://mamot.fr/@dandauge.rss, fr-FR, France +Christof Meerwald, https://cmeerw.org/blog.atom, en-US, Austria +Christof Meerwald, http://cmeerw.org/blog.rdf, en-US, Austria PureTryOut, https://mastodon.fam-ribbers.com/@bart.rss, en-US, Netherlands Tristan B. Velloza Kildaire, https://gleasonator.com/@deavmi, en-ZA, South Africa Blog on Bart Ribbers - PureTryOut, https://fam-ribbers.com/blog/index.xml, en-US, Netherlands diff --git a/datasets/news/privacy.csv b/datasets/news/privacy.csv index 86e26cb..45e1d55 100644 --- a/datasets/news/privacy.csv +++ b/datasets/news/privacy.csv @@ -1,4 +1,5 @@ name, url, language, country +Hacking around oppressions, https://bouah.net/index.xml, en-US, France The Logs, https://www.flu0r1ne.net/logs/rss.xml, en-US, United States Necuno Solutions, https://necunos.com/feed.xml, en-US, Finland ollieparanoid, https://ollieparanoid.github.io/feed.xml, en-US, United States @@ -9,3 +10,4 @@ FSFE News (FR), https://fsfe.org/news/news.fr.rss, fr-FR, Germany EFF Action Center, https://act.eff.org/action.atom, en-US, United States Spread Privacy, https://spreadprivacy.com/rss/, en-US, United States Ctrl blog: Privacy, https://feed.ctrl.blog/topic/privacy.atom, en-US, United States +Reclaim The Net, https://reclaimthenet.org/feed, en-US, United States diff --git a/datasets/news/russia.csv b/datasets/news/russia.csv index a0d5e34..c127680 100644 --- a/datasets/news/russia.csv +++ b/datasets/news/russia.csv @@ -1,3 +1,4 @@ name, url, language, country Kasparov, https://www.kasparov.com/feed/, en-US, United States РЖЕВСКАЯ ПРАВДА, https://presska.ru/feed/, ru-RU, Russia +За вашу и нашу Свободу!, https://lev-sharansky2.livejournal.com/data/atom, ru-RU, Russia diff --git a/datasets/news/surveillance.csv b/datasets/news/surveillance.csv index 18497c1..d9711ee 100644 --- a/datasets/news/surveillance.csv +++ b/datasets/news/surveillance.csv @@ -1,2 +1,3 @@ name, url, language, country Necuno Solutions, https://necunos.com/feed.xml, en-US, Finland +Reclaim The Net, https://reclaimthenet.org/feed, en-US, United States diff --git a/datasets/news/technology.csv b/datasets/news/technology.csv index fac5ce6..18b86b6 100644 --- a/datasets/news/technology.csv +++ b/datasets/news/technology.csv @@ -1,4 +1,5 @@ name, url, language, country +Hacking around oppressions, https://bouah.net/index.xml, en-US, France Redecentralize Blog, https://redecentralize.org/blog/feed.rss, en-US, United States Developpez, https://www.developpez.com/index/atom, fr-FR, France The Logs, https://www.flu0r1ne.net/logs/rss.xml, en-US, United States @@ -10,6 +11,9 @@ Futurism, https://futurism.com/feed, en-US, United States drkhsh, https://drkhsh.at/atom.xml, en-US, Austria Automation Rhapsody, https://automationrhapsody.com/feed/, en-US, United States Deep Thoughts by Raymond Hettinger, https://rhettinger.wordpress.com/feed/, en-US, United States +Lainblog, https://blog.soykaf.com/index.xml, en-US, Worldwide +code monk | hacking habits, https://drj11.wordpress.com/feed/, en-US, United States +Reclaim The Net, https://reclaimthenet.org/feed, en-US, United States Chiportal, https://chiportal.co.il/feed/, he-IL, Israel Hackaday, https://hackaday.com/feed/, en-US, United States LZone | Devops, https://lzone.de/feed/devops.xml, en-US, Germany @@ -30,3 +34,5 @@ Heracles Papatheodorou • heracl.es, https://heracl.es/feed.xml, en-US, Greece lectronice :: now, https://now.lectronice.com/feed.xml, en-US, United States Ctrl blog: Advertisement Technology, https://feed.ctrl.blog/topic/advertisement-technology.atom, en-US, United States OpenNews.opennet.ru: Общая лента новостей, https://www.opennet.ru/opennews/opennews_all_utf.rss, ru-RU, Russia +ExtremeTech, https://www.extremetech.com/feed, en-US, Worldwide +Slashdot, http://rss.slashdot.org/Slashdot/slashdot/to, en-US, Worldwide diff --git a/datasets/news/telecom.csv b/datasets/news/telecom.csv index 0a3e950..c8e4a15 100644 --- a/datasets/news/telecom.csv +++ b/datasets/news/telecom.csv @@ -3,3 +3,4 @@ The XMPP Blog on XMPP, https://xmpp.org/feeds/all.atom.xml, en-US, Worldwide Redecentralize Blog, https://redecentralize.org/blog/feed.rss, en-US, United States ollieparanoid, https://ollieparanoid.github.io/feed.xml, en-US, United States ShadowKat Studios, https://shadowkat.net/rss.xml, en-US, Worldwide +Reclaim The Net, https://reclaimthenet.org/feed, en-US, United States diff --git a/datasets/news/united_states.csv b/datasets/news/united_states.csv index a1de527..2e54d6d 100644 --- a/datasets/news/united_states.csv +++ b/datasets/news/united_states.csv @@ -8,3 +8,4 @@ Tampa Free Press, https://www.tampafp.com/feed/, en-US, United States Real Liberty Media, https://www.reallibertymedia.com/feed/, en-US, United States Robert Reich, https://robertreich.substack.com/feed, en-US, United States North Carolina Lawyers Weekly, https://nclawyersweekly.com/feed/, en-US, United States +The Good Men Project, https://goodmenproject.com/feed/, en-US, United States diff --git a/datasets/news/unix.csv b/datasets/news/unix.csv index 99a00d9..44427b7 100644 --- a/datasets/news/unix.csv +++ b/datasets/news/unix.csv @@ -1,15 +1,18 @@ name, url, language, country +Michael W Lucas, https://mwl.io/feed, en-US, United States Linux Professional Institute (LPI), https://www.lpi.org/feed/, en-US, Canada LinuxConfig, https://linuxconfig.org/feed, en-US, Worldwide The Logs, https://www.flu0r1ne.net/logs/rss.xml, en-US, United States Ctrl blog: Fedora Linux, https://feed.ctrl.blog/topic/fedora-linux.atom, en-US, United States Ctrl blog: Linux, https://feed.ctrl.blog/topic/linux.atom, en-US, United States +This Week in GNOME, https://thisweek.gnome.org/index.xml, en-US, Worldwide Anjan Momi, https://momi.ca/feed.xml, en-US, United States postmarketOS, https://postmarketos.org/blog/feed.atom, en-US, United States PureTryOut (Alpine Linux, postmarketOS), https://mastodon.fam-ribbers.com/@bart.rss, en-US, Netherlands Blog on Bart Ribbers - PureTryOut (Alpine Linux, postmarketOS), https://fam-ribbers.com/blog/index.xml, en-US, Netherlands LZone | Devops, https://lzone.de/feed/devops.xml, en-US, Germany Necuno Solutions, https://necunos.com/feed.xml, en-US, Finland +Phoronix, https://www.phoronix.com/rss.php, en-US, United States Laslo Hunhold, https://laslo.hunhold.de/index.xml, en-US, Germany Alpine Linux, https://fosstodon.org/@alpinelinux.rss, en-US, United States ollieparanoid, https://ollieparanoid.github.io/feed.xml, en-US, United States diff --git a/datasets/news/war.csv b/datasets/news/war.csv index c01a38c..877067a 100644 --- a/datasets/news/war.csv +++ b/datasets/news/war.csv @@ -1,3 +1,5 @@ name, url, language, country Jerm Warfare, https://jermwarfare.com/feed, en-US, USA Daily Stormer, https://dailystormer.in/feed/, en-US, USA +classless Kulla, https://www.classless.org/feed/atom/, de-DE, Germany +classless Kulla, https://www.classless.org/feed/atom/, en-US, Germany diff --git a/datasets/news/xmpp.csv b/datasets/news/xmpp.csv index 4a5ecb4..ea31d18 100644 --- a/datasets/news/xmpp.csv +++ b/datasets/news/xmpp.csv @@ -1,5 +1,17 @@ name, url, language, country +Hacking around oppressions, https://bouah.net/index.xml, en-US, France The XMPP Blog on XMPP, https://xmpp.org/feeds/all.atom.xml, en-US, Worldwide Swift IM, https://swift.im/wordpress/index.php/feed/, en-US, Worldwide Berlin XMPP Meetup, https://mov.im/feed/pubsub.movim.eu/berlin-xmpp-meetup, en-US, Germany Хомяк diSabler'а, https://dsy.name/feed/, ru-RU, Russia +JabberES.org, https://www.jabberes.org/index.xml, es-ES, Spain +Hot-Chilli, https://www.hot-chilli.net/feed/, de-DE, Germany +Jabber.de, https://www.jabber.de/?feed=rss2, de-DE, Germany +blabber.im, https://kanoa.de/@blabber.rss, en-US, Worldwide +monocles, https://monocles.social/@monocles.rss, en-US, Germany +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, de-DE, Germany +Martins Schmierzettel, https://blog.mdosch.de/index.xml, de-DE, Germany +Christof Meerwald, https://cmeerw.org/blog.atom, en-US, Austria +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, en-US, Germany +Alle Neuigkeiten von jotwewe.de, http://www.jotwewe.de/alles_de.xml, de-DE, Germany +Mario Sabatino Gemini capsule, https://gemini.sabatino.cloud/rss.xml, it-IT, Italy diff --git a/datasets/organizations/germany.csv b/datasets/organizations/germany.csv index e182833..e942c74 100644 --- a/datasets/organizations/germany.csv +++ b/datasets/organizations/germany.csv @@ -2,3 +2,5 @@ name, url, language, country FSFE News, https://fsfe.org/news/news.en.rss, en-US, Germany FSFE News (DE), https://fsfe.org/news/news.de.rss, de-DE, Germany FSFE News (FR), https://fsfe.org/news/news.fr.rss, fr-FR, Germany +Sovereign Tech Fund, https://www.sovereigntechfund.de/feed.rss, en-US, Germany +Sovereign Tech Fund, https://www.sovereigntechfund.de/de/feed.rss, de-DE, Germany diff --git a/datasets/organizations/united_states.csv b/datasets/organizations/united_states.csv index e05a9cf..2c412d0 100644 --- a/datasets/organizations/united_states.csv +++ b/datasets/organizations/united_states.csv @@ -1,3 +1,7 @@ name, url, language, country National Alliance, https://www.natall.com/feed/, en-US, USA EFF Action Center, https://act.eff.org/action.atom, en-US, United States +Polaris (Human Trafficking), https://polarisproject.org/feed/, en-US, United States +Thorn (Human Trafficking), https://www.thorn.org/feed/, en-US, United States +CATW (Human Trafficking), https://catwinternational.org/feed/, en-US, United States +Freedom Light (Human Trafficking), https://www.freedomlight.org/feed/, en-US, United States diff --git a/datasets/podcasts/computer.csv b/datasets/podcasts/computer.csv index 090d6ba..72225d6 100644 --- a/datasets/podcasts/computer.csv +++ b/datasets/podcasts/computer.csv @@ -2,5 +2,8 @@ name, url, language, country Hacker Public Radio, http://hackerpublicradio.org/hpr_spx_rss.php, en-US, United States postmarketOS, https://cast.postmarketos.org/feed.rss, en-US, Germany 0d - Zeroday, https://zeroday-podcast.de/feed/mp3/, de-DE, Germany +0d - Zeroday, https://0x0d.de/feed/mp3/, de-DE, Germany Software Freedom Podcast (OPUS), http://fsfe.org/news/podcast-opus.en.rss, en-US, Germany Software Freedom Podcast (MP3), http://fsfe.org/news/podcast.en.rss, en-US, Germany +Zeroday Podcast (stefan), https://podcasts.social/@zeroday.rss, de-DE, Germany +LPG Flash!, https://anchor.fm/s/430dae74/podcast/rss, it-IT, Italy diff --git a/datasets/podcasts/law.csv b/datasets/podcasts/law.csv new file mode 100644 index 0000000..8cf53d8 --- /dev/null +++ b/datasets/podcasts/law.csv @@ -0,0 +1,2 @@ +name, url, language, country +LPG Flash!, https://anchor.fm/s/430dae74/podcast/rss, it-IT, Italy diff --git a/datasets/podcasts/technology.csv b/datasets/podcasts/technology.csv index bf48452..a7f732b 100644 --- a/datasets/podcasts/technology.csv +++ b/datasets/podcasts/technology.csv @@ -5,3 +5,4 @@ Software Freedom Podcast, http://fsfe.org/news/podcast-opus.en.rss, en-US, Germa Software Freedom Podcast, http://fsfe.org/news/podcast-opus.en.rss, en-US, Germany Redecentralize Blog, https://redecentralize.org/blog/feed.rss, en-US, United States // aNONradio //, https://anonradio.net/feed/, en-US, Worldwide +LPG Flash!, https://anchor.fm/s/430dae74/podcast/rss, it-IT, Italy diff --git a/datasets/projects/chat.csv b/datasets/projects/chat.csv index b23d33a..12dc63d 100644 --- a/datasets/projects/chat.csv +++ b/datasets/projects/chat.csv @@ -6,3 +6,9 @@ Dino, https://dino.im/index.xml, en-US, Worldwide ratox, https://git.2f30.org/ratox/atom.xml, en-US, Worldwide buzz, https://git.2f30.org/buzz/atom.xml, en-US, Worldwide mIRC, https://www.mirc.com/feed/, en-US, Worldwide +blabber.im, https://kanoa.de/@blabber.rss, en-US, Worldwide +monocles, https://monocles.social/@monocles.rss, en-US, Germany +Gajim, https://gajim.org/index.xml, en-US, France +Cabal, https://mastodon.social/@cabal.rss, en-US, Worldwide +CoyIM, https://coy.im/feed.xml, en-US, Worldwide +Nextcloud, https://nextcloud.com/feed/, en-US, Worldwide diff --git a/datasets/projects/cloud.csv b/datasets/projects/cloud.csv new file mode 100644 index 0000000..2a4ca3b --- /dev/null +++ b/datasets/projects/cloud.csv @@ -0,0 +1,2 @@ +name, url, language, country +Nextcloud, https://nextcloud.com/feed/, en-US, Worldwide diff --git a/datasets/projects/desktop.csv b/datasets/projects/desktop.csv new file mode 100644 index 0000000..0844846 --- /dev/null +++ b/datasets/projects/desktop.csv @@ -0,0 +1,3 @@ +name, url, language, country +"fluxbox.git, branch master", http://git.fluxbox.org/fluxbox.git/atom/?h=master, en-US, Worldwide +This Week in GNOME, https://thisweek.gnome.org/index.xml, en-US, Worldwide diff --git a/datasets/projects/email.csv b/datasets/projects/email.csv new file mode 100644 index 0000000..ec5cb1c --- /dev/null +++ b/datasets/projects/email.csv @@ -0,0 +1,3 @@ +name, url, language, country +Nextcloud, https://nextcloud.com/feed/, en-US, Worldwide +Roundcube, https://roundcube.net/feeds/atom.xml, en-US, Worldwide diff --git a/datasets/projects/fediverse.csv b/datasets/projects/fediverse.csv new file mode 100644 index 0000000..8385b21 --- /dev/null +++ b/datasets/projects/fediverse.csv @@ -0,0 +1,2 @@ +name, url, language, country +GNU social, http://www.gnusocial.rocks/feed.rss, en-US, Worldwide diff --git a/datasets/projects/internet.csv b/datasets/projects/internet.csv index 7944827..c471944 100644 --- a/datasets/projects/internet.csv +++ b/datasets/projects/internet.csv @@ -2,6 +2,7 @@ name, url, language, country DownThemAll!, https://www.downthemall.org/feed, en-US, Worldwide Pale Moon, https://forum.palemoon.org/app.php/feed/forum/1, en-US, Worldwide Falkon - KDE web browser, https://www.falkon.org/atom.xml, en-US, Czech +Otter Browser, https://otter-browser.org/feed/, en-US, Worldwide What's Up, Fraidycat?, https://fraidyc.at/blog/feed.xml, en-US, Worldwide geomyidae, branch HEAD, gopher://bitreich.org:70/0/scm/geomyidae/atom.xml, en-US, United States rclone - rsync for cloud storage, https://rclone.org/index.xml, en-US, Worldwide @@ -9,6 +10,7 @@ Feedbin, https://feedbin.com/blog/atom.xml, en-US, United States Silvio Rizzi (reederapp.com), https://gloria.social/@rizzi.rss, en-US, Worldwide lighty, https://redmine.lighttpd.net/news.atom, en-US, Worldwide lighttpd, https://www.lighttpd.net/feed/atom.xml, en-US, Worldwide +Nextcloud, https://nextcloud.com/feed/, en-US, Worldwide Cabal, https://mastodon.social/@cabal.rss, en-US, Worldwide JSON Feed, https://www.jsonfeed.org/feed.xml, en-US, Worldwide Liferea, https://lzone.de/feed/liferea.xml, en-US, Germany @@ -18,3 +20,8 @@ Kaidan, https://www.kaidan.im/atom.xml, en-US, Worldwide Dino, https://dino.im/index.xml, en-US, Worldwide Vivaldi Browser, https://vivaldi.com/feed/, en-US, United States wicd Announcements, http://feeds.launchpad.net/wicd/announcements.atom, en-US, Worldwide +GNU social, http://www.gnusocial.rocks/feed.rss, en-US, Worldwide +CoyIM, https://coy.im/feed.xml, en-US, Worldwide +Tor Project blog, https://blog.torproject.org/feed.xml, en-US, Worldwide +blabber.im, https://kanoa.de/@blabber.rss, en-US, Worldwide +monocles, https://monocles.social/@monocles.rss, en-US, Germany diff --git a/datasets/projects/mix_network.csv b/datasets/projects/mix_network.csv new file mode 100644 index 0000000..064780e --- /dev/null +++ b/datasets/projects/mix_network.csv @@ -0,0 +1,2 @@ +name, url, language, country +Tor Project blog, https://blog.torproject.org/feed.xml, en-US, Worldwide diff --git a/datasets/projects/operating_system.csv b/datasets/projects/operating_system.csv index 4b2c227..d12668e 100644 --- a/datasets/projects/operating_system.csv +++ b/datasets/projects/operating_system.csv @@ -1,6 +1,8 @@ name, url, language, country -NixOS, https://nixos.org/blog/announcements-rss.xml, en-US, USA +NixOS, https://nixos.org/blog/announcements-rss.xml, en-US, United States ReactOS, https://reactos.org/index.xml, en-US, Germany Salix OS, https://forum.salixos.org/app.php/feed/news, en-US, Greece postmarketOS, https://postmarketos.org/blog/feed.atom, en-US, Germany cynix, https://git.2f30.org/cynix/atom.xml, en-US, Worldwide +Replicant, https://blog.replicant.us/feed/, en-US, United States +Sailfish OS, https://sailfishos.org/feed/, en-US, Finland diff --git a/datasets/projects/website.csv b/datasets/projects/website.csv index 43dffe4..2b4873f 100644 --- a/datasets/projects/website.csv +++ b/datasets/projects/website.csv @@ -13,6 +13,8 @@ Textpattern CMS, https://textpattern.com/rss, en-US, Worldwide TYPO3, https://typo3.org/rss, en-US, Worldwide WordPress, https://wordpress.org/news/feed/, en-US, Worldwide Foswiki, https://foswiki.org/Home/WebRss, en-US, Worldwide +Foswiki, https://blog.foswiki.org/Blog/WebAtom?contenttype=text/xml, en-US, Worldwide +Extensions - Foswiki, https://foswiki.org/Extensions/ExtensionNews?template=ExtensionNewsRssView&contenttype=text/xml, en-US, Worldwide Eleventy, https://www.11ty.dev/blog/feed.xml, en-US, Worldwide bliper, https://git.2f30.org/bliper/atom.xml, en-US, Worldwide LinkStack, https://linkstack.org/feed/, en-US, Worldwide diff --git a/datasets/projects/xmpp.csv b/datasets/projects/xmpp.csv new file mode 100644 index 0000000..ff6948a --- /dev/null +++ b/datasets/projects/xmpp.csv @@ -0,0 +1,7 @@ +name, url, language, country +Swift IM, https://swift.im/wordpress/index.php/feed/, en-US, Worldwide +Kaidan, https://www.kaidan.im/atom.xml, en-US, Worldwide +Dino, https://dino.im/index.xml, en-US, Worldwide +CoyIM, https://coy.im/feed.xml, en-US, Worldwide +blabber.im, https://kanoa.de/@blabber.rss, en-US, Worldwide +monocles, https://monocles.social/@monocles.rss, en-US, Germany diff --git a/datasets/tutorials/code.csv b/datasets/tutorials/code.csv index be18932..19bac1b 100644 --- a/datasets/tutorials/code.csv +++ b/datasets/tutorials/code.csv @@ -1,2 +1,3 @@ name, url, language, country Karl Bartel's Website, https://www.karl.berlin/atom.xml, en-US, Germany +freeCodeCamp, https://www.freecodecamp.org/news/rss/, en-US, Worldwide diff --git a/datasets/tutorials/python.csv b/datasets/tutorials/python.csv index 16e4773..3d4d411 100644 --- a/datasets/tutorials/python.csv +++ b/datasets/tutorials/python.csv @@ -11,3 +11,4 @@ AppDividend, https://appdividend.com/feed/, en-US, Worldwide Deep Thoughts by Raymond Hettinger, https://rhettinger.wordpress.com/feed/, en-US, USA Victor Stinner blog 3, https://vstinner.github.io/feeds/all.atom.xml, en-US, Worldwide Victor Stinner 🐍, https://mamot.fr/@vstinner.rss, en-US, Worldwide +Dan Bader, https://dbader.org/rss, en-US, Worldwide diff --git a/datasets/videos/general.csv b/datasets/videos/general.csv index 663c8d3..b3bbf57 100644 --- a/datasets/videos/general.csv +++ b/datasets/videos/general.csv @@ -3,3 +3,4 @@ denshi.live, https://denshi.live/feeds/videos.xml, en-US, United States Luke's Videos, https://videos.lukesmith.xyz/feeds/videos.xml, en-US, United States fair.tube, https://fair.tube/feeds/videos.xml, de-DE, Germany Kockatoo Tube, https://tube.kockatoo.org/feeds/videos.xml, en-US, United States +sp-codes PeerTube, https://tube.sp-codes.de/feeds/videos.xml, de-DE, Germany diff --git a/setup.py b/setup.py index dd73b52..9d012fe 100644 --- a/setup.py +++ b/setup.py @@ -3,13 +3,47 @@ from distutils.core import setup setup( - name='slixfeed', + name='Slixfeed', version='1.0', - description='rss through xmpp bot', + description='RSS news bot for XMPP', + long_description='Slixfeed is a news aggregator bot for online news feeds. This program is primarily designed for XMPP', + author='Schimon Jehudah Zakai Zockaim Zachary', + author_email='sjehuda@yandex.com', + url='https://gitgud.io/sjehuda/slixfeed', + license='MIT', + platforms=['any'], install_require=[ 'aiohttp', - 'slixmpp', - 'eliot', - 'feedparser' - ] + 'bs4', + 'feedparser', + 'lxml', + 'slixmpp' + ], + classifiers=[ + 'Framework :: slixmpp', + 'Intended Audience :: End Users/Desktop', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3.10', + 'Topic :: Communications :: Chat', + 'Topic :: Internet :: Extensible Messaging and Presence Protocol (XMPP)', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content :: News/Diary', + 'Topic :: Internet :: Instant Messaging', + 'Topic :: Internet :: XMPP', + 'Topic :: Office/Business :: News/Diary', + ], + keywords=[ + 'jabber', + 'xmpp', + 'bot', + 'chat', + 'im', + 'news', + 'atom', + 'rdf', + 'rss', + 'syndication' + ], ) diff --git a/slixfeed/confighandler.py b/slixfeed/confighandler.py index 5820be7..01a4f3a 100644 --- a/slixfeed/confighandler.py +++ b/slixfeed/confighandler.py @@ -39,16 +39,18 @@ async def get_value_default(key): result = "crim,dead,death,disaster,murder,war" case "interval": result = 300 + case "masters": + result = randrange(100000, 999999) + case "length": + result = 300 + case "old": + result = 0 case "quantum": result = 3 case "random": result = 0 - case "masters": - result = randrange(100000, 999999) case "token": result = "none" - case "old": - result = 0 return result diff --git a/slixfeed/datahandler.py b/slixfeed/datahandler.py index 7c9e314..c3470f2 100644 --- a/slixfeed/datahandler.py +++ b/slixfeed/datahandler.py @@ -110,21 +110,6 @@ async def download_updates(db_file, url=None): ) # new_entry = 0 for entry in entries: - if entry.has_key("title"): - # title = entry.title - title = "{}: *{}*".format(feed["feed"]["title"], entry.title) - else: - title = feed["feed"]["title"] - if entry.has_key("link"): - # link = complete_url(source, entry.link) - link = await join_url(source, entry.link) - link = await trim_url(link) - else: - link = source - if entry.has_key("id"): - eid = entry.id - else: - eid = link # TODO Pass date too for comparion check if entry.has_key("published"): date = entry.published @@ -139,6 +124,22 @@ async def download_updates(db_file, url=None): # NOTE Would seconds result in better database performance # date = datetime.datetime(date) # date = (date-datetime.datetime(1970,1,1)).total_seconds() + if entry.has_key("title"): + title = entry.title + # title = "{}: *{}*".format(feed["feed"]["title"], entry.title) + else: + title = date + # title = feed["feed"]["title"] + if entry.has_key("link"): + # link = complete_url(source, entry.link) + link = await join_url(source, entry.link) + link = await trim_url(link) + else: + link = source + if entry.has_key("id"): + eid = entry.id + else: + eid = link exist = await sqlitehandler.check_entry_exist( db_file, source, @@ -152,14 +153,14 @@ async def download_updates(db_file, url=None): # TODO Enhance summary if entry.has_key("summary"): summary = entry.summary - # Remove HTML tags - summary = BeautifulSoup(summary, "lxml").text - # TODO Limit text length - summary = summary.replace("\n\n\n", "\n\n") - summary = summary[:300] + " ‍⃨" - summary = summary.strip().split('\n') - summary = ["> " + line for line in summary] - summary = "\n".join(summary) + # # Remove HTML tags + # summary = BeautifulSoup(summary, "lxml").text + # # TODO Limit text length + # summary = summary.replace("\n\n\n", "\n\n") + # summary = summary[:300] + " […]‍⃨" + # summary = summary.strip().split('\n') + # summary = ["> " + line for line in summary] + # summary = "\n".join(summary) else: summary = "> *** No summary ***" read_status = 0 @@ -200,6 +201,12 @@ async def download_updates(db_file, url=None): date, read_status ) + if isinstance(date, int): + print("date is int") + print(date) + breakpoint() + print(source) + print(date) await sqlitehandler.add_entry_and_set_date( db_file, source, @@ -363,7 +370,7 @@ async def view_entry(url, num): msg = ( "{}\n" "\n" - "{}\n" + "> {}\n" "\n" "{}\n" "\n" diff --git a/slixfeed/main.py.bak b/slixfeed/main.py.bak new file mode 100644 index 0000000..c50b185 --- /dev/null +++ b/slixfeed/main.py.bak @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" + +FIXME + +1) Check feed duplication on runtime. + When feed is valid and is not yet in the database it is + posible to send a batch which would result in duplication. + Consequently, it might result in database lock error upon + feed removal attempt + +TODO + +1) SQL prepared statements. + +2) Machine Learning for scrapping Title, Link, Summary and Timstamp. + +3) Support MUC. + +4) Support categories. + +5) Default prepackaged list of feeds. + +6) XMPP commands. + +7) Bot as transport. + +8) OMEMO. + +9) Logging. + +10) Default feeds (e.g. Blacklisted News, TBOT etc.) + +11) Download and upload/send article (xHTML, xHTMLZ, Markdown, MHTML, TXT). + Use Readability. + +12) Fetch summary from URL, instead of storing summary. + +13) Support protocol Gopher + https://github.com/michael-lazar/pygopherd + https://github.com/gopherball/gb + +13) Support ActivityPub @person@domain (see Tip Of The Day). + +12) Tip Of The Day. + Did you know that you can follow you favorite Mastodon feeds by just + sending the URL address? + Supported fediverse websites are: + Akkoma, HubZilla, Mastodon, Misskey, Pixelfed, Pleroma, Soapbox. + +""" + +# vars and their meanings: +# jid = Jabber ID (XMPP) +# res = response (HTTP) + +from argparse import ArgumentParser +from getpass import getpass +import logging + +from datetime import date +import time + +# from eliot import start_action, to_file +# # to_file(open("slixfeed.log", "w")) +# # with start_action(action_type="set_date()", jid=jid): +# # with start_action(action_type="message()", msg=msg): + +#import irchandler +import xmpphandler +#import matrixhandler + + +if __name__ == '__main__': + # Setup the command line arguments. + parser = ArgumentParser(description=xmpphandler.Slixfeed.__doc__) + + # Output verbosity options. + parser.add_argument( + "-q", "--quiet", help="set logging to ERROR", + action="store_const", dest="loglevel", + const=logging.ERROR, default=logging.INFO + ) + parser.add_argument( + "-d", "--debug", help="set logging to DEBUG", + action="store_const", dest="loglevel", + const=logging.DEBUG, default=logging.INFO + ) + + # JID and password options. + parser.add_argument("-j", "--jid", dest="jid", + help="JID to use") + parser.add_argument("-p", "--password", dest="password", + help="password to use") + + args = parser.parse_args() + + # Setup logging. + logging.basicConfig(level=args.loglevel, + format='%(levelname)-8s %(message)s') + + if args.jid is None: + args.jid = input("Username: ") + if args.password is None: + args.password = getpass("Password: ") + + # Setup the Slixfeed and register plugins. Note that while plugins may + # have interdependencies, the order in which you register them does + # not matter. + xmpp = xmpphandler.Slixfeed(args.jid, args.password) + xmpp.register_plugin('xep_0004') # Data Forms + xmpp.register_plugin('xep_0030') # Service Discovery + xmpp.register_plugin('xep_0045') # Multi-User Chat + xmpp.register_plugin('xep_0048') # Bookmarks + xmpp.register_plugin('xep_0060') # PubSub + xmpp.register_plugin('xep_0199', {'keepalive': True, 'frequency': 15}) # XMPP Ping + xmpp.register_plugin('xep_0249') # Multi-User Chat + xmpp.register_plugin('xep_0402') # PEP Native Bookmarks + + # Connect to the XMPP server and start processing XMPP stanzas. + xmpp.connect() + xmpp.process() diff --git a/slixfeed/sqlitehandler.py b/slixfeed/sqlitehandler.py index 0c0455a..1148e6d 100644 --- a/slixfeed/sqlitehandler.py +++ b/slixfeed/sqlitehandler.py @@ -18,8 +18,9 @@ TODO import sqlite3 import asyncio -from sqlite3 import Error +from bs4 import BeautifulSoup from datetime import date +from sqlite3 import Error import confighandler import datahandler @@ -426,11 +427,11 @@ async def get_entry_unread(db_file, num=None): # "DESC LIMIT :num" # ) sql = ( - "SELECT id, title, summary, link, timestamp " + "SELECT id, title, summary, link, source, timestamp " "FROM entries " "WHERE read = 0 " "UNION ALL " - "SELECT id, title, summary, link, timestamp " + "SELECT id, title, summary, link, source, timestamp " "FROM archive " "ORDER BY timestamp " "DESC LIMIT :num" @@ -462,7 +463,24 @@ async def get_entry_unread(db_file, num=None): ix = result[0] title = result[1] summary = result[2] + # Remove HTML tags + summary = BeautifulSoup(summary, "lxml").text + # TODO Limit text length + summary = summary.replace("\n\n\n", "\n\n") + length = await get_settings_value(db_file, "length") + summary = summary[:length] + " […]" + summary = summary.strip().split('\n') + summary = ["> " + line for line in summary] + summary = "\n".join(summary) link = result[3] + sql = ( + "SELECT name " + "FROM feeds " + "WHERE address = :source " + ) + source = result[4] + feed = cur.execute(sql, (source,)) + feed = feed.fetchone()[0] if num > 1: news_list += ( "\n{}\n{}\n" @@ -472,11 +490,12 @@ async def get_entry_unread(db_file, num=None): ) else: news_list = ( - "{}\n\n{}\n\n{}" + "{}\n\n{}\n\n{}\n{}" ).format( str(title), str(summary), - str(link) + str(link), + str(feed) ) # TODO While `async with DBLOCK` does work well from # outside of functions, it would be better practice @@ -827,7 +846,12 @@ async def update_source_validity(db_file, source, valid): "url": source }) +""" +TODO +Investigate why causes entry[6] (date) to be int 0 + +""" async def add_entry(cur, entry): """ Add a new entry row into the entries table. @@ -852,7 +876,14 @@ async def add_entry(cur, entry): ") " "VALUES(?, ?, ?, ?, ?, ?, ?)" ) - cur.execute(sql, entry) + try: + cur.execute(sql, entry) + except: + print(entry[6]) + print(type(entry[6])) + print(entry) + print(type(entry)) + breakpoint() # NOTE See remove_nonexistent_entries @@ -1294,10 +1325,26 @@ async def search_entries(db_file, query): return "No results were found for: {}".format(query) """ -FIXME Error due to missing date, but it appears that date is present: +FIXME + +Error due to missing date, but it appears that date is present: ERROR DATE: source = https://blog.heckel.io/feed/ ERROR DATE: date = 2008-05-13T13:51:50+00:00 ERROR DATE: result = https://blog.heckel.io/feed/ + +19:32:05 ERROR DATE: source = https://mwl.io/feed +19:32:05 ERROR DATE: date = 2023-11-30T10:56:39+00:00 +19:32:05 ERROR DATE: result = https://mwl.io/feed +19:32:05 ERROR DATE: source = https://mwl.io/feed +19:32:05 ERROR DATE: date = 2023-11-22T16:59:08+00:00 +19:32:05 ERROR DATE: result = https://mwl.io/feed +19:32:06 ERROR DATE: source = https://mwl.io/feed +19:32:06 ERROR DATE: date = 2023-11-16T10:33:57+00:00 +19:32:06 ERROR DATE: result = https://mwl.io/feed +19:32:06 ERROR DATE: source = https://mwl.io/feed +19:32:06 ERROR DATE: date = 2023-11-09T07:37:57+00:00 +19:32:06 ERROR DATE: result = https://mwl.io/feed + """ async def check_entry_exist(db_file, source, eid=None, title=None, link=None, date=None): diff --git a/slixfeed/taskhandler.py b/slixfeed/taskhandler.py index 3c3f70f..35786b8 100644 --- a/slixfeed/taskhandler.py +++ b/slixfeed/taskhandler.py @@ -272,7 +272,7 @@ async def send_status(self, jid): print(">>> not feeds:", feeds, "jid:", jid) status_mode = "available" status_text = ( - "📭️ Send a URL from a blog or a news website" + "📪️ Send a URL from a blog or a news website" ) else: unread = await filehandler.initdb( @@ -292,7 +292,7 @@ async def send_status(self, jid): # ).format(str(unread)) else: status_mode = "available" - status_text = "📪️ No news" + status_text = "📭️ No news" # breakpoint() # print(await datetimehandler.current_time(), status_text, "for", jid) diff --git a/slixfeed/xmpphandler.py b/slixfeed/xmpphandler.py index c9722ef..fbfd7be 100644 --- a/slixfeed/xmpphandler.py +++ b/slixfeed/xmpphandler.py @@ -59,6 +59,8 @@ from slixmpp.plugins.xep_0363.http_upload import FileTooBig, HTTPError, UploadSe # from slixmpp.plugins.xep_0402 import BookmarkStorage, Conference from slixmpp.plugins.xep_0048.stanza import Bookmarks +# TODO datahandler -> slixfeed.datahandler +# See project feed2toot import datahandler import datetimehandler import filehandler @@ -796,11 +798,24 @@ class Slixfeed(slixmpp.ClientXMPP): url = message if url.startswith("feed:"): url = await datahandler.feed_to_http(url) + await taskhandler.clean_tasks_xmpp( + jid, + ["status"] + ) + task = ( + "📫️ Processing request to fetch data from {}" + ).format(url) + process_task_message(self, jid, task) action = await filehandler.initdb( jid, datahandler.add_feed, url ) + await taskhandler.start_tasks_xmpp( + self, + jid, + ["status"] + ) # action = "> " + message + "\n" + action # FIXME Make the taskhandler to update status message # await taskhandler.refresh_task( @@ -890,6 +905,25 @@ class Slixfeed(slixmpp.ClientXMPP): case _ if message_lowercase.startswith("join"): muc = message[5:] await self.join_muc(jid, muc) + case _ if message_lowercase.startswith("length"): + key = message[:6] + val = message[7:] + if val: + await filehandler.initdb( + jid, + sqlitehandler.set_settings_value, + [key, val] + ) + if val == 0: + action = ( + "Summary length limit is disabled." + ) + else: + action = ( + "Summary maximum length is set to {} characters." + ).format(val) + else: + action = "Missing value." case _ if message_lowercase.startswith("mastership"): key = message[:7] val = message[11:] @@ -981,6 +1015,14 @@ class Slixfeed(slixmpp.ClientXMPP): data = message[5:] data = data.split() url = data[0] + task = ( + "📫️ Processing request to fetch data from {}" + ).format(url) + process_task_message(self, jid, task) + await taskhandler.clean_tasks_xmpp( + jid, + ["status"] + ) if url.startswith("feed:"): url = await datahandler.feed_to_http(url) match len(data): @@ -1001,6 +1043,11 @@ class Slixfeed(slixmpp.ClientXMPP): "`read URL` or `read URL NUMBER`\n" "URL must not contain white space." ) + await taskhandler.start_tasks_xmpp( + self, + jid, + ["status"] + ) case _ if message_lowercase.startswith("recent"): num = message[7:] if num: @@ -1111,10 +1158,13 @@ class Slixfeed(slixmpp.ClientXMPP): sqlitehandler.set_settings_value, [key, val] ) - await taskhandler.clean_tasks_xmpp(jid, ["interval"]) + await taskhandler.clean_tasks_xmpp( + jid, + ["interval", "status"] + ) self.send_presence( pshow="xa", - pstatus="Send \"Start\" to receive news.", + pstatus="💡️ Send \"Start\" to receive Jabber news", pto=jid, ) action = "Updates are disabled." @@ -1132,6 +1182,14 @@ class Slixfeed(slixmpp.ClientXMPP): if action: msg.reply(action).send() +def process_task_message(self, jid, task): + self.send_presence( + pshow="dnd", + pstatus=task, + pto=jid, + ) + + def print_info(): """ Print information. @@ -1252,22 +1310,24 @@ def print_help(): " Display most recent 20 titles of given URL.\n" " read URL N\n" " Display specified entry number from given URL.\n" - " new\n" - " Send only new items of added feeds.\n" - " old\n" - " Send all items of added feeds.\n" "\n" "MESSAGE OPTIONS\n" + " interval N\n" + " Set interval update to every N minutes.\n" + " length\n" + " Set maximum length of news item description. (0 for no limit)\n" + " new\n" + " Send only new items of added feeds.\n" + " next N\n" + " Send N next updates.\n" + " old\n" + " Send all items of added feeds.\n" + " quantum N\n" + " Set N amount of updates per interval.\n" " start\n" " Enable bot and send updates.\n" " stop\n" " Disable bot and stop updates.\n" - " interval N\n" - " Set interval update to every N minutes.\n" - " next N\n" - " Send N next updates.\n" - " quantum N\n" - " Set N amount of updates per interval.\n" "\n" "GROUPCHAT OPTIONS\n" " ! (command initiation)\n"