sch/Slixfeed
sch/Slixfeed
3
1
Fork 1
Code Issues 15 Pull requests Projects Releases Packages Wiki Activity Actions

Improve redirection mechanism

Browse source
This commit is contained in:
Schimon Jehudah 2023-12-29 01:35:20 +00:00
parent 76b2713223
commit 8e3e06b36b
5 changed files with 290 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

284
assets/proxies.yaml
View file

@ -1,7 +1,48 @@
invidious: anonymousoverflow:
clearnet:
- https://ao.phreedom.club
- https://overflow.hostux.net
- https://ao.foss.wtf
- https://overflow.adminforge.de
- https://overflow.lunar.icu
- https://anonymousoverflow.esmailelbob.xyz
- https://overflow.smnz.de
- https://ao.vern.cc
- https://overflow.777.tf
- https://code.whatever.social
- https://stackoverflow.vern.cc
- https://anonymousoverflow.vern.cc
- https://ao.bloatcat.tk
- https://se.chaotic.ninja
- https://anonymousoverflow.privacyfucking.rocks
- https://overflow.projectsegfau.lt
- https://anonoverflow.frontendfriendly.xyz
- https://overflow.fascinated.cc
i2p:
- http://vernmzgraj6aaoafmehupvtkkynpaa67rxcdj2kinwiy6konn6rq.b32.i2p
loki: []
tor:
- http://anonymousoverflow.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion
- http://ao.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion
yggdrasil:
- http://[301:f69c:2017:b6b8::8]
hostname: hostname:
- youtu.be - stackoverflow.com
- youtube.com type:
- link
dumb:
clearnet:
- https://dumb.privacydev.net
- https://dm.vern.cc
- https://dumb.lunar.icu
- https://dumb.esmailelbob.xyz
hostname:
- genius.com
type:
- link
invidious:
clearnet: clearnet:
- https://incogtube.com - https://incogtube.com
- https://vid.puffyan.us - https://vid.puffyan.us
@ -108,10 +149,14 @@ invidious:
- http://w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion - http://w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd.onion
yggdrasil: yggdrasil:
- http://[200:168a:c80a:b258:1dfe:f920:4414:6897] - http://[200:168a:c80a:b258:1dfe:f920:4414:6897]
hostname:
- youtu.be
- youtube.com
type:
- feed
- link
librarian: librarian:
hostname:
- odysee.com
clearnet: clearnet:
- https://librarian.pussthecat.org - https://librarian.pussthecat.org
- https://odysee.076.ne.jp - https://odysee.076.ne.jp
@ -132,11 +177,124 @@ librarian:
- http://5znbzx2xcymhddzekfjib3isgqq4ilcyxa2bsq6vqmnvbtgu4f776lqd.onion - http://5znbzx2xcymhddzekfjib3isgqq4ilcyxa2bsq6vqmnvbtgu4f776lqd.onion
- http://bxewpsswttslepw27w2hhxhlizwm7l7y54x3jw5cfrb64hb6lgc557ad.onion - http://bxewpsswttslepw27w2hhxhlizwm7l7y54x3jw5cfrb64hb6lgc557ad.onion
yggdrasil: [] yggdrasil: []
hostname:
- odysee.com
type:
- feed
- link
libreddit:
clearnet:
- https://libreddit.spike.codes
- https://libreddit.hu
- https://libreddit.nl
- https://libreddit.bus-hit.me
- https://libreddit.strongthany.cc
- https://libreddit.esmailelbob.xyz
- https://lr.riverside.rocks
- https://libreddit.40two.app
- https://libreddit.albony.xyz
- https://libreddit.domain.glass
- https://discuss.whatever.social
- https://libreddit.kavin.rocks
- https://libreddit.privacy.com.de
- https://libreddit.eu.org
- https://libreddit.bloatcat.tk
- https://libreddit.pabloferreiro.es
- https://lr.foss.wtf
- https://libreddit.no-logs.com
- https://lr.slipfox.xyz
- https://lr.creller.net
- https://libreddit.dcs0.hu
- https://l.opnxng.com
- https://libreddit.tux.pizza
- https://reddit.leptons.xyz
- https://reddit.baby
- https://snoo.habedieeh.re
- https://lr.4201337.xyz
- https://libreddit.private.coffee
- https://lr.artemislena.eu
- https://libreddit.privacyfucking.rocks
- https://libreddit.qwik.space
- https://farside.link/libreddit
- https://de.leddit.xyz
- https://leddit.xyz
- https://libreddit.alefvanoon.xyz
- https://libreddit.autarkic.org
- https://libreddit.awesomehub.io
- https://libreddit.crewz.me
- https://libreddit.database.red
- https://libreddit.datatunnel.xyz
- https://libreddit.de
- https://libreddit.dothq.co
- https://libreddit.drivet.xyz
- https://libreddit.flux.industries
- https://libreddit.igna.rocks
- https://libredd.it
- https://libreddit.jamiethalacker.dev
- https://libreddit.kylrth.com
- https://libreddit.lunar.icu
- https://libreddit.mutahar.rocks
- https://libreddit.northboot.xyz
- https://libreddit.pussthecat.org
- https://libreddit.silkky.cloud
- https://libreddit.some-things.org
- https://libreddit.sugoma.tk
- https://libreddit.tiekoetter.com
- https://libreddit.totaldarkness.net
- https://libreddit.winscloud.net
- https://libreddit.yonalee.eu
- https://lr.cowfee.moe
- https://lr.mint.lgbt
- https://lr.oversold.host
- https://lr.stilic.ml
- https://r.nf
- https://r.walkx.org
- https://reddi.tk
- https://reddit.artemislena.eu
- https://reddit.invak.id
- https://reddit.phii.me
- https://reddit.rtrace.io
- https://reddit.stuehieyr.com
- https://safereddit.com
- https://libreddit.nohost.network
- https://libreddit.projectsegfau.lt
- https://reddit.simo.sh
i2p:
- http://woo5ugmoomzbtaq6z46q4wgei5mqmc6jkafqfi5c37zni7xc4ymq.b32.i2p
loki: []
tor:
- http://spjmllawtheisznfs7uryhxumin26ssv2draj7oope3ok3wuhy43eoyd.onion
- http://qwikxxeiw4kgmml6vjw2bsxtviuwjce735dunai2djhu6q7qbacq73id.onion
- http://qwikxx4xqvhdyyazkrw7pwdpdppfnmn7j2n6cvq5zecm4atbppaslzad.onion
- http://ecue64ybzvn6vjzl37kcsnwt4ycmbsyf74nbttyg7rkc3t3qwnj7mcyd.onion
- http://fwhhsbrbltmrct5hshrnqlqygqvcgmnek3cnka55zj4y7nuus5muwyyd.onion
- http://inytumdgnri7xsqtvpntjevaelxtgbjqkuqhtf6txxhwbll2fwqtakqd.onion
- http://kphht2jcflojtqte4b4kyx7p2ahagv4debjj32nre67dxz7y57seqwyd.onion
- http://kzhfp3nvb4qp575vy23ccbrgfocezjtl5dx66uthgrhu7nscu6rcwjyd.onion
- http://lbrdtjaj7567ptdd4rv74lv27qhxfkraabnyphgcvptl64ijx2tijwid.onion
- http://libreddit.2syis2nnyytz6jnusnjurva4swlaizlnleiks5mjp46phuwjbdjqwgqd.onion
- http://ledditqo2mxfvlgobxnlhrkq4dh34jss6evfkdkb2thlvy6dn4f4gpyd.onion
- http://libreddit.lqs5fjmajyp7rvp4qvyubwofzi6d4imua7vs237rkc4m5qogitqwrgyd.onion
- http://libredoxhxwnmsb6dvzzd35hmgzmawsq5i764es7witwhddvpc2razid.onion
- http://ol5begilptoou34emq2sshf3may3hlblvipdjtybbovpb7c7zodxmtqd.onion
- http://liredejj74h5xjqr2dylnl5howb2bpikfowqoveub55ru27x43357iid.onion
yggdrasil: []
hostname:
- reddit.com
type:
- link
neuters:
clearnet:
- https://neuters.de
- https://neuters.privacyfucking.rocks
hostname:
- reuters.com
type:
- link
nitter: nitter:
hostname:
- twitter.com
- x.com
clearnet: clearnet:
- https://nitter.hu - https://nitter.hu
- https://nitter.actionsack.com - https://nitter.actionsack.com
@ -207,3 +365,113 @@ nitter:
- http://qwikxx2erhx6qrymued6ox2qkf2yeogjwypqvzoif4fqkljixasr6oid.onion - http://qwikxx2erhx6qrymued6ox2qkf2yeogjwypqvzoif4fqkljixasr6oid.onion
- http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion - http://n.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion
yggdrasil: [] yggdrasil: []
hostname:
- twitter.com
- x.com
type:
- feed
- link
proxitok:
clearnet:
- https://proxitok.lunar.icu
- https://tik.hostux.net
- https://proxitok.pabloferreiro.es
- https://proxitok.privacy.com.de
- https://tok.adminforge.de
- https://tok.habedieeh.re
- https://proxitok.pussthecat.org
- https://proxitok.privacyfucking.rocks
- https://cringe.whatever.social
- https://proxitok.esmailelbob.xyz
- https://proxitok.privacydev.net
- https://proxitok.pufe.org
- https://tok.artemislena.eu
- https://tok.thekitty.zone
- https://tiktok.chauvet.pro
- https://tt.vern.cc
- https://farside.link/proxitok
i2p:
- http://qr.vern.i2p
loki: []
tor: []
yggdrasil: []
hostname:
- tiktok.com
type:
- link
quetre:
clearnet:
- https://quetre.privacydev.net
- https://quetre.pufe.org
- https://que.wilbvr.me
- https://quetre.iket.me
- https://quetre.pussthecat.org
- https://quetre.tokhmi.xyz
- https://quetre.projectsegfau.lt
- https://quetre.esmailelbob.xyz
- https://quetre.odyssey346.dev
- ://ask.habedieeh.re
- https://quetre.marcopisco.com
- https://quetre.blackdrgn.nl
- https://quetre.lunar.icu
- https://quora.femboy.hu
- https://quora.vern.cc
- https://farside.link/quetre
- https://quetre.fascinated.cc
i2p: []
loki: []
tor:
- http://ask.habeehrhadazsw3izbrbilqajalfyqqln54mrja3iwpqxgcuxnus7eid.onion
- http://qr.vernccvbvyi5qhfzyqengccj7lkove6bjot2xhh5kajhwvidqafczrad.onion
- http://quetre.esmail5pdn24shtvieloeedh7ehz3nrwcdivnfhfcedl7gf4kwddhkqd.onion
- http://quetre.g4c3eya4clenolymqbpgwz3q3tawoxw56yhzk4vugqrl6dtu3ejvhjid.onion
- http://quora.cepyxplublbyw2f4axy4pyztfbxmf63lrt2c7uwv6wl4iixz53czload.onion
yggdrasil: []
hostname:
- quora.com
type:
- link
teddit:
clearnet:
- https://teddit.pussthecat.org
- https://teddit.zaggy.nl
- https://teddit.bus-hit.me
- https://teddit.adminforge.de
- https://incogsnoo.com
- https://teddit.hostux.net
- https://teddit.ggc-project.de
- https://teddit.httpjames.space
- https://snoo.ioens.is
- https://teddit.no-logs.com
- https://teddit.net
- https://i.opnxng.com
- https://tedd.it
- https://teddit.projectsegfau.lt
- https://reddit.lol
- https://rdt.trom.tf
- https://t.sneed.network
- https://farside.link/teddit
- https://teddit.alefvanoon.xyz
- https://teddit.domain.glass
- https://teddit.froth.zone
- https://teddit.namazso.eu
- https://teddit.sethforprivacy.com
- https://teddit.tinfoil-hat.net
- https://teddit.totaldarkness.net
- https://td.vern.cc
i2p:
- http://k62ptris7p72aborr4zoanee7xai6wguucveptwgxs5vbgt7qzpq.b32.i2p
- http://teddit.i2p
loki: []
tor:
- http://t.sneed4fmhevap3ci4xhf4wgkf72lwk275lcgomnfgwniwmqvaxyluuid.onion
- http://tedditfyn6idalzso5wam5qd3kdtxoljjhbrbbx34q2xkcisvshuytad.onion
yggdrasil:
- http://[200:5e4b:515c:e42b:3e73:6fbf:2f11:779d]
hostname:
- reddit.com
type:
- link

2
slixfeed/sqlite.py
View file

@ -492,7 +492,7 @@ async def get_entry_unread(db_file, num=None):
# summary = "\n".join(summary) # summary = "\n".join(summary)
link = result[2] link = result[2]
link = await remove_tracking_parameters(link) link = await remove_tracking_parameters(link)
link = (await replace_hostname(link)) or link link = (await replace_hostname(link, "link")) or link
sql = ( sql = (
"SELECT name " "SELECT name "
"FROM feeds " "FROM feeds "

6
slixfeed/url.py
View file

@ -31,7 +31,7 @@ from urllib.parse import (
# proxies.yaml. Perhaps a better practice would be to have # proxies.yaml. Perhaps a better practice would be to have
# them separated. File proxies.yaml will remainas is in order # them separated. File proxies.yaml will remainas is in order
# to be coordinated with the dataset of project LibRedirect. # to be coordinated with the dataset of project LibRedirect.
async def replace_hostname(url): async def replace_hostname(url, url_type):
""" """
Replace hostname. Replace hostname.
@ -39,6 +39,8 @@ async def replace_hostname(url):
---------- ----------
url : str url : str
URL. URL.
url_type : str
"feed" or "link".
Returns Returns
------- -------
@ -55,7 +57,7 @@ async def replace_hostname(url):
proxies = await config.get_list("proxies.yaml") proxies = await config.get_list("proxies.yaml")
for proxy in proxies: for proxy in proxies:
proxy = proxies[proxy] proxy = proxies[proxy]
if hostname in proxy["hostname"]: if hostname in proxy["hostname"] and url_type in proxy["type"]:
select_proxy = random.choice(proxy["clearnet"]) select_proxy = random.choice(proxy["clearnet"])
parted_proxy = urlsplit(select_proxy) parted_proxy = urlsplit(select_proxy)
protocol_new = parted_proxy.scheme protocol_new = parted_proxy.scheme

1
slixfeed/xmpp/client.py
View file

@ -492,7 +492,6 @@ class Slixfeed(slixmpp.ClientXMPP):
# print("message") # print("message")
# print(message) # print(message)
if message["type"] in ("chat", "groupchat", "normal"): if message["type"] in ("chat", "groupchat", "normal"):
action = 0
jid = message["from"].bare jid = message["from"].bare
if message["type"] == "groupchat": if message["type"] == "groupchat":
# nick = message["from"][message["from"].index("/")+1:] # nick = message["from"][message["from"].index("/")+1:]

17
slixfeed/xmpp/compose.py
View file

@ -16,11 +16,12 @@ from slixfeed.datetime import current_time
import slixfeed.fetch as fetcher import slixfeed.fetch as fetcher
import slixfeed.sqlite as sqlite import slixfeed.sqlite as sqlite
import slixfeed.task as task import slixfeed.task as task
import slixfeed.url as urlfixer import slixfeed.url as uri
import slixfeed.xmpp.status as status import slixfeed.xmpp.status as status
import slixfeed.xmpp.text as text import slixfeed.xmpp.text as text
async def message(self, jid, message): async def message(self, jid, message):
action = None
message_text = " ".join(message["body"].split()) message_text = " ".join(message["body"].split())
if message["type"] == "groupchat": if message["type"] == "groupchat":
message_text = message_text[1:] message_text = message_text[1:]
@ -244,11 +245,11 @@ async def message(self, jid, message):
).format(url) ).format(url)
status.process_task_message(self, jid, status_message) status.process_task_message(self, jid, status_message)
if url.startswith("feed:"): if url.startswith("feed:"):
url = urlfixer.feed_to_http(url) url = uri.feed_to_http(url)
# url_alt = await urlfixer.replace_hostname(url) # url_alt = await uri.replace_hostname(url, "feed")
# if url_alt: # if url_alt:
# url = url_alt # url = url_alt
url = (await urlfixer.replace_hostname(url)) or url url = (await uri.replace_hostname(url, "feed")) or url
action = await initdb( action = await initdb(
jid, jid,
fetcher.add_feed, fetcher.add_feed,
@ -346,7 +347,7 @@ async def message(self, jid, message):
else: else:
action = "Missing value." action = "Missing value."
case _ if message_lowercase.startswith("join"): case _ if message_lowercase.startswith("join"):
muc = urlfixer.check_xmpp_uri(message_text[5:]) muc = uri.check_xmpp_uri(message_text[5:])
if muc: if muc:
"TODO probe JID and confirm it's a groupchat" "TODO probe JID and confirm it's a groupchat"
await self.join_muc(jid, muc) await self.join_muc(jid, muc)
@ -488,8 +489,8 @@ async def message(self, jid, message):
).format(url) ).format(url)
status.process_task_message(self, jid, status_message) status.process_task_message(self, jid, status_message)
if url.startswith("feed:"): if url.startswith("feed:"):
url = urlfixer.feed_to_http(url) url = uri.feed_to_http(url)
url = (await urlfixer.replace_hostname(url)) or url url = (await uri.replace_hostname(url, "feed")) or url
match len(data): match len(data):
case 1: case 1:
if url.startswith("http"): if url.startswith("http"):
@ -676,7 +677,7 @@ async def message(self, jid, message):
# TODO Send an invitation. # TODO Send an invitation.
action = "Join xmpp:slixfeed@chat.woodpeckersnest.space?join" action = "Join xmpp:slixfeed@chat.woodpeckersnest.space?join"
case _ if message_lowercase.startswith("xmpp:"): case _ if message_lowercase.startswith("xmpp:"):
muc = urlfixer.check_xmpp_uri(message_text) muc = uri.check_xmpp_uri(message_text)
if muc: if muc:
"TODO probe JID and confirm it's a groupchat" "TODO probe JID and confirm it's a groupchat"
await self.join_muc(jid, muc) await self.join_muc(jid, muc)