forked from sch/Slixfeed
Only operate on sources of http code 200
What potentially could happen, is removing all news items of a 403 feed, which would cause in resending all of its news items, including those that were already seen by user.
This commit is contained in:
Schimon Jehudah
parent
52fad12660
commit
10735697ce
1 changed files with 52 additions and 30 deletions
80
slixfeed.py
80
slixfeed.py
|
|
@ -1,5 +1,11 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*- coding: utf-8 -*-
|
# -*- coding: utf-8 -*-
|
||||||
|
|
||||||
|
# vars and their meanings:
|
||||||
|
# cur = Cursor (SQL)
|
||||||
|
# jid = Jabber ID (XMPP)
|
||||||
|
# res = response (HTTP)
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from argparse import ArgumentParser
|
from argparse import ArgumentParser
|
||||||
from asyncio.exceptions import IncompleteReadError
|
from asyncio.exceptions import IncompleteReadError
|
||||||
|
|
@ -167,7 +173,7 @@ def print_help():
|
||||||
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
|
msg = ("Slixfeed - News syndication bot for Jabber/XMPP \n"
|
||||||
"\n"
|
"\n"
|
||||||
"DESCRIPTION: \n"
|
"DESCRIPTION: \n"
|
||||||
" Slixfeed is an aggregator bot for online news feeds. \n"
|
" Slixfeed is a news aggregator bot for online news feeds. \n"
|
||||||
"\n"
|
"\n"
|
||||||
"BASIC USAGE: \n"
|
"BASIC USAGE: \n"
|
||||||
" enable \n"
|
" enable \n"
|
||||||
|
|
@ -179,7 +185,7 @@ def print_help():
|
||||||
"\n"
|
"\n"
|
||||||
"EDIT OPTIONS: \n"
|
"EDIT OPTIONS: \n"
|
||||||
" feed add URL \n"
|
" feed add URL \n"
|
||||||
" Add URL to the subscriptions list. \n"
|
" Add URL to subscription list. \n"
|
||||||
" feed remove ID \n"
|
" feed remove ID \n"
|
||||||
" Remove feed from subscription list. \n"
|
" Remove feed from subscription list. \n"
|
||||||
" feed status ID \n"
|
" feed status ID \n"
|
||||||
|
|
@ -191,6 +197,16 @@ def print_help():
|
||||||
" feed recent N \n"
|
" feed recent N \n"
|
||||||
" List recent N news items (up to 50 items). \n"
|
" List recent N news items (up to 50 items). \n"
|
||||||
"\n"
|
"\n"
|
||||||
|
"BACKUP OPTIONS: \n"
|
||||||
|
" export opml \n"
|
||||||
|
" Send an OPML file with your feeds. \n"
|
||||||
|
" backup news html\n"
|
||||||
|
" Send an HTML formatted file of your news items. \n"
|
||||||
|
" backup news md \n"
|
||||||
|
" Send a Markdown file of your news items. \n"
|
||||||
|
" backup news text \n"
|
||||||
|
" Send a Plain Text file of your news items. \n"
|
||||||
|
"\n"
|
||||||
"DOCUMENTATION: \n"
|
"DOCUMENTATION: \n"
|
||||||
" Slixfeed \n"
|
" Slixfeed \n"
|
||||||
" https://gitgud.io/sjehuda/slixfeed \n"
|
" https://gitgud.io/sjehuda/slixfeed \n"
|
||||||
|
|
@ -328,36 +344,42 @@ async def download_updates(db_file):
|
||||||
return
|
return
|
||||||
# TODO Place these couple of lines back down
|
# TODO Place these couple of lines back down
|
||||||
# NOTE Need to correct the SQL statement to do so
|
# NOTE Need to correct the SQL statement to do so
|
||||||
entries = feed.entries
|
# NOT SURE WHETHER I MEANT THE LINES ABOVE OR BELOW
|
||||||
length = len(entries)
|
|
||||||
async with DBLOCK:
|
|
||||||
with conn:
|
|
||||||
await remove_entry(conn, source, length)
|
|
||||||
|
|
||||||
for entry in entries:
|
if res[1] == 200:
|
||||||
if entry.has_key("title"):
|
# NOT SURE WHETHER I MEANT THE LINES ABOVE OR BELOW
|
||||||
title = entry.title
|
# TODO Place these couple of lines back down
|
||||||
else:
|
# NOTE Need to correct the SQL statement to do so
|
||||||
title = feed["feed"]["title"]
|
entries = feed.entries
|
||||||
link = source if not entry.link else entry.link
|
length = len(entries)
|
||||||
with conn:
|
async with DBLOCK:
|
||||||
exist = await check_entry(conn, title, link)
|
with conn:
|
||||||
|
await remove_entry(conn, source, length)
|
||||||
|
|
||||||
if not exist:
|
for entry in entries:
|
||||||
if entry.has_key("summary"):
|
if entry.has_key("title"):
|
||||||
summary = entry.summary
|
title = entry.title
|
||||||
# Remove HTML tags
|
|
||||||
summary = BeautifulSoup(summary, "lxml").text
|
|
||||||
# TODO Limit text length
|
|
||||||
summary = summary.replace("\n\n", "\n")[:300] + " ⃨"
|
|
||||||
else:
|
else:
|
||||||
summary = '*** No summary ***'
|
title = feed["feed"]["title"]
|
||||||
#print('~~~~~~summary not in entry')
|
link = source if not entry.link else entry.link
|
||||||
entry = (title, summary, link, source, 0);
|
with conn:
|
||||||
async with DBLOCK:
|
exist = await check_entry(conn, title, link)
|
||||||
with conn:
|
|
||||||
await add_entry(conn, entry)
|
if not exist:
|
||||||
await set_date(conn, source)
|
if entry.has_key("summary"):
|
||||||
|
summary = entry.summary
|
||||||
|
# Remove HTML tags
|
||||||
|
summary = BeautifulSoup(summary, "lxml").text
|
||||||
|
# TODO Limit text length
|
||||||
|
summary = summary.replace("\n\n", "\n")[:300] + " ⃨"
|
||||||
|
else:
|
||||||
|
summary = '*** No summary ***'
|
||||||
|
#print('~~~~~~summary not in entry')
|
||||||
|
entry = (title, summary, link, source, 0);
|
||||||
|
async with DBLOCK:
|
||||||
|
with conn:
|
||||||
|
await add_entry(conn, entry)
|
||||||
|
await set_date(conn, source)
|
||||||
|
|
||||||
|
|
||||||
async def download_feed(url):
|
async def download_feed(url):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue