0x80
/
rsstube
1
1
Fork 0
Code Issues 6 Pull Requests Projects Releases Wiki Activity

Add output options. 

User can now output to file and/or output in OPML format.
This commit is contained in:
0x80 2021-12-29 00:00:00 +00:00
parent 334f7dbe49
commit cfebf73510
Signed by: 0x80
GPG Key ID: 68368BCBC000EF51
5 changed files with 151 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scripts/opml.py Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/python3
# class for representing and writing an OPML outline for RSS feeds
class Opml:
def __init__(self, title=""):
self.title = title
self.feeds = []
@staticmethod
def get_date():
from datetime import datetime
return datetime.now().strftime("%a %b %d %H:%M:%S %Y")
def add_feed(self, feed, title="", html_url=""):
line = '<outline text="' + title + '" type="rss" htmlUrl="' + html_url + '" xmlUrl="' + feed + '"/>'
self.feeds.append(line)
def get_opml(self):
opml = '''<?xml version="1.0" encoding="UTF-8"?>
<opml version="2.0">
<head>
<title>''' + self.title + '''</title>
<dateModified>''' + self.get_date() + '''</dateModified>
</head>
<body>'''
for line in self.feeds:
opml += "\n\t\t" + line
opml += '''
</body>
</opml>'''
return opml

37
scripts/options.py
View File

@ -47,6 +47,8 @@ def options(params):
# general settings
network = True
output_format = ""
output_filename = None
## verbosity: in addition to the feed, print...
## 0: no messages (suppress errors)
@ -64,7 +66,7 @@ def options(params):
arg_count = 0
try:
opts, args = getopt.getopt(params,"A:c:H:hnp:qtUVv", [
opts, args = getopt.getopt(params,"A:c:H:hno:p:qtUVv", [
"user-agent=",
"ciphers=",
"compressed",
@ -72,6 +74,8 @@ def options(params):
"help",
"license",
"non-network",
"output=",
"output-format=",
"proxy",
"quiet",
"sites",
@ -86,33 +90,44 @@ def options(params):
error ("Invalid options. See the README or manual for legal rsstube flags.")
sys.exit(2)
for opt, arg in opts:
if arg == "":
arg_count += 1
else:
arg_count += 2
if opt in ("-A", "--user-agent"):
d["user_agent"] = arg
arg_count += 2
elif opt == "--ciphers":
d["ciphers"] = arg
arg_count += 2
elif opt == "--compressed":
d["compressed"] = True
arg_count += 1
elif opt in ("-h", "--help"):
print ("Usage: rsstube [OPTIONS] URL")
# not available yet
# print ("Use `man rsstube` to see the manual for rsstube.")
arg_count += 1
sys.exit()
elif opt in ("-H", "--header"):
header.append(arg)
arg_count += 2
elif opt in ("--license"):
print(license)
arg_count += 1
sys.exit()
elif opt in ("-n", "--non-network"):
network = False
arg_count += 1
elif opt in ("-o", "--output"):
output_filename = arg
arg_count += 2
elif opt == "--output-format":
if str.lower(arg) in ("opml", "url"):
output_format = str.lower(arg)
arg_count += 2
elif opt in ("-p", "--proxy"):
d["proxy"] = arg
arg_count += 2
elif opt in ("-q", "--quiet"):
verbosity = 1
arg_count += 1
elif opt in ("--sites"):
print ("Site-specific support:")
for test in sorted(glob.glob(path + "/tests/*.txt")):
@ -123,30 +138,38 @@ def options(params):
for test in sorted(glob.glob(path + "/tests/generic/*.txt")):
site = test[test.rfind("/")+1:(-4)]
print ("- " + site)
arg_count += 1
sys.exit()
elif opt in ("--suppress-errors"):
verbosity = 0
arg_count += 1
elif opt == "--tls-max":
d["tls_max"] = arg
arg_count += 2
elif opt == "--tls13-ciphers":
d["tls13_ciphers"] = arg
arg_count += 2
elif opt in ("-U", "--update"):
update()
arg_count += 1
sys.exit()
elif opt in ("-v", "--verbose"):
verbosity = 4
arg_count += 1
elif opt == "--verbosity":
v = int(arg)
if v >= 0 and v <= 4:
verbosity = v
else:
print ("Invalid verbosity: " + arg)
arg_count += 2
elif opt in ("-V", "--version"):
version = open(path + "/docs/version","r")
# only go to -1 to cut EOL character
print (version.readline()[:-1])
arg_count += 1
sys.exit()
d["header"] = header
return network,verbosity,d,arg_count
return network,verbosity,d,arg_count,output_format,output_filename

97
scripts/rsstube.py
View File

@ -3,11 +3,16 @@
import sys,importlib
from utils import debug,notify,warn,error,success
import opml
network = True
verbosity = 3
args = {}
arg_count = 0
output_format = "url"
output_opml = None
output_filename = None
output = None
config = None
try:
@ -22,16 +27,29 @@ except FileNotFoundError:
except FileNotFoundError:
# no change
config = None
file_params = None
file_params = ""
if not config is None:
for line in config:
line = line.strip()
# comment lines should begin with # after stripping
if line[0] != "#":
if line != "" and line[0] != "#":
file_params += " " + line
from options import options
def process_args (network,verbosity,args,arg_count,network_new,verbosity_new,args_new,arg_count_new):
def process_args (
network,
verbosity,
args,
arg_count,
output_format,
output_filename,
network_new,
verbosity_new,
args_new,
arg_count_new,
output_format_new,
output_filename_new
):
if network_new == False:
network = network_new
if not verbosity_new is None:
@ -39,18 +57,54 @@ def process_args (network,verbosity,args,arg_count,network_new,verbosity_new,arg
for i in args_new:
args[i] = args_new[i]
arg_count = arg_count_new
return network,verbosity,args,arg_count
if output_format_new != "":
output_format = output_format_new
if not output_filename_new is None:
output_filename = output_filename_new
return network,verbosity,args,arg_count,output_format,output_filename
# config file options
if not file_params == "" and not file_params is None:
network_new,verbosity_new,args_new,arg_count_new = options(file_params.split())
network,verbosity,args,arg_count = process_args(network,verbosity,args,arg_count,network_new,verbosity_new,args_new,arg_count_new)
if not file_params == "":
network_new,verbosity_new,args_new,arg_count_new,output_format_new,output_filename_new = options(file_params.split())
network,verbosity,args,arg_count,output_format,output_filename = process_args(
network,
verbosity,
args,
arg_count,
output_format,
output_filename,
network_new,
verbosity_new,
args_new,
arg_count_new,
output_format_new,
output_filename_new
)
# command-line options
network_new,verbosity_new,args_new,arg_count_new = options(sys.argv[1:])
network,verbosity,args,arg_count = process_args(network,verbosity,args,arg_count,network_new,verbosity_new,args_new,arg_count_new)
#if not verbosity_temp is None:
# verbosity = verbosity_temp
network_new,verbosity_new,args_new,arg_count_new,output_format_new,output_filename_new = options(sys.argv[1:])
network,verbosity,args,arg_count,output_format,output_filename = process_args(
network,
verbosity,
args,
arg_count,
output_format,
output_filename,
network_new,
verbosity_new,
args_new,
arg_count_new,
output_format_new,
output_filename_new
)
if output_format == "opml":
debug ("Formatting output as OPML.", verbosity)
output_opml = opml.Opml("rsstube feeds")
if not output_filename is None and output_filename != "":
debug ("Output will be saved in " + output_filename, verbosity)
output = open(output_filename, "w")
if len(sys.argv) == arg_count+1:
error ("Please provide one or more URL.", verbosity)
@ -69,7 +123,10 @@ for url in sys.argv[arg_count+1:]:
if feed is None:
error ("Unable to get RSS feed for " + url, verbosity, site)
else:
success (feed)
if not output_opml is None:
output_opml.add_feed (feed, site + ": " + url, url)
else:
success (feed, output)
elif network:
from download_page import download
page = download (None, url, args, verbosity)
@ -89,7 +146,10 @@ for url in sys.argv[arg_count+1:]:
if feed is None:
notify ("Unable to get RSS feed for " + url + " with " + software + " extractor", verbosity, software)
else:
success (feed)
if not output_opml is None:
output_opml.add_feed (feed, software + ": " + url, url)
else:
success (feed, output)
continue
# try generic extractor even if software is known
@ -99,6 +159,15 @@ for url in sys.argv[arg_count+1:]:
if feed is None:
error ("Unable to get RSS feed for " + url, verbosity, "generic")
else:
success (feed)
if not output_opml is None:
output_opml.add_feed (feed, url, url)
else:
success (feed, output)
else:
error ("Unable to get RSS feed for " + url + " without downloading page", verbosity)
if not output_opml is None:
success (output_opml.get_opml(), output)
if not output is None:
output.close()

9
scripts/utils.py
View File

@ -24,10 +24,11 @@ def error (message, verbosity=1, platform=None):
if verbosity >= 1:
print (color.ERR + "[" + str(platform) + "] " + message + color.NC)
def success (message):
# caused issues with piping output into other stuff
# print (color.SUCCESS + message + color.NC)
print (message)
def success (message, output):
if output is None:
print (message)
else:
output.write (message + "\n")
def search (content, begins_with, ends_with, index=0, reverse=False):
# hack to search based on ends_with being significant

2
tests/test.sh
View File

@ -33,7 +33,7 @@ function test_site {
echo "Goal:"
echo "${links[1]}"
output=$(/usr/bin/python3 ../rsstube "${links[0]}" | tail -1)
output=$(/usr/bin/python3 ../rsstube -o "" --output-format url "${links[0]}" | tail -1)
echo "Output:"
echo "${output}"