0x80
/
rsstube
1
1
Fork 0
Code Issues 6 Pull Requests Projects Releases Wiki Activity

Add output options. 

User can now output to file and/or output in OPML format.
This commit is contained in:
0x80 2021-12-29 00:00:00 +00:00
parent 334f7dbe49
commit cfebf73510
Signed by: 0x80
GPG Key ID: 68368BCBC000EF51
5 changed files with 151 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
scripts/opml.py Normal file
View File

@ -0,0 +1,32 @@
#!/usr/bin/python3
# class for representing and writing an OPML outline for RSS feeds
class Opml:
def __init__(self, title=""):
self.title = title
self.feeds = []
@staticmethod
def get_date():
from datetime import datetime
return datetime.now().strftime("%a %b %d %H:%M:%S %Y")
def add_feed(self, feed, title="", html_url=""):
line = '<outline text="' + title + '" type="rss" htmlUrl="' + html_url + '" xmlUrl="' + feed + '"/>'
self.feeds.append(line)
def get_opml(self):
opml = '''<?xml version="1.0" encoding="UTF-8"?>
<opml version="2.0">
<head>
<title>''' + self.title + '''</title>
<dateModified>''' + self.get_date() + '''</dateModified>
</head>
<body>'''
for line in self.feeds:
opml += "\n\t\t" + line
opml += '''
</body>
</opml>'''
return opml

37
scripts/options.py
View File

@ -47,6 +47,8 @@ def options(params):
# general settings # general settings
network = True network = True
output_format = ""
output_filename = None
## verbosity: in addition to the feed, print... ## verbosity: in addition to the feed, print...
## 0: no messages (suppress errors) ## 0: no messages (suppress errors)
@ -64,7 +66,7 @@ def options(params):
arg_count = 0 arg_count = 0
try: try:
opts, args = getopt.getopt(params,"A:c:H:hnp:qtUVv", [ opts, args = getopt.getopt(params,"A:c:H:hno:p:qtUVv", [
"user-agent=", "user-agent=",
"ciphers=", "ciphers=",
"compressed", "compressed",
@ -72,6 +74,8 @@ def options(params):
"help", "help",
"license", "license",
"non-network", "non-network",
"output=",
"output-format=",
"proxy", "proxy",
"quiet", "quiet",
"sites", "sites",
@ -86,33 +90,44 @@ def options(params):
error ("Invalid options. See the README or manual for legal rsstube flags.") error ("Invalid options. See the README or manual for legal rsstube flags.")
sys.exit(2) sys.exit(2)
for opt, arg in opts: for opt, arg in opts:
if arg == "":
arg_count += 1
else:
arg_count += 2
if opt in ("-A", "--user-agent"): if opt in ("-A", "--user-agent"):
d["user_agent"] = arg d["user_agent"] = arg
arg_count += 2
elif opt == "--ciphers": elif opt == "--ciphers":
d["ciphers"] = arg d["ciphers"] = arg
arg_count += 2
elif opt == "--compressed": elif opt == "--compressed":
d["compressed"] = True d["compressed"] = True
arg_count += 1
elif opt in ("-h", "--help"): elif opt in ("-h", "--help"):
print ("Usage: rsstube [OPTIONS] URL") print ("Usage: rsstube [OPTIONS] URL")
# not available yet # not available yet
# print ("Use `man rsstube` to see the manual for rsstube.") # print ("Use `man rsstube` to see the manual for rsstube.")
arg_count += 1
sys.exit() sys.exit()
elif opt in ("-H", "--header"): elif opt in ("-H", "--header"):
header.append(arg) header.append(arg)
arg_count += 2
elif opt in ("--license"): elif opt in ("--license"):
print(license) print(license)
arg_count += 1
sys.exit() sys.exit()
elif opt in ("-n", "--non-network"): elif opt in ("-n", "--non-network"):
network = False network = False
arg_count += 1
elif opt in ("-o", "--output"):
output_filename = arg
arg_count += 2
elif opt == "--output-format":
if str.lower(arg) in ("opml", "url"):
output_format = str.lower(arg)
arg_count += 2
elif opt in ("-p", "--proxy"): elif opt in ("-p", "--proxy"):
d["proxy"] = arg d["proxy"] = arg
arg_count += 2
elif opt in ("-q", "--quiet"): elif opt in ("-q", "--quiet"):
verbosity = 1 verbosity = 1
arg_count += 1
elif opt in ("--sites"): elif opt in ("--sites"):
print ("Site-specific support:") print ("Site-specific support:")
for test in sorted(glob.glob(path + "/tests/*.txt")): for test in sorted(glob.glob(path + "/tests/*.txt")):
@ -123,30 +138,38 @@ def options(params):
for test in sorted(glob.glob(path + "/tests/generic/*.txt")): for test in sorted(glob.glob(path + "/tests/generic/*.txt")):
site = test[test.rfind("/")+1:(-4)] site = test[test.rfind("/")+1:(-4)]
print ("- " + site) print ("- " + site)
arg_count += 1
sys.exit() sys.exit()
elif opt in ("--suppress-errors"): elif opt in ("--suppress-errors"):
verbosity = 0 verbosity = 0
arg_count += 1
elif opt == "--tls-max": elif opt == "--tls-max":
d["tls_max"] = arg d["tls_max"] = arg
arg_count += 2
elif opt == "--tls13-ciphers": elif opt == "--tls13-ciphers":
d["tls13_ciphers"] = arg d["tls13_ciphers"] = arg
arg_count += 2
elif opt in ("-U", "--update"): elif opt in ("-U", "--update"):
update() update()
arg_count += 1
sys.exit() sys.exit()
elif opt in ("-v", "--verbose"): elif opt in ("-v", "--verbose"):
verbosity = 4 verbosity = 4
arg_count += 1
elif opt == "--verbosity": elif opt == "--verbosity":
v = int(arg) v = int(arg)
if v >= 0 and v <= 4: if v >= 0 and v <= 4:
verbosity = v verbosity = v
else: else:
print ("Invalid verbosity: " + arg) print ("Invalid verbosity: " + arg)
arg_count += 2
elif opt in ("-V", "--version"): elif opt in ("-V", "--version"):
version = open(path + "/docs/version","r") version = open(path + "/docs/version","r")
# only go to -1 to cut EOL character # only go to -1 to cut EOL character
print (version.readline()[:-1]) print (version.readline()[:-1])
arg_count += 1
sys.exit() sys.exit()
d["header"] = header d["header"] = header
return network,verbosity,d,arg_count return network,verbosity,d,arg_count,output_format,output_filename

97
scripts/rsstube.py
View File

@ -3,11 +3,16 @@
import sys,importlib import sys,importlib
from utils import debug,notify,warn,error,success from utils import debug,notify,warn,error,success
import opml
network = True network = True
verbosity = 3 verbosity = 3
args = {} args = {}
arg_count = 0 arg_count = 0
output_format = "url"
output_opml = None
output_filename = None
output = None
config = None config = None
try: try:
@ -22,16 +27,29 @@ except FileNotFoundError:
except FileNotFoundError: except FileNotFoundError:
# no change # no change
config = None config = None
file_params = None file_params = ""
if not config is None: if not config is None:
for line in config: for line in config:
line = line.strip() line = line.strip()
# comment lines should begin with # after stripping # comment lines should begin with # after stripping
if line[0] != "#": if line != "" and line[0] != "#":
file_params += " " + line file_params += " " + line
from options import options from options import options
def process_args (network,verbosity,args,arg_count,network_new,verbosity_new,args_new,arg_count_new): def process_args (
network,
verbosity,
args,
arg_count,
output_format,
output_filename,
network_new,
verbosity_new,
args_new,
arg_count_new,
output_format_new,
output_filename_new
):
if network_new == False: if network_new == False:
network = network_new network = network_new
if not verbosity_new is None: if not verbosity_new is None:
@ -39,18 +57,54 @@ def process_args (network,verbosity,args,arg_count,network_new,verbosity_new,arg
for i in args_new: for i in args_new:
args[i] = args_new[i] args[i] = args_new[i]
arg_count = arg_count_new arg_count = arg_count_new
return network,verbosity,args,arg_count if output_format_new != "":
output_format = output_format_new
if not output_filename_new is None:
output_filename = output_filename_new
return network,verbosity,args,arg_count,output_format,output_filename
# config file options # config file options
if not file_params == "" and not file_params is None: if not file_params == "":
network_new,verbosity_new,args_new,arg_count_new = options(file_params.split()) network_new,verbosity_new,args_new,arg_count_new,output_format_new,output_filename_new = options(file_params.split())
network,verbosity,args,arg_count = process_args(network,verbosity,args,arg_count,network_new,verbosity_new,args_new,arg_count_new) network,verbosity,args,arg_count,output_format,output_filename = process_args(
network,
verbosity,
args,
arg_count,
output_format,
output_filename,
network_new,
verbosity_new,
args_new,
arg_count_new,
output_format_new,
output_filename_new
)
# command-line options # command-line options
network_new,verbosity_new,args_new,arg_count_new = options(sys.argv[1:]) network_new,verbosity_new,args_new,arg_count_new,output_format_new,output_filename_new = options(sys.argv[1:])
network,verbosity,args,arg_count = process_args(network,verbosity,args,arg_count,network_new,verbosity_new,args_new,arg_count_new) network,verbosity,args,arg_count,output_format,output_filename = process_args(
#if not verbosity_temp is None: network,
# verbosity = verbosity_temp verbosity,
args,
arg_count,
output_format,
output_filename,
network_new,
verbosity_new,
args_new,
arg_count_new,
output_format_new,
output_filename_new
)
if output_format == "opml":
debug ("Formatting output as OPML.", verbosity)
output_opml = opml.Opml("rsstube feeds")
if not output_filename is None and output_filename != "":
debug ("Output will be saved in " + output_filename, verbosity)
output = open(output_filename, "w")
if len(sys.argv) == arg_count+1: if len(sys.argv) == arg_count+1:
error ("Please provide one or more URL.", verbosity) error ("Please provide one or more URL.", verbosity)
@ -69,7 +123,10 @@ for url in sys.argv[arg_count+1:]:
if feed is None: if feed is None:
error ("Unable to get RSS feed for " + url, verbosity, site) error ("Unable to get RSS feed for " + url, verbosity, site)
else: else:
success (feed) if not output_opml is None:
output_opml.add_feed (feed, site + ": " + url, url)
else:
success (feed, output)
elif network: elif network:
from download_page import download from download_page import download
page = download (None, url, args, verbosity) page = download (None, url, args, verbosity)
@ -89,7 +146,10 @@ for url in sys.argv[arg_count+1:]:
if feed is None: if feed is None:
notify ("Unable to get RSS feed for " + url + " with " + software + " extractor", verbosity, software) notify ("Unable to get RSS feed for " + url + " with " + software + " extractor", verbosity, software)
else: else:
success (feed) if not output_opml is None:
output_opml.add_feed (feed, software + ": " + url, url)
else:
success (feed, output)
continue continue
# try generic extractor even if software is known # try generic extractor even if software is known
@ -99,6 +159,15 @@ for url in sys.argv[arg_count+1:]:
if feed is None: if feed is None:
error ("Unable to get RSS feed for " + url, verbosity, "generic") error ("Unable to get RSS feed for " + url, verbosity, "generic")
else: else:
success (feed) if not output_opml is None:
output_opml.add_feed (feed, url, url)
else:
success (feed, output)
else: else:
error ("Unable to get RSS feed for " + url + " without downloading page", verbosity) error ("Unable to get RSS feed for " + url + " without downloading page", verbosity)
if not output_opml is None:
success (output_opml.get_opml(), output)
if not output is None:
output.close()

9
scripts/utils.py
View File

@ -24,10 +24,11 @@ def error (message, verbosity=1, platform=None):
if verbosity >= 1: if verbosity >= 1:
print (color.ERR + "[" + str(platform) + "] " + message + color.NC) print (color.ERR + "[" + str(platform) + "] " + message + color.NC)
def success (message): def success (message, output):
# caused issues with piping output into other stuff if output is None:
# print (color.SUCCESS + message + color.NC) print (message)
print (message) else:
output.write (message + "\n")
def search (content, begins_with, ends_with, index=0, reverse=False): def search (content, begins_with, ends_with, index=0, reverse=False):
# hack to search based on ends_with being significant # hack to search based on ends_with being significant

2
tests/test.sh
View File

@ -33,7 +33,7 @@ function test_site {
echo "Goal:" echo "Goal:"
echo "${links[1]}" echo "${links[1]}"
output=$(/usr/bin/python3 ../rsstube "${links[0]}" | tail -1) output=$(/usr/bin/python3 ../rsstube -o "" --output-format url "${links[0]}" | tail -1)
echo "Output:" echo "Output:"
echo "${output}" echo "${output}"