33 lines
933 B
Python
33 lines
933 B
Python
#!/usr/bin/python3
|
|
|
|
import pycurl
|
|
from io import BytesIO
|
|
from utils import notify,debug
|
|
|
|
# args should be a dictionary of arguments
|
|
# return page bytes, response code
|
|
def download (platform, url, args, verbosity, return_http_code=False):
|
|
page_bytes = BytesIO()
|
|
c = pycurl.Curl()
|
|
|
|
c.setopt(c.URL, url)
|
|
c.setopt(c.WRITEDATA, page_bytes)
|
|
c.setopt(c.FOLLOWLOCATION, True)
|
|
|
|
# TODO: handle possible arguments
|
|
# if args["user_agent"]:
|
|
# c.setopt(pycurl.USERAGENT, args["user_agent"]
|
|
# if args["ciphers"]:
|
|
# c.setopt(pycurl.CIPHERS, args["ciphers"]
|
|
|
|
notify ("Downloading " + url + "...", verbosity, platform)
|
|
c.perform()
|
|
response_code = c.getinfo(c.RESPONSE_CODE)
|
|
c.close()
|
|
debug (url + " downloaded!", verbosity, platform)
|
|
debug ("Response code: " + str(response_code), verbosity, platform)
|
|
if return_http_code:
|
|
return page_bytes.getvalue().decode('utf8'),response_code
|
|
else:
|
|
return page_bytes.getvalue().decode('utf8')
|