#!/usr/bin/python3 import pycurl from io import BytesIO from utils import notify,debug,error # args should be a dictionary of arguments # return page bytes, response code def download (platform, url, args, verbosity, return_http_code=False, follow_location=True): page_bytes = BytesIO() c = pycurl.Curl() c.setopt(c.URL, url) c.setopt(c.WRITEDATA, page_bytes) c.setopt(c.FOLLOWLOCATION, follow_location) # TODO: handle possible arguments # if args["user_agent"]: # c.setopt(pycurl.USERAGENT, args["user_agent"] # if args["ciphers"]: # c.setopt(pycurl.CIPHERS, args["ciphers"] notify ("Downloading " + url + "...", verbosity, platform) try: c.perform() except pycurl.error as e: error (str(e), verbosity, platform) return None response_code = c.getinfo(c.RESPONSE_CODE) c.close() debug (url + " downloaded!", verbosity, platform) debug ("Response code: " + str(response_code), verbosity, platform) if return_http_code: return page_bytes.getvalue().decode('utf8'),response_code else: return page_bytes.getvalue().decode('utf8')