Python urllib2 模块，HTTPPasswordMgrWithDefaultRealm() 实例源码

def run(self):
        password = getword()
        try:
            print "-"*12
            print "User:",username,"Password:",password
            req = urllib2.Request(sys.argv[1])
            passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
            passman.add_password(None, sys.argv[1], username, password)
            authhandler = urllib2.HTTPBasicAuthHandler(passman)
            opener = urllib2.build_opener(authhandler)
            fd = opener.open(req)
            print "\t\n\n[+] Login successful: Username:",username,"Password:",password,"\n"            
            print "[+] Retrieved", fd.geturl()
            info = fd.info()
            for key, value in info.items():
                    print "%s = %s" % (key, value)
            sys.exit(2)
        except (urllib2.HTTPError,socket.error):
            pass

def run(self):
        username, password = getword()
        try:
            print "-"*12
            print "User:",username,"Password:",password
            req = urllib2.Request(sys.argv[1])
            passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
            passman.add_password(None, sys.argv[1], username, password)
            authhandler = urllib2.HTTPBasicAuthHandler(passman)
            opener = urllib2.build_opener(authhandler)
            fd = opener.open(req)
            print "\t\n\nUsername:",username,"Password:",password,"----- Login successful!!!\n\n"           
            print "Retrieved", fd.geturl()
            info = fd.info()
            for key, value in info.items():
                    print "%s = %s" % (key, value)
            sys.exit(2)
        except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
            print "An error occurred:", msg
            pass

def add_proxy(self, addr, proxy_type='all',
                  user=None, password=None):
        """Add proxy"""
        if proxy_type == 'all':
            self.proxies = {
                'http': addr,
                'https': addr,
                'ftp': addr
            }
        else:
            self.proxies[proxy_type] = addr
        proxy_handler = urllib2.ProxyHandler(self.proxies)
        self.__build_opener()
        self.opener.add_handler(proxy_handler)

        if user and password:
            pwd_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
            pwd_manager.add_password(None, addr, user, password)
            proxy_auth_handler = urllib2.ProxyBasicAuthHandler(pwd_manager)
            self.opener.add_handler(proxy_auth_handler)

        urllib2.install_opener(self.opener)

def authenticate(top_level_url=u'https://api.github.com'):
    try:
        if 'GH_AUTH_USER' not in os.environ:
            try:
                username =  raw_input(u'Username: ')
            except NameError:
                username =  input(u'Username: ')
        else:
            username = os.environ['GH_AUTH_USER']
        if 'GH_AUTH_PASS' not in os.environ:
            password = getpass.getpass(u'Password: ')
        else:
            password = os.environ['GH_AUTH_USER']
    except KeyboardInterrupt:
        sys.exit(u'')
    try:
        import urllib.request as urllib_alias
    except ImportError:
        import urllib2 as urllib_alias
    password_mgr = urllib_alias.HTTPPasswordMgrWithDefaultRealm()
    password_mgr.add_password(None, top_level_url, username, password)
    handler = urllib_alias.HTTPBasicAuthHandler(password_mgr)
    opener = urllib_alias.build_opener(handler)
    urllib_alias.install_opener(opener)

def setup_wsse_handler(base_url, username, password, preempt = True):
  """
  Configure urllib2 to try/use WSSE authentication, with a specific
  `username` and `password` when visiting any page that have a given
  `base_url`. Once this function has been called, all future requests
  through urllib2 should be able to handle WSSE authentication.
  """

  # Create a password manager
  passman = urllib2.HTTPPasswordMgrWithDefaultRealm()

  # Add username/password for domain defined by base_url
  passman.add_password(None, base_url, username, password)

  # Create the auth handler and install it in urllib2
  authhandler = WSSEAuthHandler(passman, preempt = preempt)
  opener = urllib2.build_opener(authhandler)
  urllib2.install_opener(opener)


# Example of how to use without handlers

def update_tags(self,tags,expression):
        # support cases where we might be doing basic auth through a proxy
        if self.MOLOCH_AUTH == "basic":
            auth_handler = urllib2.HTTPPasswordMgrWithDefaultRealm()
            auth_handler.add_password(None, self.MOLOCH_URL, self.MOLOCH_USER, self.MOLOCH_PASSWORD)
            handler = urllib2.HTTPBasicAuthHandler(auth_handler)
            opener = urllib2.build_opener(handler)
        else: 
            auth_handler = urllib2.HTTPDigestAuthHandler()
            auth_handler.add_password(self.MOLOCH_REALM, self.MOLOCH_URL, self.MOLOCH_USER, self.MOLOCH_PASSWORD)
            opener = urllib2.build_opener(auth_handler)

        data = urllib.urlencode({'tags' : tags})
        qstring = urllib.urlencode({'date' : "-1",'expression' : expression})
        TAG_URL = self.MOLOCH_URL + 'addTags?' + qstring
        try:
            response = opener.open(TAG_URL,data=data)
            if response.code == 200:
                plain_answer = response.read()
                json_data = json.loads(plain_answer)
        except Exception, e:
            log.warning("Moloch: Unable to update tags %s" % (e))

def get_digest_authorization(self, path):
        """
        Returns the digest auth header
        """
        u = self.parsed_url
        if self.digest_auth_handler is None:
            pw_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
            pw_manager.add_password(None, self.conn.host, self.username, self.password)
            self.digest_auth_handler = urllib2.HTTPDigestAuthHandler(pw_manager)

        # building a dummy request that gets never sent,
        # needed for call to auth_handler.get_authorization
        scheme = u.scheme == 'webdavs' and 'https' or 'http'
        hostname = u.port and "%s:%s" % (u.hostname, u.port) or u.hostname
        dummy_url = "%s://%s%s" % (scheme, hostname, path)
        dummy_req = CustomMethodRequest(self.conn._method, dummy_url)
        auth_string = self.digest_auth_handler.get_authorization(dummy_req,
                                                                 self.digest_challenge)
        return 'Digest %s' % auth_string

def __init__(self, base_url, exc_class=None, logger=None):
    """
    @param base_url: The base url to the API.
    @param exc_class: An exception class to handle non-200 results.

    Creates an HTTP(S) client to connect to the Cloudera Manager API.
    """
    self._base_url = base_url.rstrip('/')
    self._exc_class = exc_class or RestException
    self._logger = logger or LOG
    self._headers = { }

    # Make a basic auth handler that does nothing. Set credentials later.
    self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    authhandler = urllib2.HTTPBasicAuthHandler(self._passmgr)

    # Make a cookie processor
    cookiejar = cookielib.CookieJar()

    self._opener = urllib2.build_opener(
        HTTPErrorProcessor(),
        urllib2.HTTPCookieProcessor(cookiejar),
        authhandler)

def httpConnection(url,  proxy):



    #TODO: habilitar autenticacion ntlm
    if (proxy.auth == "ntlm"):
        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
        passman.add_password(None, proxy.url, proxy.user, proxy.password)
        auth = HTTPNtlmAuthHandler.HTTPNtlmAuthHandler(passman)
    else:
        passman = urllib2.HTTPPasswordMgr()
        passman.add_password(None, proxy.url, proxy.user, proxy.password)
        auth = urllib2.HTTPBasicAuthHandler(passman)


    if (proxy.url):
        proxy = urllib2.ProxyHandler({'http': proxy.url})
        opener = urllib2.build_opener(proxy.url, auth, urllib2.HTTPHandler)
        urllib2.install_opener(opener)

    return urllib2.urlopen(url)

def __init__(self, password_mgr=None, debuglevel=0):
        """Initialize an instance of a AbstractNtlmAuthHandler.

Verify operation with all default arguments.
>>> abstrct = AbstractNtlmAuthHandler()

Verify "normal" operation.
>>> abstrct = AbstractNtlmAuthHandler(urllib2.HTTPPasswordMgrWithDefaultRealm())
"""
        if password_mgr is None:
            password_mgr = urllib2.HTTPPasswordMgr()
        self.passwd = password_mgr
        self.add_password = self.passwd.add_password
        self._debuglevel = debuglevel

def threader(site):
    username, password = getword()
    global logins
    try:
        print "-"*12
        print "User:",username,"Password:",password
        req = urllib2.Request(site)
        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
        passman.add_password(None, site, username, password)
        authhandler = urllib2.HTTPBasicAuthHandler(passman)
        opener = urllib2.build_opener(authhandler)
        fd = opener.open(req)
        site = urllib2.urlopen(fd.geturl()).read()
        print "\n[+] Checking the authenticity of the login...\n"
        if not re.search(('denied'), site.lower()):
            print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
            print "[+] Writing Successful Login:",sys.argv[5],"\n"
            logins +=1
            file = open(sys.argv[5], "a")
            file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
            file.close()            
            print "Retrieved", fd.geturl()
            info = fd.info()
            for key, value in info.items():
                    print "%s = %s" % (key, value)
        else: 
            print "- Redirection"
    except (urllib2.HTTPError, httplib.BadStatusLine,socket.error), msg: 
        print "An error occurred:", msg
        pass

def threader(site):
    username, password = getword()
    global logins
    try:
        print "-"*12
        print "User:",username,"Password:",password
        req = urllib2.Request(site)
        passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
        passman.add_password(None, site, username, password)
        authhandler = urllib2.HTTPBasicAuthHandler(passman)
        opener = urllib2.build_opener(authhandler)
        fd = opener.open(req)
        site = urllib2.urlopen(fd.geturl()).read()
        print "\n[+] Checking the authenticity of the login...\n"
        if not re.search(('denied'), site.lower()):
            print "\t\n\n[+] Username:",username,"Password:",password,"----- Login successful!!!\n\n"
            print "[+] Writing Successful Login:",sys.argv[5],"\n"
            logins +=1
            file = open(sys.argv[5], "a")
            file.writelines("Site: "+site+" Username: "+username+ " Password: "+password+"\n")
            file.close()            
            print "Retrieved", fd.geturl()
            info = fd.info()
            for key, value in info.items():
                    print "%s = %s" % (key, value)
        else: 
            print "- Redirection\n"
    except (urllib2.HTTPError,httplib.BadStatusLine,socket.error), msg: 
        print "An error occurred:", msg
        pass

def __init__(self, username, password,  # pylint: disable=E1002
                 verbose=0, use_datetime=False, https_handler=None):
        """Initialize"""
        if PY2:
            Transport.__init__(self, use_datetime=False)
        else:
            super().__init__(use_datetime=False)
        self._username = username
        self._password = password
        self._use_datetime = use_datetime
        self.verbose = verbose
        self._username = username
        self._password = password

        self._handlers = []

        if self._username and self._password:
            self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            self._auth_handler = urllib2.HTTPDigestAuthHandler(self._passmgr)
        else:
            self._auth_handler = None
            self._passmgr = None

        if https_handler:
            self._handlers.append(https_handler)
            self._scheme = 'https'
        else:
            self._scheme = 'http'

        if self._auth_handler:
            self._handlers.append(self._auth_handler)

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def __init__(self, username, password,  #pylint: disable=E1002
                 verbose=0, use_datetime=False, https_handler=None):
        """Initialize"""
        if PY2:
            Transport.__init__(self, use_datetime=False)
        else:
            super().__init__(use_datetime=False)
        self._username = username
        self._password = password
        self._use_datetime = use_datetime
        self.verbose = verbose
        self._username = username
        self._password = password

        self._handlers = []

        if self._username and self._password:
            self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            self._auth_handler = urllib2.HTTPDigestAuthHandler(self._passmgr)
        else:
            self._auth_handler = None
            self._passmgr = None

        if https_handler:
            self._handlers.append(https_handler)
            self._scheme = 'https'
        else:
            self._scheme = 'http'

        if self._auth_handler:
            self._handlers.append(self._auth_handler)

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def __init__(self, username, password=None):
        # Get password if necessary
        if password is None:
            password = getpass()
        # Get URL for the database
        self.db_url = "http://galaxy-catalogue.dur.ac.uk:8080/Eagle"
        # Set up authentication and cookies
        self.password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        self.password_mgr.add_password(None, self.db_url, username, password)
        self.opener = urllib2.OpenerDirector()
        self.auth_handler   = urllib2.HTTPBasicAuthHandler(self.password_mgr)
        self.cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)

    ## This functions executes an SQL query on the database and returns the result as a record array.

def __init__(self, username, password=None):
        # Get password if necessary
        if password is None:
            password = getpass()
        # Get URL for the database
        self.db_url = "http://galaxy-catalogue.dur.ac.uk:8080/Eagle"
        # Set up authentication and cookies
        self.password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        self.password_mgr.add_password(None, self.db_url, username, password)
        self.opener = urllib2.OpenerDirector()
        self.auth_handler   = urllib2.HTTPBasicAuthHandler(self.password_mgr)
        self.cookie_handler = urllib2.HTTPCookieProcessor(cookie_jar)

    ## This functions executes an SQL query on the database and returns the result as a record array.

def call_api(self, path):
        '''Call the REST API and convert the results into JSON.'''
        url = '{0}://{1}:{2}/api/{3}'.format(self.protocol, self.host_name,
                                             self.port, path)
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(None, url, self.user_name, self.password)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        logging.debug('Issue a rabbit API call to get data on ' + path)
        return json.loads(urllib2.build_opener(handler).open(url).read())

def auth (usr, pwd, dhusAlt=None):
  """Globally install Basic Auth, and set site specific string constants."""
  SITE["NAME"] = dhusAlt if dhusAlt is not None else "SciHub"
  site,collspec = "https://scihub.copernicus.eu/", "producttype:S2MSI%s"
  if SITE["NAME"]=="CODE-DE": site,collspec = "https://code-de.org/", "platformname:Sentinel-2"
  SITE["BASE"] = site+"dhus/"
#  pm=urllib2.HTTPPasswordMgrWithDefaultRealm()
#  pm.add_password(None, SITE["BASE"], usr, pwd)
#  urllib2.install_opener(urllib2.build_opener(urllib2.HTTPBasicAuthHandler(pm)))
#...does not work transparently in combination with proxy support => workaround: urlOpen().
  import base64; SITE["AUTH"] = "Basic " + base64.b64encode("%s:%s" % (usr, pwd))
  SITE["SEARCH"] = SITE["BASE"] + "search?format=xml&sortedby=beginposition&order=desc&rows=%d&q=%s" % (ROWSSTEP, collspec)
  product = SITE["BASE"] + "odata/v1/Products('%s')/"
  SITE["CHECKSUM"], SITE["SAFEZIP"], SITE["SAFEROOT"] = product+"Checksum/Value/$value", product+"$value", product+"Nodes('%s.SAFE')/"

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def _login_BA(self):
        try:
            # Creates a PasswordMgr instance
            passmanager = urllib2.HTTPPasswordMgrWithDefaultRealm()
            passmanager.add_password(None, self.url, self.username, self.password)
            # Creates an auth handling object and builds it with opener
            auth = urllib2.HTTPBasicAuthHandler(passmanager)
            opener = urllib2.build_opener(auth)
            response = opener.open(self.url, timeout=8)
            data = response.read()
            response.close()
            return data
        except Exception, e:
            if 'Error 401' in str(e):
                raise Exception('Login credentials incorrect.')

def _open_url(self, url):
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(None, self._routes.host, '', self._api_key)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener.open(url)

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def __init__(self, password_mgr=None, debuglevel=0):
        """Initialize an instance of a AbstractNtlmAuthHandler.

Verify operation with all default arguments.
>>> abstrct = AbstractNtlmAuthHandler()

Verify "normal" operation.
>>> abstrct = AbstractNtlmAuthHandler(urllib2.HTTPPasswordMgrWithDefaultRealm())
"""
        if password_mgr is None:
            password_mgr = urllib2.HTTPPasswordMgr()
        self.passwd = password_mgr
        self.add_password = self.passwd.add_password
        self._debuglevel = debuglevel

def fetch_file(uri, file=None, username=None, password=None):
    """
    Fetch a file based on the URI provided. If you do not pass in a file pointer
    a tempfile.NamedTemporaryFile, or None if the file could not be
    retrieved is returned.
    The URI can be either an HTTP url, or "s3://bucket_name/key_name"
    """
    boto.log.info('Fetching %s' % uri)
    if file == None:
        file = tempfile.NamedTemporaryFile()
    try:
        if uri.startswith('s3://'):
            bucket_name, key_name = uri[len('s3://'):].split('/', 1)
            c = boto.connect_s3(aws_access_key_id=username,
                                aws_secret_access_key=password)
            bucket = c.get_bucket(bucket_name)
            key = bucket.get_key(key_name)
            key.get_contents_to_file(file)
        else:
            if username and password:
                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
                passman.add_password(None, uri, username, password)
                authhandler = urllib2.HTTPBasicAuthHandler(passman)
                opener = urllib2.build_opener(authhandler)
                urllib2.install_opener(opener)
            s = urllib2.urlopen(uri)
            file.write(s.read())
        file.seek(0)
    except:
        raise
        boto.log.exception('Problem Retrieving file: %s' % uri)
        file = None
    return file

def fetch_file(uri, file=None, username=None, password=None):
    """
    Fetch a file based on the URI provided. If you do not pass in a file pointer
    a tempfile.NamedTemporaryFile, or None if the file could not be
    retrieved is returned.
    The URI can be either an HTTP url, or "s3://bucket_name/key_name"
    """
    boto.log.info('Fetching %s' % uri)
    if file == None:
        file = tempfile.NamedTemporaryFile()
    try:
        if uri.startswith('s3://'):
            bucket_name, key_name = uri[len('s3://'):].split('/', 1)
            c = boto.connect_s3(aws_access_key_id=username,
                                aws_secret_access_key=password)
            bucket = c.get_bucket(bucket_name)
            key = bucket.get_key(key_name)
            key.get_contents_to_file(file)
        else:
            if username and password:
                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
                passman.add_password(None, uri, username, password)
                authhandler = urllib2.HTTPBasicAuthHandler(passman)
                opener = urllib2.build_opener(authhandler)
                urllib2.install_opener(opener)
            s = urllib2.urlopen(uri)
            file.write(s.read())
        file.seek(0)
    except:
        raise
        boto.log.exception('Problem Retrieving file: %s' % uri)
        file = None
    return file

def http_auth_request(url, host, user, passwd, user_agent=USER_AGENT):
    """Call an HTTP server with authorization credentials using urllib2.
    """
    if DEBUG: httplib.HTTPConnection.debuglevel = 1

    # Hook up handler/opener to urllib2
    password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
    password_manager.add_password(None, host, user, passwd)
    auth_handler = urllib2.HTTPBasicAuthHandler(password_manager)
    opener = urllib2.build_opener(auth_handler)
    urllib2.install_opener(opener)

    return http_request(url, user_agent)

def __init__(self, ipaddress, username = 'admin', password = ''):
        self.address = ipaddress
        self.username = username
        self.password = password

        pwd_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        pwd_mgr.add_password(None, 'http://{}'.format(self.address), self.username, self.password)
        handler = urllib2.HTTPBasicAuthHandler(pwd_mgr)

        self.conn = urllib2.build_opener(handler)

        self._url_snapshot = 'http://{address}/snapshot.cgi'.format(address = self.address)
        self._url_command = 'http://{address}/decoder_control.cgi?username={username}&password={password}&command='.format(address = self.address, username = self.username, password = self.password) + '{command}'

def fetch_file(uri, file=None, username=None, password=None):
    """
    Fetch a file based on the URI provided. If you do not pass in a file pointer
    a tempfile.NamedTemporaryFile, or None if the file could not be
    retrieved is returned.
    The URI can be either an HTTP url, or "s3://bucket_name/key_name"
    """
    boto.log.info('Fetching %s' % uri)
    if file == None:
        file = tempfile.NamedTemporaryFile()
    try:
        if uri.startswith('s3://'):
            bucket_name, key_name = uri[len('s3://'):].split('/', 1)
            c = boto.connect_s3(aws_access_key_id=username,
                                aws_secret_access_key=password)
            bucket = c.get_bucket(bucket_name)
            key = bucket.get_key(key_name)
            key.get_contents_to_file(file)
        else:
            if username and password:
                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
                passman.add_password(None, uri, username, password)
                authhandler = urllib2.HTTPBasicAuthHandler(passman)
                opener = urllib2.build_opener(authhandler)
                urllib2.install_opener(opener)
            s = urllib2.urlopen(uri)
            file.write(s.read())
        file.seek(0)
    except:
        raise
        boto.log.exception('Problem Retrieving file: %s' % uri)
        file = None
    return file

def readUrl(self,host,port,url,user,password):
        error=False
        tomcatUrl = "http://"+host+":"+str(port)+url
        try:
            pwdManager = urlconnection.HTTPPasswordMgrWithDefaultRealm()
            pwdManager.add_password(None,tomcatUrl,user,password)
            authHandler = urlconnection.HTTPBasicAuthHandler(pwdManager)
            opener=urlconnection.build_opener(authHandler)
            urlconnection.install_opener(opener)
            req = urlconnection.Request(tomcatUrl)
            handle = urlconnection.urlopen(req, None)
            data = handle.read()
        except HTTPError as e:
            if(e.code==401):
                data="ERROR: Unauthorized user. Does not have permissions. %s" %(e)
            elif(e.code==403):
                data="ERROR: Forbidden, yours credentials are not correct. %s" %(e)
            else:
                data="ERROR: The server couldn\'t fulfill the request. %s" %(e)
            error=True
        except URLError as e:
            data = 'ERROR: We failed to reach a server. Reason: %s' %(e.reason)
            error = True
        except socket.timeout as e:
            data = 'ERROR: Timeout error'
            error = True
        except socket.error as e:
            data = "ERROR: Unable to connect with host "+self.host+":"+self.port
            error = True
        except:
            data = "ERROR: Unexpected error: %s"%(sys.exc_info()[0])
            error = True

        return data,error

def readUrl(self,host,port,url,user,password):
        error=False
        tomcatUrl = "http://"+host+":"+str(port)+url
        try:
            pwdManager = urlconnection.HTTPPasswordMgrWithDefaultRealm()
            pwdManager.add_password(None,tomcatUrl,user,password)
            authHandler = urlconnection.HTTPBasicAuthHandler(pwdManager)
            opener=urlconnection.build_opener(authHandler)
            urlconnection.install_opener(opener)
            req = urlconnection.Request(tomcatUrl)
            handle = urlconnection.urlopen(req, None)
            data = handle.read()
        except HTTPError as e:
            if(e.code==401):
                data="ERROR: Unauthorized user. Does not have permissions. %s" %(e)
            elif(e.code==403):
                data="ERROR: Forbidden, yours credentials are not correct. %s" %(e)
            else:
                data="ERROR: The server couldn\'t fulfill the request. %s" %(e)
            error=True
        except URLError as e:
            data = 'ERROR: We failed to reach a server. Reason: %s' %(e.reason)
            error = True
        except socket.timeout as e:
            data = 'ERROR: Timeout error'
            error = True
        except socket.error as e:
            data = "ERROR: Unable to connect with host "+self.host+":"+self.port
            error = True
        except:
            data = "ERROR: Unexpected error: %s"%(sys.exc_info()[0])
            error = True

        return data,error

def metricCollector():
    data = {}

    #defaults
    data['plugin_version'] = PLUGIN_VERSION

    data['heartbeat_required']=HEARTBEAT

    data['units']=METRICS_UNITS

    URL = "http://"+COUCHDB_HOST+":"+COUCHDB_PORT+COUCHDB_STATS_URI

    try:
        if COUCHDB_USERNAME and COUCHDB_PASSWORD:
            password_mgr = connector.HTTPPasswordMgrWithDefaultRealm()
            password_mgr.add_password(REALM, URL, COUCHDB_USERNAME, COUCHDB_PASSWORD)
            auth_handler = connector.HTTPBasicAuthHandler(password_mgr)
            opener = connector.build_opener(auth_handler)
            connector.install_opener(opener)
        response = connector.urlopen(URL, timeout=10)
        byte_responseData = response.read()
        str_responseData = byte_responseData.decode('UTF-8')
        couch_dict = json.loads(str_responseData)
        for attribute, attribute_value in couch_dict.items():
            for metric, val in attribute_value.items():
                if 'current' in val and val['current'] is not None:
                    if metric in METRICS_KEY_VS_NAME:
                        metric = METRICS_KEY_VS_NAME[metric]
                    data[metric]=val['current']
    except Exception as e:
            data['status']=0
            data['msg']=str(e)  

    return data

def metricCollector():
    data = {}

    #defaults
    data['plugin_version'] = PLUGIN_VERSION

    data['heartbeat_required']=HEARTBEAT

    data['units']=METRICS_UNITS

    URL = "http://"+RIAK_HOST+":"+RIAK_PORT+"/"+RIAK_STATS_URI

    try:

        if RIAK_USERNAME and RIAK_PASSWORD:
            password_mgr = connector.HTTPPasswordMgrWithDefaultRealm()
            password_mgr.add_password(REALM, URL, RIAK_USERNAME, RIAK_PASSWORD)
            auth_handler = connector.HTTPBasicAuthHandler(password_mgr)
            opener = connector.build_opener(auth_handler)
            connector.install_opener(opener)

        response = connector.urlopen(URL, timeout=10)

        byte_responseData = response.read()
        str_responseData = byte_responseData.decode('UTF-8')

        riak_dict = json.loads(str_responseData)

        for metric in riak_dict:
            if metric in METRICS_TO_BE_PUSHED_TO_SERVER:
                value=riak_dict[metric]
                if metric in BYTES_TO_MB_LIST:
                     value=convertBytesToMB(value)
                data[metric]=value 
    except Exception as e:
            data['status']=0
            data['msg']=str(e)

    return data

def readUrl(self,host,port,url,user,password):
        error=False
        tomcatUrl = "http://"+host+":"+str(port)+url
        try:
            pwdManager = urlconnection.HTTPPasswordMgrWithDefaultRealm()
            pwdManager.add_password(None,tomcatUrl,user,password)
            authHandler = urlconnection.HTTPBasicAuthHandler(pwdManager)
            opener=urlconnection.build_opener(authHandler)
            urlconnection.install_opener(opener)
            req = urlconnection.Request(tomcatUrl)
            handle = urlconnection.urlopen(req, None)
            data = handle.read()
        except HTTPError as e:
            if(e.code==401):
                data="ERROR: Unauthorized user. Does not have permissions. %s" %(e)
            elif(e.code==403):
                data="ERROR: Forbidden, yours credentials are not correct. %s" %(e)
            else:
                data="ERROR: The server couldn\'t fulfill the request. %s" %(e)
            error=True
        except URLError as e:
            data = 'ERROR: We failed to reach a server. Reason: %s' %(e.reason)
            error = True
        except socket.timeout as e:
            data = 'ERROR: Timeout error'
            error = True
        except socket.error as e:
            data = "ERROR: Unable to connect with host "+self.host+":"+self.port
            error = True
        except:
            data = "ERROR: Unexpected error: %s"%(sys.exc_info()[0])
            error = True

        return data,error

def __build_url_opener(self, uri):
        """
        Build the url opener for managing HTTP Basic Athentication
        """
        # Create an OpenerDirector with support
        # for Basic HTTP Authentication...
        password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        password_mgr.add_password(realm=None,
                                  uri=uri,
                                  user=self.client_id,
                                  passwd=self.client_secret)
        handler = urllib2.HTTPBasicAuthHandler(password_mgr)
        opener = urllib2.build_opener(handler)
        return opener

def __init__(self, username, password,  #pylint: disable=E1002
                 verbose=0, use_datetime=False, https_handler=None):
        """Initialize"""
        if PY2:
            Transport.__init__(self, use_datetime=False)
        else:
            super().__init__(use_datetime=False)
        self._username = username
        self._password = password
        self._use_datetime = use_datetime
        self.verbose = verbose
        self._username = username
        self._password = password

        self._handlers = []

        if self._username and self._password:
            self._passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
            self._auth_handler = urllib2.HTTPDigestAuthHandler(self._passmgr)
        else:
            self._auth_handler = None
            self._passmgr = None

        if https_handler:
            self._handlers.append(https_handler)
            self._scheme = 'https'
        else:
            self._scheme = 'http'

        if self._auth_handler:
            self._handlers.append(self._auth_handler)

def __setHTTPAuthentication():
    """
    Check and set the HTTP authentication method (Basic or Digest),
    username and password to perform HTTP requests with.
    """

    global authHandler

    if not conf.aType and not conf.aCred:
        return

    elif conf.aType and not conf.aCred:
        errMsg  = "you specified the HTTP Authentication type, but "
        errMsg += "did not provide the credentials"
        raise sqlmapSyntaxException, errMsg

    elif not conf.aType and conf.aCred:
        errMsg  = "you specified the HTTP Authentication credentials, "
        errMsg += "but did not provide the type"
        raise sqlmapSyntaxException, errMsg

    parseTargetUrl()

    debugMsg = "setting the HTTP Authentication type and credentials"
    logger.debug(debugMsg)

    aTypeLower = conf.aType.lower()

    if aTypeLower not in ( "basic", "digest" ):
        errMsg  = "HTTP Authentication type value must be "
        errMsg += "Basic or Digest"
        raise sqlmapSyntaxException, errMsg

    aCredRegExp = re.search("^(.*?)\:(.*?)$", conf.aCred)

    if not aCredRegExp:
        errMsg  = "HTTP Authentication credentials value must be "
        errMsg += "in format username:password"
        raise sqlmapSyntaxException, errMsg

    authUsername = aCredRegExp.group(1)
    authPassword = aCredRegExp.group(2)

    passwordMgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
    passwordMgr.add_password(None, "%s://%s" % (conf.scheme, conf.hostname), authUsername, authPassword)

    if aTypeLower == "basic":
        authHandler = urllib2.HTTPBasicAuthHandler(passwordMgr)
    elif aTypeLower == "digest":
        authHandler = urllib2.HTTPDigestAuthHandler(passwordMgr)

def openWebIF(self, part = None, reader = None):
        self.proto = "http"
        if config.oscaminfo.userdatafromconf.value:
            self.ip = "127.0.0.1"
            udata = self.getUserData()
            if isinstance(udata, str):
                if "httpuser" in udata:
                    self.username=""
                elif "httppwd" in udata:
                    self.password = ""
                else:
                    return False, udata
            else:
                self.port = udata[2]
                self.username = udata[0]
                self.password = udata[1]
        else:
            self.ip = ".".join("%d" % d for d in config.oscaminfo.ip.value)
            self.port = str(config.oscaminfo.port.value)
            self.username = str(config.oscaminfo.username.value)
            self.password = str(config.oscaminfo.password.value)

        if self.port.startswith( '+' ):
            self.proto = "https"
            self.port.replace("+","")

        if part is None:
            self.url = "%s://%s:%s/oscamapi.html?part=status" % ( self.proto, self.ip, self.port )
        else:
            self.url = "%s://%s:%s/oscamapi.html?part=%s" % ( self.proto, self.ip, self.port, part )
        if part is not None and reader is not None:
            self.url = "%s://%s:%s/oscamapi.html?part=%s&label=%s" % ( self.proto, self.ip, self.port, part, reader )

        pwman = urllib2.HTTPPasswordMgrWithDefaultRealm()
        pwman.add_password( None, self.url, self.username, self.password )
        handlers = urllib2.HTTPDigestAuthHandler( pwman )
        opener = urllib2.build_opener( urllib2.HTTPHandler, handlers )
        urllib2.install_opener( opener )
        request = urllib2.Request( self.url )
        err = False
        try:
            data = urllib2.urlopen( request ).read()
            # print data
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                err = str(e.reason)
            elif hasattr(e, "code"):
                err = str(e.code)
        if err is not False:
            print "[openWebIF] Fehler: %s" % err
            return False, err
        else:
            return True, data

def openWebIF(self, part = None, reader = None):
        self.proto = "http"
        if config.ncaminfo.userdatafromconf.value:
            self.ip = "127.0.0.1"
            udata = self.getUserData()
            if isinstance(udata, str):
                if "httpuser" in udata:
                    self.username=""
                elif "httppwd" in udata:
                    self.password = ""
                else:
                    return False, udata
            else:
                self.port = udata[2]
                self.username = udata[0]
                self.password = udata[1]
        else:
            self.ip = ".".join("%d" % d for d in config.ncaminfo.ip.value)
            self.port = str(config.ncaminfo.port.value)
            self.username = str(config.ncaminfo.username.value)
            self.password = str(config.ncaminfo.password.value)

        if self.port.startswith( '+' ):
            self.proto = "https"
            self.port.replace("+","")

        if part is None:
            self.url = "%s://%s:%s/ncamapi.html?part=status" % ( self.proto, self.ip, self.port )
        else:
            self.url = "%s://%s:%s/ncamapi.html?part=%s" % ( self.proto, self.ip, self.port, part )
        if part is not None and reader is not None:
            self.url = "%s://%s:%s/ncamapi.html?part=%s&label=%s" % ( self.proto, self.ip, self.port, part, reader )

        pwman = urllib2.HTTPPasswordMgrWithDefaultRealm()
        pwman.add_password( None, self.url, self.username, self.password )
        handlers = urllib2.HTTPDigestAuthHandler( pwman )
        opener = urllib2.build_opener( urllib2.HTTPHandler, handlers )
        urllib2.install_opener( opener )
        request = urllib2.Request( self.url )
        err = False
        try:
            data = urllib2.urlopen( request ).read()
            # print data
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                err = str(e.reason)
            elif hasattr(e, "code"):
                err = str(e.code)
        if err is not False:
            print "[openWebIF] Fehler: %s" % err
            return False, err
        else:
            return True, data

def create_jira_issue(self, server_url, username, password, issue_summary, issue_description, project_key, issue_type='Bug'):
        """
            connect to jira server and create an issue under a specific project
        """
        status = True
        output_dict = {}
        wdesc = "Creates a JIRA issue"
        pSubStep(wdesc)
        issue_summary = issue_summary.replace('"', " ")
        issue_description = issue_description.replace('"', "-")
        fetchuri = server_url
        postdata_url=fetchuri+'/rest/api/2/issue/'
        postdata = """
        {
            "fields": {
                "project":
                {
                    "key": \""""+project_key+"""\"
                },
                "summary": \""""+issue_summary+"""\",
                "description": \""""+issue_description+"""\",
                "issuetype": {
                    "name": \""""+issue_type+"""\"
                }
            }
        }
        """
        credential_handler=urllib2.HTTPPasswordMgrWithDefaultRealm()
        credential_handler.add_password(None, postdata_url, username, password)
        auth = urllib2.HTTPBasicAuthHandler(credential_handler)
        userpassword = username + ":" + password
        password = base64.b64encode(userpassword)
        #Create an Authentication handler
        opener = urllib2.build_opener(auth)
        urllib2.install_opener(opener)
        opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1))
        #Create a POST request
        headers={"Authorization" : "Basic "+password,"Content-Type": "application/json"}
        request=urllib2.Request(str(postdata_url),postdata,headers)
        try:
            handler = urllib2.urlopen(request)
            extension = json.loads(handler.read())
            issue_id = str(extension['key'])
            pNote("JIRA Issue Created. Issue-Id: {0}".format(issue_id))
            output_dict["issue_id"] = issue_id
        except Exception as e:
            status = False
            pNote("Problem creating JIRA issue." , "error")
            pNote("JIRA Error Code: ({0})".format(e) , "error")

        Utils.data_Utils.update_datarepository(output_dict)
        Utils.testcase_Utils.report_substep_status(status)
        return status

def openWebIF(self, part = None, reader = None):
        if config.oscaminfo.userdatafromconf.getValue():
            self.ip = "127.0.0.1"
            udata = self.getUserData()
            if isinstance(udata, str):
                if "httpuser" in udata:
                    self.username=""
                elif "httppwd" in udata:
                    self.password = ""
                else:
                    return (False, udata)
            else:
                self.port = udata[2]
                self.username = udata[0]
                self.password = udata[1]
        else:
            self.ip = ".".join("%d" % d for d in config.oscaminfo.ip.getValue())
            self.port = config.oscaminfo.port.getValue()
            self.username = config.oscaminfo.username.getValue()
            self.password = config.oscaminfo.password.getValue()
        if part is None:
            self.url = "http://%s:%s/oscamapi.html?part=status" % ( self.ip, self.port )
        else:
            self.url = "http://%s:%s/oscamapi.html?part=%s" % (self.ip, self.port, part )
        if part is not None and reader is not None:
            self.url = "http://%s:%s/oscamapi.html?part=%s&label=%s" % ( self.ip, self.port, part, reader )

        print "URL=%s" % self.url
        pwman = urllib2.HTTPPasswordMgrWithDefaultRealm()
        pwman.add_password( None, self.url, self.username, self.password )
        handlers = urllib2.HTTPDigestAuthHandler( pwman )
        opener = urllib2.build_opener( urllib2.HTTPHandler, handlers )
        urllib2.install_opener( opener )
        request = urllib2.Request( self.url )
        err = False
        try:
            data = urllib2.urlopen( request ).read()
            # print data
        except urllib2.URLError, e:
            if hasattr(e, "reason"):
                err = str(e.reason)
            elif hasattr(e, "code"):
                err = str(e.code)
        if err is not False:
            print "[openWebIF] Fehler: %s" % err
            return (False, err)
        else:
            return (True, data)

def SHOWS(url):
    if __settings__.getSetting('proxy_use') == 'true':
        proxy_server = None
        proxy_type_id = 0
        proxy_port = 8080
        proxy_user = None
        proxy_pass = None
        try:
            proxy_server = __settings__.getSetting('proxy_server')
            proxy_type_id = __settings__.getSetting('proxy_type')
            proxy_port = __settings__.getSetting('proxy_port')
            proxy_user = __settings__.getSetting('proxy_user')
            proxy_pass = __settings__.getSetting('proxy_pass')
        except:
            pass
        passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
        proxy_details = 'http://' + proxy_server + ':' + proxy_port
        passmgr.add_password(None, proxy_details, proxy_user, proxy_pass) 
        authinfo = urllib2.ProxyBasicAuthHandler(passmgr)
        proxy_support = urllib2.ProxyHandler({"http" : proxy_details})

        opener = urllib2.build_opener(proxy_support, authinfo)
        urllib2.install_opener(opener)
    f = urllib2.urlopen(url)
    buf = f.read()
    buf=re.sub('&','&',buf)
    buf=re.sub('·','',buf)
    #print "BUF %s" % buf
    f.close()
    buf = buf.split('grid-list__item width--one-half width--custard--one-third')
    for p in buf:
        try:
            linkurl= re.compile('href="(.+?)"').findall (p)[0]
            #print linkurl
            image= re.compile('srcset="(.+?)"').findall (p)[0].replace('w=304&h=174','w=800&h=450')
            #print image
            name= re.compile('"tout__title complex-link__target theme__target">(.+?)</h3',re.DOTALL).findall (p)[0].strip()
            #print name
            episodes = re.compile('"tout__meta theme__meta">(.+?)</p',re.DOTALL).findall (p)[0].strip()
            #print episodes
            if 'day left' in episodes or 'days left' in episodes :
                addDir2(name+' - '+episodes,linkurl,3,'', '',image,'',isFolder=False)
            else:
                if not 'no episodes' in episodes.lower():
                    addDir(name+' - '+episodes,linkurl,2,image)
        except:pass        
    setView('tvshows', 'show')

def Send(self, uri, query_headers, query_data,ID):
        self.uri = uri
        self.query_headers = query_headers
        self.query_data = query_data
        self.ID = ID

        # Connect-timeout in seconds
        timeout = 5
        socket.setdefaulttimeout(timeout)

        url = '{}://{}{}'.format(self.proto, self.host, self.uri)

        if self.verbose:
            print "[Verbose] Sending:", url

        if self.proto == 'https':
            if hasattr(ssl, '_create_unverified_context'):
                print "[i] Creating SSL Unverified Context"
                ssl._create_default_https_context = ssl._create_unverified_context

        if self.credentials:
            Basic_Auth = self.credentials.split(':')
            if self.verbose:
                print "[Verbose] User:",Basic_Auth[0],"Password:",Basic_Auth[1]
            try:
                pwd_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
                pwd_mgr.add_password(None, url, Basic_Auth[0], Basic_Auth[1])
                auth_handler = urllib2.HTTPBasicAuthHandler(pwd_mgr)
                opener = urllib2.build_opener(auth_handler)
                urllib2.install_opener(opener)
            except Exception as e:
                print "[!] Basic Auth Error:",e
                sys.exit(1)

        if self.query_data:
            request = urllib2.Request(url, data=json.dumps(self.query_data), headers=self.query_headers)
        else:
            request = urllib2.Request(url, None, headers=self.query_headers)
        response = urllib2.urlopen(request)
#       print response
        if response:
            print "[<] {} OK".format(response.code)

        if self.Raw:
            return response
        else:
            html = response.read()
            return html

def Send(self, uri):

        # The SSI daemon are looking for this, and opens a new FD (5), but this does'nt actually
        # matter for the functionality of this exploit, only for future references.
        headers = { 
            'User-Agent' : 'MSIE',
        }

        # Connect-timeout in seconds
        timeout = 5
        socket.setdefaulttimeout(timeout)

        url = '%s://%s%s' % (self.proto, self.host, uri)

        if self.verbose:
            print "[Verbose] Sending:", url

        if self.proto == 'https':
            if hasattr(ssl, '_create_unverified_context'):
                print "[i] Creating SSL Default Context"
                ssl._create_default_https_context = ssl._create_unverified_context

        if self.credentials:
            Basic_Auth = self.credentials.split(':')
            if self.verbose:
                print "[Verbose] User:",Basic_Auth[0],"Password:",Basic_Auth[1]
            try:
                pwd_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
                pwd_mgr.add_password(None, url, Basic_Auth[0], Basic_Auth[1])
                auth_handler = urllib2.HTTPBasicAuthHandler(pwd_mgr)
                opener = urllib2.build_opener(auth_handler)
                urllib2.install_opener(opener)
            except Exception as e:
                print "[!] Basic Auth Error:",e
                sys.exit(1)

        if self.noexploit and not self.verbose:
            print "[<] 204 Not Sending!"
            html =  "Not sending any data"
        else:
            data = None
            req = urllib2.Request(url, data, headers)
            rsp = urllib2.urlopen(req)
            if rsp:
                print "[<] %s OK" % rsp.code
                html = rsp.read()
        return html