我们从Python开源项目中，提取了以下27个代码示例，用于说明如何使用urllib.request.HTTPCookieProcessor()。
def download(self, url, retry_count=3, headers=None, proxy=None, data=None): if url is None: return None try: req = request.Request(url, headers=headers, data=data) cookie = cookiejar.CookieJar() cookie_process = request.HTTPCookieProcessor(cookie) opener = request.build_opener() if proxy: proxies = {urlparse(url).scheme: proxy} opener.add_handler(request.ProxyHandler(proxies)) content = opener.open(req).read() except error.URLError as e: print('HtmlDownLoader download error:', e.reason) content = None if retry_count > 0: if hasattr(e, 'code') and 500 <= e.code < 600: #??? HTTPError ??? HTTP CODE ? 5XX ??????????????????? return self.download(url, retry_count-1, headers, proxy, data) return content
def __init__(self, timeout=None, proxy=None, cacert=None, sessions=False): if (timeout is not None) and not self.supports_feature('timeout'): raise RuntimeError('timeout is not supported with urllib2 transport') if proxy: raise RuntimeError('proxy is not supported with urllib2 transport') if cacert: raise RuntimeError('cacert is not support with urllib2 transport') handlers = [] if ((sys.version_info[0] == 2 and sys.version_info >= (2,7,9)) or (sys.version_info[0] == 3 and sys.version_info >= (3,2,0))): context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE handlers.append(urllib2.HTTPSHandler(context=context)) if sessions: handlers.append(urllib2.HTTPCookieProcessor(CookieJar())) opener = urllib2.build_opener(*handlers) self.request_opener = opener.open self._timeout = timeout
def __init__(self): self.articles = [] self.query = None self.cjar = MozillaCookieJar() # If we have a cookie file, load it: if ScholarConf.COOKIE_JAR_FILE and \ os.path.exists(ScholarConf.COOKIE_JAR_FILE): try: self.cjar.load(ScholarConf.COOKIE_JAR_FILE, ignore_discard=True) ScholarUtils.log('info', 'loaded cookies file') except Exception as msg: ScholarUtils.log('warn', 'could not load cookies file: %s' % msg) self.cjar = MozillaCookieJar() # Just to be safe self.opener = build_opener(HTTPCookieProcessor(self.cjar)) self.settings = None # Last settings object, if any
def cookie_friendly_download(referer_url, file_url, store_dir='.', timeout=1000): from http.cookiejar import CookieJar from urllib import request cj = CookieJar() cp = request.HTTPCookieProcessor(cj) opener = request.build_opener(cp) with opener.open(referer_url) as fin: fin.headers.items() import os from os import path with opener.open(file_url, timeout=timeout) as fin: file_bin = fin.read() filename = fin.headers['Content-Disposition'] filename = filename.split(';')[-1].split('=')[1] os.makedirs(store_dir, exist_ok=True) with open(path.join(store_dir, filename), mode='wb') as fout: fout.write(file_bin) return path.join(store_dir, filename)
def index(request): if request.method == "GET": try: ssl._create_default_https_context = ssl._create_unverified_context opener = wdf_urllib.build_opener( wdf_urllib.HTTPCookieProcessor(CookieJar())) wdf_urllib.install_opener(opener) except: pass uuid = getUUID() url = 'https://login.weixin.qq.com/qrcode/' + uuid params = { 't': 'webwx', '_': int(time.time()), } request = getRequest(url=url, data=urlencode(params)) response = wdf_urllib.urlopen(request) context = { 'uuid': uuid, 'response': response.read(), 'delyou': '', } return render_to_response('index.html', context)
def __init__(self): self.articles = [] self.query = None self.cjar = MozillaCookieJar() # If we have a cookie file, load it: if ScholarConf.COOKIE_JAR_FILE and \ os.path.exists(ScholarConf.COOKIE_JAR_FILE): try: self.cjar.load(ScholarConf.COOKIE_JAR_FILE, ignore_discard=True) print "Using cookie file" ScholarUtils.log('info', 'loaded cookies file') except Exception as msg: print "Ignoring cookie file: %s" % msg ScholarUtils.log('warn', 'could not load cookies file: %s' % msg) self.cjar = MozillaCookieJar() # Just to be safe self.opener = build_opener(HTTPCookieProcessor(self.cjar)) self.settings = None # Last settings object, if any
def __get_cookies(self, req): cookies = cookiejar.CookieJar() handler = request.HTTPCookieProcessor(cookies) opener = request.build_opener(handler) try: with opener.open(req) as f: if f.code == 200: pattern = re.compile(r"<input.*?type='hidden'.*?name='csrfmiddlewaretoken'.*?value='(.*?)'.*>") try: self.csrfmiddlewaretoken = pattern.search(f.read().decode("utf-8")).group(1) print("Achieved cookies and csrfmiddlewaretoken sucessfully") except: print("Achieved cookies sucessfully") return cookies else: print("Lost cookies") except error.URLError as e: if hasattr(e, "reason"): print ("We failed to reach a server. Please check your url and read the Reason") print ("Reason: {}".format(e.reason)) elif hasattr(e, "code"): print("The server couldn't fulfill the request.") print("Error code: {}".format(e.code)) exit()
def login(self, username, pwd, cookie_file): """" Login with use name, password and cookies. (1) If cookie file exists then try to load cookies; (2) If no cookies found then do login """ # If cookie file exists then try to load cookies if os.path.exists(cookie_file): try: cookie_jar = cookielib.LWPCookieJar(cookie_file) cookie_jar.load(ignore_discard=True, ignore_expires=True) loaded = 1 except cookielib.LoadError: loaded = 0 LOG.info('Loading cookies error') # install loaded cookies for urllib2 if loaded: cookie_support = urllib2.HTTPCookieProcessor(cookie_jar) opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler) urllib2.install_opener(opener) LOG.info('Loading cookies success') return 1 else: return self.do_login(username, pwd, cookie_file) else: # If no cookies found return self.do_login(username, pwd, cookie_file)
def save_cookie(self, text, cookie_file=CONF.cookie_file): cookie_jar2 = cookielib.LWPCookieJar() cookie_support2 = urllib2.HTTPCookieProcessor(cookie_jar2) opener2 = urllib2.build_opener(cookie_support2, urllib2.HTTPHandler) urllib2.install_opener(opener2) if six.PY3: text = text.decode('gbk') p = re.compile('location\.replace\(\'(.*?)\'\)') # ???httpfox?????????????? # location.replace('http://weibo.com ????????? # ?????????????# ????login_url?? ??????re????? # p = re.compile('location\.replace\(\B'(.*?)'\B\)') # ??? ??????? re?????\'??????? try: # Search login redirection URL login_url = p.search(text).group(1) data = urllib2.urlopen(login_url).read() # Verify login feedback, check whether result is TRUE patt_feedback = 'feedBackUrlCallBack\((.*)\)' p = re.compile(patt_feedback, re.MULTILINE) feedback = p.search(data).group(1) feedback_json = json.loads(feedback) if feedback_json['result']: cookie_jar2.save(cookie_file, ignore_discard=True, ignore_expires=True) return 1 else: return 0 except: return 0
def login(self, username, pwd, cookie_file): """" Login with use name, password and cookies. (1) If cookie file exists then try to load cookies; (2) If no cookies found then do login """ # If cookie file exists then try to load cookies if os.path.exists(cookie_file): try: cookie_jar = cookielib.LWPCookieJar(cookie_file) cookie_jar.load(ignore_discard=True, ignore_expires=True) loaded = 1 except cookielib.LoadError: loaded = 0 print('Loading cookies error') #install loaded cookies for urllib2 if loaded: cookie_support = urllib2.HTTPCookieProcessor(cookie_jar) opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler) urllib2.install_opener(opener) print('Loading cookies success') return 1 else: return self.do_login(username, pwd, cookie_file) else: #If no cookies found return self.do_login(username, pwd, cookie_file)
def build_opener(): cookie = http.cookiejar.CookieJar() cookie_processor = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_processor) opener.addheaders = [("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"), ("Referer", "https://passport.weibo.cn"), ("Origin", "https://passport.weibo.cn"), ("Host", "passport.weibo.cn")] request.install_opener(opener) #??
def build_opener(): cookie = http.cookiejar.CookieJar() cookie_processor = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_processor) opener.addheaders = [("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0"), ("Referer", "http://cn.v2ex.com/signin"), ("Origin", "http://cn.v2ex.com"), ("Host", "cn.v2ex.com")] request.install_opener(opener)
def build_opener(): cookie = http.cookiejar.CookieJar() cookie_processor = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_processor) opener.addheaders = [("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36"), ("Referer", "https://wx.qq.com/"), ("Origin", "https://wx.qq.com/"), ("Host", "wx.qq.com")] request.install_opener(opener) #??uuid
def build_opener(): cookie = http.cookiejar.CookieJar() cookie_processor = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_processor) opener.addheaders = [("User-Agent", "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1"), ("Referer", "https://www.zhihu.com/"), ("Origin", "https://www.zhihu.com/"), ("Host", "www.zhihu.com")] request.install_opener(opener)
def create_cookie_opener(self): ''' ????Cookie :return: ????????opener ''' cookie = cookiejar.CookieJar() cookie_process = request.HTTPCookieProcessor(cookie) opener = request.build_opener(cookie_process) return opener
def __init__(self): self.cj = MozillaCookieJar() self.opener = request.build_opener( request.HTTPCookieProcessor(self.cj)) self.threadPool = Pool(processes=10)
def download_image(url, output_dir, image, creds): cookies = HTTPCookieProcessor() opener = build_opener(cookies) install_opener(opener) data = urlopen("https://ers.cr.usgs.gov").read().decode('utf-8') m = re.search(r'<input .*?name="csrf_token".*?value="(.*?)"', data) if m: token = m.group(1) else: print("Error : CSRF_Token not found") params = urlencode(dict(username=creds['account'], password=creds['passwd'], csrf_token=token)) params = params.encode('ascii') request = Request("https://ers.cr.usgs.gov/login", params, headers={}) f = urlopen(request) data = f.read().decode('utf-8') f.close() if data.find('You must sign in as a registered user to download data or place orders for USGS EROS products') > 0: print("Authentification failed") req = urlopen(url) uri = req.url response = requests.get(uri) if response.status_code == 200: with open(os.path.join(output_dir, image), 'wb') as f: for chunk in response.iter_content(chunk_size=1024 * 1024 * 8): f.write(chunk) elif response.status_code > 399: print('Code {}'.format(response.status_code)) raise BadRequestsResponse(Exception)
def __init__(self): self.cookiejar = CookieJar() self._cookie_processor = HTTPCookieProcessor(self.cookiejar) self.form = None self.url = "http://0.0.0.0:8080/" self.path = "/" self.status = None self.data = None self._response = None self._forms = None
def request(url, data={}, headers={}, timeout=10): """ Returns a file-like object to the given URL. """ if cookies is not None: f = urllib2.HTTPCookieProcessor(cookies) f = urllib2.build_opener(f) else: f = urllib2.build_opener() try: f = f.open(Request(url, urlencode(data) if data else None, headers), timeout=timeout) except URLError as e: status = getattr(e, 'code', None) # HTTPError if status == 401: raise Forbidden if status == 403: raise Forbidden if status == 404: raise NotFound if status == 420: raise TooManyRequests if status == 429: raise TooManyRequests raise e except socket.error as e: if 'timed out' in repr(e.args): raise Timeout else: raise e log.info(url) return f
def __init__(self, username="admin", password="admin", hostname="192.168.1.1"): self.username = username self.password = password self.hostname = hostname self.baseurl = "https://" + self.hostname try: self.gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) except AttributeError: try: self.gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) except AttributeError: self.gcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1) self.cj = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cj), HTTPSHandler(context=self.gcontext)) self.loginresponse = self._login()
def __init__(self): super(WebService, self).__init__() self.cache = defaultdict(defaultdict) self._cookie = CookieJar() self._opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self._cookie)) self.query_interval = 1
def __init__(self, begin_date_str): self.__base_url = "http://xk.autoisp.shu.edu.cn" monday = datetime.strptime(begin_date_str, '%Y.%m.%d').date() tuesday = monday + timedelta(days=1) wednesday = tuesday + timedelta(days=1) thursday = wednesday + timedelta(days=1) friday = thursday + timedelta(days=1) self.__weekday_table = {'?': monday, '?': tuesday, '?': wednesday, '?': thursday, '?': friday} self.__course_time_table = (time(hour=8, minute=0, tzinfo=timezone("Asia/Shanghai")), time(hour=8, minute=55, tzinfo=timezone("Asia/Shanghai")), time(hour=10, minute=0, tzinfo=timezone("Asia/Shanghai")), time(hour=10, minute=55, tzinfo=timezone("Asia/Shanghai")), time(hour=12, minute=10, tzinfo=timezone("Asia/Shanghai")), time(hour=13, minute=5, tzinfo=timezone("Asia/Shanghai")), time(hour=14, minute=10, tzinfo=timezone("Asia/Shanghai")), time(hour=15, minute=5, tzinfo=timezone("Asia/Shanghai")), time(hour=16, minute=0, tzinfo=timezone("Asia/Shanghai")), time(hour=16, minute=55, tzinfo=timezone("Asia/Shanghai")), time(hour=18, minute=0, tzinfo=timezone("Asia/Shanghai")), time(hour=18, minute=55, tzinfo=timezone("Asia/Shanghai")), time(hour=19, minute=50, tzinfo=timezone("Asia/Shanghai"))) # Create a opener with the ability to record cookies. self.__opener = build_opener(HTTPCookieProcessor(CookieJar())) self.__validate_img_path = os.path.join(os.getcwd(), "validate_code.jpg") self.term_index = 0 self.terms_and_ports = [] ports = (80, 8080) for port in ports: url = '%s:%d' % (self.__base_url, port) data = self.__opener.open(url).read().decode('utf-8') term = re.search(pattern=r'<div style="color: Red; font-size: 26px; text-align: center;">(.+?)</div>', string=data).group(1) self.terms_and_ports.append((term, port))
def __init__ (self, args) : self.modulus = None self.exponent = None self.args = args self.jar = j = LWPCookieJar () self.has_cookies = False if self.args.cookiefile : self.has_cookies = True try : j.load (self.args.cookiefile, ignore_discard = True) except IOError : self.has_cookies = False self.opener = build_opener (HTTPCookieProcessor (j)) self.nextfile = args.file # end def __init__
