我们从Python开源项目中，提取了以下25个代码示例，用于说明如何使用pycurl.PROXY。
def request(self, endpoint, post=None): buffer = BytesIO() ch = pycurl.Curl() ch.setopt(pycurl.URL, Constants.API_URL + endpoint) ch.setopt(pycurl.USERAGENT, self.userAgent) ch.setopt(pycurl.WRITEFUNCTION, buffer.write) ch.setopt(pycurl.FOLLOWLOCATION, True) ch.setopt(pycurl.HEADER, True) ch.setopt(pycurl.VERBOSE, False) ch.setopt(pycurl.COOKIEFILE, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat")) ch.setopt(pycurl.COOKIEJAR, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat")) if post is not None: ch.setopt(pycurl.POST, True) ch.setopt(pycurl.POSTFIELDS, post) if self.proxy: ch.setopt(pycurl.PROXY, self.proxyHost) if self.proxyAuth: ch.setopt(pycurl.PROXYUSERPWD, self.proxyAuth) ch.perform() resp = buffer.getvalue() header_len = ch.getinfo(pycurl.HEADER_SIZE) header = resp[0: header_len] body = resp[header_len:] ch.close() if self.debug: print("REQUEST: " + endpoint) if post is not None: if not isinstance(post, list): print("DATA: " + str(post)) print("RESPONSE: " + body) return [header, json_decode(body)]
def _set_proxy(self, c, freq): ip, port, ptype = self._proxies.next() freq.wf_proxy = (("%s:%s" % (ip, port)), ptype) c.setopt(pycurl.PROXY, "%s:%s" % (ip, port)) if ptype == "SOCKS5": c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5) elif ptype == "SOCKS4": c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4) elif ptype == "HTML": pass else: raise FuzzException(FuzzException.FATAL, "Bad proxy type specified, correct values are HTML, SOCKS4 or SOCKS5.") return c
def request(self, url, method, body, headers): c = pycurl.Curl() c.setopt(pycurl.URL, url) if 'proxy_host' in self.proxy: c.setopt(pycurl.PROXY, self.proxy['proxy_host']) if 'proxy_port' in self.proxy: c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port']) if 'proxy_user' in self.proxy: c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy) self.buf = StringIO() c.setopt(pycurl.WRITEFUNCTION, self.buf.write) #c.setopt(pycurl.READFUNCTION, self.read) #self.body = StringIO(body) #c.setopt(pycurl.HEADERFUNCTION, self.header) if self.cacert: c.setopt(c.CAINFO, self.cacert) c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0) c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0) c.setopt(pycurl.CONNECTTIMEOUT, self.timeout) c.setopt(pycurl.TIMEOUT, self.timeout) if method == 'POST': c.setopt(pycurl.POST, 1) c.setopt(pycurl.POSTFIELDS, body) if headers: hdrs = ['%s: %s' % (k, v) for k, v in headers.items()] log.debug(hdrs) c.setopt(pycurl.HTTPHEADER, hdrs) c.perform() c.close() return {}, self.buf.getvalue()
def setUrl (self, urltmp): if not (urltmp.startswith("http://") or urltmp.startswith("https://")): urltmp = "http://" + urltmp self.__variablesGET=VariablesSet() self.schema,self.__host,self.__path,self.__params,variables,f=urlparse(urltmp) self._headers["Host"]=self.__host if variables: self.__variablesGET.parseUrlEncoded(variables) ############### PROXY ##################################
def request(self, url, method, body, headers): c = pycurl.Curl() c.setopt(pycurl.URL, url) if 'proxy_host' in self.proxy: c.setopt(pycurl.PROXY, self.proxy['proxy_host']) if 'proxy_port' in self.proxy: c.setopt(pycurl.PROXYPORT, self.proxy['proxy_port']) if 'proxy_user' in self.proxy: c.setopt(pycurl.PROXYUSERPWD, "%(proxy_user)s:%(proxy_pass)s" % self.proxy) self.buf = StringIO() c.setopt(pycurl.WRITEFUNCTION, self.buf.write) #c.setopt(pycurl.READFUNCTION, self.read) #self.body = StringIO(body) #c.setopt(pycurl.HEADERFUNCTION, self.header) if self.cacert: c.setopt(c.CAINFO, self.cacert) c.setopt(pycurl.SSL_VERIFYPEER, self.cacert and 1 or 0) c.setopt(pycurl.SSL_VERIFYHOST, self.cacert and 2 or 0) c.setopt(pycurl.CONNECTTIMEOUT, self.timeout / 6) c.setopt(pycurl.TIMEOUT, self.timeout) if method == 'POST': c.setopt(pycurl.POST, 1) c.setopt(pycurl.POSTFIELDS, body) if headers: hdrs = ['%s: %s' % (k, v) for k, v in headers.items()] log.debug(hdrs) c.setopt(pycurl.HTTPHEADER, hdrs) c.perform() c.close() return {}, self.buf.getvalue()
def test_pycurl_proxy(self): """If provided, the proxy is set in the request.""" curl = CurlStub(b"result") proxy = "http://my.little.proxy" result = fetch("http://example.com", curl=curl, proxy=proxy) self.assertEqual(b"result", result) self.assertEqual(proxy.encode('ascii'), curl.options[pycurl.PROXY])
def setInterface(self, options): interface, proxy, ipv6 = options["interface"], options["proxies"], options["ipv6"] if interface and interface.lower() != "none": self.c.setopt(pycurl.INTERFACE, str(interface)) if proxy: if proxy["type"] == "socks4": self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4) elif proxy["type"] == "socks5": self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5) else: self.c.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) self.c.setopt(pycurl.PROXY, str(proxy["address"])) self.c.setopt(pycurl.PROXYPORT, proxy["port"]) if proxy["username"]: self.c.setopt(pycurl.PROXYUSERPWD, str("%s:%s" % (proxy["username"], proxy["password"]))) if ipv6: self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) else: self.c.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) if "auth" in options: self.c.setopt(pycurl.USERPWD, str(options["auth"])) if "timeout" in options: self.c.setopt(pycurl.LOW_SPEED_TIME, options["timeout"])
def getKeyword(i):#??json try: time.sleep(1) headers = [ 'Host:fengchao.baidu.com', 'User-Agent: %s' %getUA(), 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3', 'Accept-Encoding: gzip, deflate', 'Referer: http://fengchao.baidu.com/nirvana/main.html?userid=8048066', 'Connection: keep-alive', 'COOKIE:%s' %COOKIE, ] post = urllib.urlencode({ 'params': '{"entry":"kr_station","query":"%s","querytype":1,"pageNo":1,"pageSize":300}' % keyword_list[i], 'path': 'jupiter/GET/kr/word', 'token': TOKEN, 'userid': USERID, }) url = 'http://fengchao.baidu.com/nirvana/request.ajax?path=jupiter/GET/kr/word' c = pycurl.Curl() # c.setopt(pycurl.PROXY, getRandomAlbIp()) c.setopt(pycurl.URL, url) c.setopt(pycurl.FOLLOWLOCATION, True) c.setopt(pycurl.MAXREDIRS,5) c.setopt(pycurl.CONNECTTIMEOUT, 20) c.setopt(pycurl.TIMEOUT,20) c.setopt(pycurl.ENCODING, 'gzip,deflate') c.fp = StringIO.StringIO() c.setopt(pycurl.HTTPHEADER,headers) c.setopt(pycurl.POST, 1) c.setopt(pycurl.POSTFIELDS, post) c.setopt(c.WRITEFUNCTION, c.fp.write) c.perform() # mutex.acquire()#?? jsonData = c.fp.getvalue() analyseJsonData(i,jsonData) # mutex.release()#?? except Exception,e: print e pass
def set_interface(self, options): interface, proxy, ipv6 = options[ 'interface'], options['proxies'], options['ipv6'] if interface and interface.lower() != "none": self.setopt(pycurl.INTERFACE, interface) if proxy: if proxy['type'] == "socks4": self.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS4) elif proxy['type'] == "socks5": self.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5) else: self.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_HTTP) self.setopt(pycurl.PROXY, proxy['host']) self.setopt(pycurl.PROXYPORT, proxy['port']) if proxy['username']: userpwd = "{0}:{1}".format( proxy['username'], proxy['password']) self.setopt(pycurl.PROXYUSERPWD, userpwd) if ipv6: self.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER) else: self.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) if "timeout" in options: self.setopt(pycurl.LOW_SPEED_TIME, options['timeout']) if "auth" in options: self.setopt(pycurl.USERPWD, self.options['auth'])
def query(url): output = io.BytesIO() query = pycurl.Curl() query.setopt(pycurl.URL, url) query.setopt(pycurl.HTTPHEADER, getHeaders()) query.setopt(pycurl.PROXY, 'localhost') query.setopt(pycurl.PROXYPORT, SOCKS_PORT) query.setopt(pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME) query.setopt(pycurl.WRITEFUNCTION, output.write) through = False while not through: try: query.perform() http_code = query.getinfo(pycurl.HTTP_CODE) if http_code == 200: through = True else: # renew tor to retry print 'error httpcode:' +str(http_code) renew_tor() # time.sleep(3) except pycurl.error as exc: print "pycurl error in tor.py %s" % exc # return "Unable to reach %s (%s)" % (url, exc) return output.getvalue()
def _do_write(self, query): # send provision query to server if not self._write_url. # after send provision query, set self._write_url. # send query to server, return JSON rethinker = doublethink.Rethinker(db="trough_configuration", servers=self.rethinkdb) services = doublethink.ServiceRegistry(rethinker) master_node = services.unique_service('trough-sync-master') logging.info('master_node=%r', master_node) if not master_node: raise Exception('no healthy trough-sync-master in service registry') if not self._write_url: buffer = BytesIO() c = pycurl.Curl() c.setopt(c.URL, master_node.get('url')) c.setopt(c.POSTFIELDS, self.database) if self.proxy: c.setopt(pycurl.PROXY, self.proxy) c.setopt(pycurl.PROXYPORT, int(self.proxy_port)) c.setopt(pycurl.PROXYTYPE, self.proxy_type) c.setopt(c.WRITEDATA, buffer) c.perform() c.close() self._write_url = buffer.getvalue() logging.info('self._write_url=%r', self._write_url) buffer = BytesIO() c = pycurl.Curl() c.setopt(c.URL, self._write_url) c.setopt(c.POSTFIELDS, query) if self.proxy: c.setopt(pycurl.PROXY, self.proxy) c.setopt(pycurl.PROXYPORT, int(self.proxy_port)) c.setopt(pycurl.PROXYTYPE, self.proxy_type) c.setopt(c.WRITEDATA, buffer) c.perform() c.close() response = buffer.getvalue() if response.strip() != b'OK': raise Exception('Trough Query Failed: Database: %r Response: %r Query: %.200r' % (self.database, response, query)) self._last_results = None
def pideURL(self,url,compressed = False, cookie=False, contador_curl = 0): time.sleep(3) Scrape.contador+=1 print ("\n"+url) print ("\n\t.l."+str(Scrape.contador)) c = pycurl.Curl() if cookie: c.setopt(pycurl.COOKIEJAR, 'cookie.txt') c.setopt(pycurl.COOKIEFILE, 'cookie.txt') c.setopt(pycurl.URL, url) c.setopt(pycurl.CONNECTTIMEOUT, 15) c.setopt(pycurl.TIMEOUT, 25) c.setopt(pycurl.HTTPHEADER, self.headers) c.setopt( pycurl.PROXY, '127.0.0.1' ) c.setopt( pycurl.PROXYPORT, 9050 ) c.setopt( pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME ) b = BytesIO() BytesIO c.setopt(pycurl.WRITEFUNCTION, b.write) self.url = url try: c.perform() self.response_string = b.getvalue() #print (self.response_string) b.close() except Exception as e: #self.log ('Razon:',e) self.response_string = None if contador_curl<=10: time.sleep(5) self.pideURL(url,contador_curl+1) else: print ('Error: ',url) print ('Error log: ',e)
def pidePOST(self,url,data,compressed = False,cookie=False, contador_curl = 0, debug=False): time.sleep(3) Scrape.contador+=1 print ("\n"+url) print ("\n\t.l."+str(Scrape.contador)) c = pycurl.Curl() if cookie: c.setopt(pycurl.COOKIEJAR, 'cookie.txt') c.setopt(pycurl.COOKIEFILE, 'cookie.txt') c.setopt(pycurl.URL, url) c.setopt(pycurl.CONNECTTIMEOUT, 15) c.setopt(pycurl.TIMEOUT, 25) c.setopt(pycurl.HTTPHEADER, self.headers) if compressed: c.setopt(pycurl.ENCODING, 'gzip,deflate') c.setopt(c.POSTFIELDS, data) if debug: c.setopt(c.VERBOSE, True) c.setopt( pycurl.PROXY, '127.0.0.1' ) c.setopt( pycurl.PROXYPORT, 9050 ) c.setopt( pycurl.PROXYTYPE, pycurl.PROXYTYPE_SOCKS5_HOSTNAME ) b = BytesIO() BytesIO c.setopt(pycurl.WRITEFUNCTION, b.write) self.url = url try: c.perform() self.response_string = b.getvalue() #print (self.response_string) b.close() except Exception as e: #print ('Razon:',e) self.response_string = None
def perform(self): self.__performHead="" self.__performBody="" conn=pycurl.Curl() conn.setopt(pycurl.SSL_VERIFYPEER,False) conn.setopt(pycurl.SSL_VERIFYHOST,1) conn.setopt(pycurl.URL,self.completeUrl) if self.__method or self.__userpass: if self.__method=="basic": conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) elif self.__method=="ntlm": conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_NTLM) elif self.__method=="digest": conn.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) conn.setopt(pycurl.USERPWD, self.__userpass) if self.__timeout: conn.setopt(pycurl.CONNECTTIMEOUT, self.__timeout) conn.setopt(pycurl.NOSIGNAL, 1) if self.__totaltimeout: conn.setopt(pycurl.TIMEOUT, self.__totaltimeout) conn.setopt(pycurl.NOSIGNAL, 1) conn.setopt(pycurl.WRITEFUNCTION, self.body_callback) conn.setopt(pycurl.HEADERFUNCTION, self.header_callback) if self.__proxy!=None: conn.setopt(pycurl.PROXY,self.__proxy) if self.__headers.has_key("Proxy-Connection"): del self.__headers["Proxy-Connection"] conn.setopt(pycurl.HTTPHEADER,self.__getHeaders()) if self.method=="POST": conn.setopt(pycurl.POSTFIELDS,self.__postdata) conn.perform() rp=Response() rp.parseResponse(self.__performHead) rp.addContent(self.__performBody) self.response=rp ######### ESTE conjunto de funciones no es necesario para el uso habitual de la clase
def connect_tor_http(controller, circuit_path, job, conn_timeout, curlopts=None, curlinfos=None): """ This helper function will perform an HTTP request over Tor. It will not perform any special action in the event that this is the experimental flow, but can be customised on a per-call basis through the curlopts argument. """ if curlopts is None: curlopts = {} curlopts[pycurl.PROXY] = "localhost" curlopts[pycurl.PROXYPORT] = 9050 curlopts[pycurl.PROXYTYPE] = pycurl.PROXYTYPE_SOCKS5_HOSTNAME attach_error = [] try: if circuit_path is not None: circuit_path = circuit_path.split(",") circuit_id = controller.new_circuit(circuit_path, await_build=True) except stem.CircuitExtensionFailed: return {"spdr_state": CONN_DISCARD} def attach_stream(stream): try: if stream.status == 'NEW': if (stream.target_address == job['dip'] and stream.target_port == job['dp']): controller.attach_stream(stream.id, circuit_id) except stem.OperationFailed: attach_error.append(None) controller.add_event_listener(attach_stream, stem.control.EventType.STREAM) # pylint: disable=no-member result = connect_http(None, job, conn_timeout, curlopts, curlinfos) controller.remove_event_listener(attach_stream) if len(attach_error) > 0: return {"spdr_state": CONN_DISCARD} return result
def to_pycurl_object(c, req): c.setopt(pycurl.MAXREDIRS, 5) c.setopt(pycurl.WRITEFUNCTION, req.body_callback) c.setopt(pycurl.HEADERFUNCTION, req.header_callback) c.setopt(pycurl.NOSIGNAL, 1) c.setopt(pycurl.SSL_VERIFYPEER, False) c.setopt(pycurl.SSL_VERIFYHOST, 0) c.setopt(pycurl.URL,req.completeUrl) if req.getConnTimeout(): c.setopt(pycurl.CONNECTTIMEOUT, req.getConnTimeout()) if req.getTotalTimeout(): c.setopt(pycurl.TIMEOUT, req.getTotalTimeout()) authMethod, userpass = req.getAuth() if authMethod or userpass: if authMethod == "basic": c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) elif authMethod == "ntlm": c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_NTLM) elif authMethod == "digest": c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST) c.setopt(pycurl.USERPWD, userpass) c.setopt(pycurl.HTTPHEADER, req.getHeaders()) if req.method == "POST": c.setopt(pycurl.POSTFIELDS, req.postdata) if req.method != "GET" and req.method != "POST": c.setopt(pycurl.CUSTOMREQUEST, req.method) if req.method == "HEAD": c.setopt(pycurl.NOBODY, True) if req.followLocation: c.setopt(pycurl.FOLLOWLOCATION, 1) proxy = req.getProxy() if proxy != None: c.setopt(pycurl.PROXY, proxy) if req.proxytype=="SOCKS5": c.setopt(pycurl.PROXYTYPE,pycurl.PROXYTYPE_SOCKS5) elif req.proxytype=="SOCKS4": c.setopt(pycurl.PROXYTYPE,pycurl.PROXYTYPE_SOCKS4) req.delHeader("Proxy-Connection") return c
def request(self, method, url, headers, post_data=None): s = util.StringIO.StringIO() rheaders = util.StringIO.StringIO() curl = pycurl.Curl() proxy = self._get_proxy(url) if proxy: if proxy.hostname: curl.setopt(pycurl.PROXY, proxy.hostname) if proxy.port: curl.setopt(pycurl.PROXYPORT, proxy.port) if proxy.username or proxy.password: curl.setopt( pycurl.PROXYUSERPWD, "%s:%s" % (proxy.username, proxy.password)) if method == 'get': curl.setopt(pycurl.HTTPGET, 1) elif method == 'post': curl.setopt(pycurl.POST, 1) curl.setopt(pycurl.POSTFIELDS, post_data) else: curl.setopt(pycurl.CUSTOMREQUEST, method.upper()) # pycurl doesn't like unicode URLs curl.setopt(pycurl.URL, util.utf8(url)) curl.setopt(pycurl.WRITEFUNCTION, s.write) curl.setopt(pycurl.HEADERFUNCTION, rheaders.write) curl.setopt(pycurl.NOSIGNAL, 1) curl.setopt(pycurl.CONNECTTIMEOUT, 30) curl.setopt(pycurl.TIMEOUT, 80) curl.setopt(pycurl.HTTPHEADER, ['%s: %s' % (k, v) for k, v in headers.items()]) if self._verify_ssl_certs: curl.setopt(pycurl.CAINFO, os.path.join( os.path.dirname(__file__), 'data/ca-certificates.crt')) else: curl.setopt(pycurl.SSL_VERIFYHOST, False) try: curl.perform() except pycurl.error as e: self._handle_request_error(e) rbody = s.getvalue() rcode = curl.getinfo(pycurl.RESPONSE_CODE) return rbody, rcode, self.parse_headers(rheaders.getvalue())
