Python pycurl 模块，HEADER_SIZE 实例源码

def request(self, endpoint, post=None):
        buffer = BytesIO()

        ch = pycurl.Curl()
        ch.setopt(pycurl.URL, Constants.API_URL + endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        ch.setopt(pycurl.VERBOSE, False)
        ch.setopt(pycurl.COOKIEFILE, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat"))
        ch.setopt(pycurl.COOKIEJAR, os.path.join(self.IGDataPath, self.username, self.username + "-cookies.dat"))

        if post is not None:
            ch.setopt(pycurl.POST, True)
            ch.setopt(pycurl.POSTFIELDS, post)

        if self.proxy:
            ch.setopt(pycurl.PROXY, self.proxyHost)
            if self.proxyAuth:
                ch.setopt(pycurl.PROXYUSERPWD, self.proxyAuth)

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]

        ch.close()

        if self.debug:
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print("DATA: " + str(post))
            print("RESPONSE: " + body)

        return [header, json_decode(body)]

def request(self, endpoint, headers=None, post=None, first=True):
        buffer = BytesIO()

        ch = pycurl.Curl()

        ch.setopt(pycurl.URL, endpoint)
        ch.setopt(pycurl.USERAGENT, self.userAgent)
        ch.setopt(pycurl.WRITEFUNCTION, buffer.write)
        ch.setopt(pycurl.FOLLOWLOCATION, True)
        ch.setopt(pycurl.HEADER, True)
        if headers:
            ch.setopt(pycurl.HTTPHEADER, headers)

        ch.setopt(pycurl.VERBOSE, self.debug)
        ch.setopt(pycurl.SSL_VERIFYPEER, False)
        ch.setopt(pycurl.SSL_VERIFYHOST, False)
        ch.setopt(pycurl.COOKIEFILE, self.settingsPath + self.username + '-cookies.dat')
        ch.setopt(pycurl.COOKIEJAR, self.settingsPath + self.username + '-cookies.dat')

        if post:
            import urllib
            ch.setopt(pycurl.POST, len(post))
            ch.setopt(pycurl.POSTFIELDS, urllib.urlencode(post))

        ch.perform()
        resp = buffer.getvalue()
        header_len = ch.getinfo(pycurl.HEADER_SIZE)
        header = resp[0: header_len]
        body = resp[header_len:]
        ch.close()

        if self.debug:
            import urllib
            print("REQUEST: " + endpoint)
            if post is not None:
                if not isinstance(post, list):
                    print('DATA: ' + urllib.unquote_plus(json.dumps(post)))
            print("RESPONSE: " + body + "\n")

        return [header, json_decode(body)]

def getPage (self, url, requestHeader = []) :
        resultFormate = StringIO.StringIO()

        fakeIp = self.fakeIp()
        requestHeader.append('CLIENT-IP:' + fakeIp)
        requestHeader.append('X-FORWARDED-FOR:' + fakeIp)

        try:
            curl = pycurl.Curl()
            curl.setopt(pycurl.URL, url.strip())
            curl.setopt(pycurl.ENCODING, 'gzip,deflate')
            curl.setopt(pycurl.HEADER, 1)
            curl.setopt(pycurl.TIMEOUT, 120)
            curl.setopt(pycurl.SSL_VERIFYPEER, 0)   
            curl.setopt(pycurl.SSL_VERIFYHOST, 0)
            curl.setopt(pycurl.HTTPHEADER, requestHeader)
            curl.setopt(pycurl.WRITEFUNCTION, resultFormate.write)
            curl.perform()
            headerSize = curl.getinfo(pycurl.HEADER_SIZE)
            curl.close()

            header = resultFormate.getvalue()[0 : headerSize].split('\r\n')
            body = resultFormate.getvalue()[headerSize : ]
        except Exception, e:
            header = ''
            body = ''

        return header, body

def info(c):
    "Return a dictionary with all info on the last response."
    m = {}
    m['effective-url'] = c.getinfo(pycurl.EFFECTIVE_URL)
    m['http-code'] = c.getinfo(pycurl.HTTP_CODE)
    m['total-time'] = c.getinfo(pycurl.TOTAL_TIME)
    m['namelookup-time'] = c.getinfo(pycurl.NAMELOOKUP_TIME)
    m['connect-time'] = c.getinfo(pycurl.CONNECT_TIME)
    m['pretransfer-time'] = c.getinfo(pycurl.PRETRANSFER_TIME)
    m['redirect-time'] = c.getinfo(pycurl.REDIRECT_TIME)
    m['redirect-count'] = c.getinfo(pycurl.REDIRECT_COUNT)
    # m['size-upload'] = c.getinfo(pycurl.SIZE_UPLOAD)
    m['size-download'] = c.getinfo(pycurl.SIZE_DOWNLOAD)
    # m['speed-upload'] = c.getinfo(pycurl.SPEED_UPLOAD)
    m['header-size'] = c.getinfo(pycurl.HEADER_SIZE)
    m['request-size'] = c.getinfo(pycurl.REQUEST_SIZE)
    m['content-length-download'] = c.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
    m['content-length-upload'] = c.getinfo(pycurl.CONTENT_LENGTH_UPLOAD)
    m['content-type'] = c.getinfo(pycurl.CONTENT_TYPE)
    m['response-code'] = c.getinfo(pycurl.RESPONSE_CODE)
    m['speed-download'] = c.getinfo(pycurl.SPEED_DOWNLOAD)
    # m['ssl-verifyresult'] = c.getinfo(pycurl.SSL_VERIFYRESULT)
    m['filetime'] = c.getinfo(pycurl.INFO_FILETIME)
    m['starttransfer-time'] = c.getinfo(pycurl.STARTTRANSFER_TIME)
    m['redirect-time'] = c.getinfo(pycurl.REDIRECT_TIME)
    m['redirect-count'] = c.getinfo(pycurl.REDIRECT_COUNT)
    m['http-connectcode'] = c.getinfo(pycurl.HTTP_CONNECTCODE)
    # m['httpauth-avail'] = c.getinfo(pycurl.HTTPAUTH_AVAIL)
    # m['proxyauth-avail'] = c.getinfo(pycurl.PROXYAUTH_AVAIL)
    # m['os-errno'] = c.getinfo(pycurl.OS_ERRNO)
    m['num-connects'] = c.getinfo(pycurl.NUM_CONNECTS)
    # m['ssl-engines'] = c.getinfo(pycurl.SSL_ENGINES)
    # m['cookielist'] = c.getinfo(pycurl.INFO_COOKIELIST)
    # m['lastsocket'] = c.getinfo(pycurl.LASTSOCKET)
    # m['ftp-entry-path'] = c.getinfo(pycurl.FTP_ENTRY_PATH)
    return m

def info(self):
        "Return a dictionary with all info on the last response."
        m = {}
        m['effective-url'] = self.handle.getinfo(pycurl.EFFECTIVE_URL)
        m['http-code'] = self.handle.getinfo(pycurl.HTTP_CODE)
        m['total-time'] = self.handle.getinfo(pycurl.TOTAL_TIME)
        m['namelookup-time'] = self.handle.getinfo(pycurl.NAMELOOKUP_TIME)
        m['connect-time'] = self.handle.getinfo(pycurl.CONNECT_TIME)
        m['pretransfer-time'] = self.handle.getinfo(pycurl.PRETRANSFER_TIME)
        m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME)
        m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT)
        m['size-upload'] = self.handle.getinfo(pycurl.SIZE_UPLOAD)
        m['size-download'] = self.handle.getinfo(pycurl.SIZE_DOWNLOAD)
        m['speed-upload'] = self.handle.getinfo(pycurl.SPEED_UPLOAD)
        m['header-size'] = self.handle.getinfo(pycurl.HEADER_SIZE)
        m['request-size'] = self.handle.getinfo(pycurl.REQUEST_SIZE)
        m['content-length-download'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_DOWNLOAD)
        m['content-length-upload'] = self.handle.getinfo(pycurl.CONTENT_LENGTH_UPLOAD)
        m['content-type'] = self.handle.getinfo(pycurl.CONTENT_TYPE)
        m['response-code'] = self.handle.getinfo(pycurl.RESPONSE_CODE)
        m['speed-download'] = self.handle.getinfo(pycurl.SPEED_DOWNLOAD)
        m['ssl-verifyresult'] = self.handle.getinfo(pycurl.SSL_VERIFYRESULT)
        m['filetime'] = self.handle.getinfo(pycurl.INFO_FILETIME)
        m['starttransfer-time'] = self.handle.getinfo(pycurl.STARTTRANSFER_TIME)
        m['redirect-time'] = self.handle.getinfo(pycurl.REDIRECT_TIME)
        m['redirect-count'] = self.handle.getinfo(pycurl.REDIRECT_COUNT)
        m['http-connectcode'] = self.handle.getinfo(pycurl.HTTP_CONNECTCODE)
        m['httpauth-avail'] = self.handle.getinfo(pycurl.HTTPAUTH_AVAIL)
        m['proxyauth-avail'] = self.handle.getinfo(pycurl.PROXYAUTH_AVAIL)
        m['os-errno'] = self.handle.getinfo(pycurl.OS_ERRNO)
        m['num-connects'] = self.handle.getinfo(pycurl.NUM_CONNECTS)
        m['ssl-engines'] = self.handle.getinfo(pycurl.SSL_ENGINES)
        m['cookielist'] = self.handle.getinfo(pycurl.INFO_COOKIELIST)
        m['lastsocket'] = self.handle.getinfo(pycurl.LASTSOCKET)
        m['ftp-entry-path'] = self.handle.getinfo(pycurl.FTP_ENTRY_PATH)
        return m

def curl_result(c):
    effective_url = c.getinfo(pycurl.EFFECTIVE_URL)
    primary_ip = c.getinfo(pycurl.PRIMARY_IP)
    primary_port = c.getinfo(pycurl.PRIMARY_PORT)
    local_ip = c.getinfo(pycurl.LOCAL_IP)
    local_port = c.getinfo(pycurl.LOCAL_PORT)
    speed_download = c.getinfo(pycurl.SPEED_DOWNLOAD)
    size_download = c.getinfo(pycurl.SIZE_DOWNLOAD)
    redirect_time = c.getinfo(pycurl.REDIRECT_TIME)
    redirect_count = c.getinfo(pycurl.REDIRECT_COUNT)
    redirect_url = c.getinfo(pycurl.REDIRECT_URL)
    http_code = c.getinfo(pycurl.HTTP_CODE)
    response_code = c.getinfo(pycurl.RESPONSE_CODE)
    total_time = c.getinfo(pycurl.TOTAL_TIME)
    content_type = c.getinfo(pycurl.CONTENT_TYPE)
    namelookup_time = c.getinfo(pycurl.NAMELOOKUP_TIME)
    info_filetime = c.getinfo(pycurl.INFO_FILETIME)
    http_connectcode = c.getinfo(pycurl.HTTP_CONNECTCODE)
    starttransfer_time = c.getinfo(pycurl.STARTTRANSFER_TIME)
    pretransfer_time = c.getinfo(pycurl.PRETRANSFER_TIME)
    header_size = c.getinfo(pycurl.HEADER_SIZE)
    request_size = c.getinfo(pycurl.REQUEST_SIZE)
    ssl_verifyresult = c.getinfo(pycurl.SSL_VERIFYRESULT)
    num_connects = c.getinfo(pycurl.NUM_CONNECTS)

    return {
        'effective_url': effective_url,
        'primary_ip': primary_ip,
        'primary_port': primary_port,
        'local_ip': local_ip,
        'local_port': local_port,
        'speed_download': speed_download,
        'size_download': size_download,
        'redirect_time': redirect_time,
        'redirect_count': redirect_count,
        'redirect_url': redirect_url,
        'http_code': http_code,
        'response_code': response_code,
        'total_time': total_time,
        'content_type': content_type,
        'namelookup_time': namelookup_time,
        'info_filetime': info_filetime,
        'http_connectcode': http_connectcode,
        'starttransfer_time': starttransfer_time,
        'pretransfer_time': pretransfer_time,
        'header_size': header_size,
        'request_size': request_size,
        'ssl_verifyresult': ssl_verifyresult,
        'num_connects': num_connects,
        # 'proxy_ssl_verifyresult': proxy_ssl_verifyresult,
        # 'app_connecttime': app_connecttime,

    }