public static CloseableHttpClient buildHttpClient() { CloseableHttpClient httpClient = HttpClients.custom() .addInterceptorFirst((HttpRequestInterceptor) (request, context) -> { if (!request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip"); } }).addInterceptorFirst((HttpResponseInterceptor) (response1, context) -> { HttpEntity entity = response1.getEntity(); if (entity != null) { Header ceHeader = entity.getContentEncoding(); if (ceHeader != null) { HeaderElement[] codecs = ceHeader.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { response1.setEntity( new GzipDecompressingEntity(response1.getEntity())); return; } } } } }).build(); return httpClient; }
@Test public void testContentEncodingRequestParameter() throws Exception { final HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, 200, "OK"); final StringEntity original = new StringEntity("encoded stuff"); original.setContentEncoding("GZip"); response.setEntity(original); final RequestConfig config = RequestConfig.custom() .setDecompressionEnabled(false) .build(); final HttpContext context = new BasicHttpContext(); context.setAttribute(HttpClientContext.REQUEST_CONFIG, config); final HttpResponseInterceptor interceptor = new ResponseContentEncoding(); interceptor.process(response, context); final HttpEntity entity = response.getEntity(); Assert.assertNotNull(entity); Assert.assertFalse(entity instanceof GzipDecompressingEntity); }
private HttpClient initClient() { DefaultHttpClient client = new DefaultHttpClient(); client.addRequestInterceptor((request, context) -> { if (!request.containsHeader(ACCEPT_ENCODING)) { request.addHeader(ACCEPT_ENCODING, GZIP); } }); client.addResponseInterceptor((response, context) -> { HttpEntity entity = response.getEntity(); if (entity != null) { Header header = entity.getContentEncoding(); if (header != null) { HeaderElement[] elements = header.getElements(); for (int i = 0; i < elements.length; i++) { if (elements[i].getName().equalsIgnoreCase(GZIP)) { response.setEntity( new GzipDecompressingEntity(response.getEntity())); return; } } } } }); return client; }
@Override public void process(HttpResponse response, HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); if (entity != null) { Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity( response.getEntity())); return; } } } } }
public static String getResponseString(HttpResponse response) { if (response == null) { return null; } try { HttpEntity entity = response.getEntity(); Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { if (contentEncoding.getValue().toLowerCase(Locale.getDefault()) .contains("gzip")) { GzipDecompressingEntity gzipEntity = new GzipDecompressingEntity( entity); return EntityUtils.toString(gzipEntity); } } return EntityUtils.toString(entity); } catch (Exception e) { e.printStackTrace(); return null; } }
@Override public void process(HttpResponse response, HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); if (entity != null) { Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response.setEntity( new GzipDecompressingEntity(response.getEntity())); return; } } } } }
@Override public void process(HttpResponse httpResponse, HttpContext httpContext) throws HttpException, IOException { HttpEntity entity = httpResponse.getEntity(); if (entity != null) { Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { httpResponse.setEntity(new GzipDecompressingEntity( httpResponse.getEntity())); return; } } } } }
public JSONObject postJSON() throws JSONException, NForumException, IOException{ HttpResponse response = httpClient.execute(httpPost); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != 200) throw new NForumException(NForumException.EXCEPTION_NETWORK + ":" + statusCode); Header header = response.getEntity().getContentEncoding(); if (header != null) { for (HeaderElement element : header.getElements()) { if (element.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); break; } } } String result = ResponseProcessor.getStringFromResponse(response); httpPost.abort(); JSONObject json = new JSONObject(result); if(json.has("msg")){ throw new NForumException(json.getString("msg")); } return json; }
public JSONObject getJSON() throws JSONException, NForumException, IOException{ HttpResponse response = httpClient.execute(httpGet); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != 200) throw new NForumException(NForumException.EXCEPTION_NETWORK + ":" + statusCode); Header header = response.getEntity().getContentEncoding(); if (header != null) { for (HeaderElement element : header.getElements()) { if (element.getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); break; } } } String result = ResponseProcessor.getStringFromResponse(response); httpGet.abort(); JSONObject json = new JSONObject(result); if(json.has("msg")){ throw new NForumException(json.getString("msg")); } return json; }
@Override public void process(HttpResponse response, HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); if (entity != null) { Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) { HeaderElement[] codecs = contentEncoding.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase(GZIP)) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } if (codec.getName().equalsIgnoreCase(DEFLATE)) { response.setEntity(new DeflateDecompressingEntity(response.getEntity())); return; } } } } }
private void handleContentEncoding(HttpResponse response) throws ServletException { HttpEntity entity = response.getEntity(); if (entity != null) { Header contentEncodingHeader = entity.getContentEncoding(); if (contentEncodingHeader != null) { HeaderElement[] codecs = contentEncodingHeader.getElements(); LOGGER.debug("Content-Encoding in response:"); for (HeaderElement codec : codecs) { String codecname = codec.getName().toLowerCase(); LOGGER.debug(" => codec: " + codecname); if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } else if ("deflate".equals(codecname)) { response.setEntity(new DeflateDecompressingEntity(response.getEntity())); return; } else if ("identity".equals(codecname)) { return; } else { throw new ServletException("Unsupported Content-Encoding: " + codecname); } } } } }
public HttpResponse getHttpResponse(String url) throws ClientProtocolException, IOException { HttpResponse response = null; HttpGet get = new HttpGet(url); get.addHeader("Accept", "text/html"); get.addHeader("Accept-Charset", "utf-8"); get.addHeader("Accept-Encoding", "gzip"); get.addHeader("Accept-Language", "en-US,en"); int uai = rand.nextInt() % userAgents.size(); if (uai < 0) { uai = -uai; } get.addHeader("User-Agent", userAgents.get(uai)); response = getHttpClient().execute(get); HttpEntity entity = response.getEntity(); Header header = entity.getContentEncoding(); if (header != null) { HeaderElement[] codecs = header.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(entity)); } } } return response; }
/** * Handles the following {@code Content-Encoding}s by * using the appropriate decompressor to wrap the response Entity: * <ul> * <li>gzip - see {@link GzipDecompressingEntity}</li> * <li>deflate - see {@link DeflateDecompressingEntity}</li> * <li>identity - no action needed</li> * </ul> * * @param response the response which contains the entity * @param context not currently used * * @throws HttpException if the {@code Content-Encoding} is none of the above */ public void process( final HttpResponse response, final HttpContext context) throws HttpException, IOException { HttpEntity entity = response.getEntity(); // It wasn't a 304 Not Modified response, 204 No Content or similar if (entity != null) { Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (HeaderElement codec : codecs) { String codecname = codec.getName().toLowerCase(Locale.US); if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); if (context != null) context.setAttribute(UNCOMPRESSED, true); return; } else if ("deflate".equals(codecname)) { response.setEntity(new DeflateDecompressingEntity(response.getEntity())); if (context != null) context.setAttribute(UNCOMPRESSED, true); return; } else if ("identity".equals(codecname)) { /* Don't need to transform the content - no-op */ return; } else { throw new HttpException("Unsupported Content-Coding: " + codec.getName()); } } } } }
public static HttpEntity handleCompressedEntity(HttpEntity entity) { org.apache.http.Header contentEncoding = entity.getContentEncoding(); if (contentEncoding != null) for (HeaderElement e : contentEncoding.getElements()) { if (CONTENT_ENCODING_GZIP.equalsIgnoreCase(e.getName())) { return new GzipDecompressingEntity(entity); } if (CONTENT_ENCODING_DEFLATE.equalsIgnoreCase(e.getName())) { return new DeflateDecompressingEntity(entity); } } return entity; }
/** * If gzip is enabled, this will decode things. * @return */ private HttpResponseInterceptor createGzipResponseInterceptor() { return new HttpResponseInterceptor() { @Override public void process( HttpResponse response, HttpContext context ) throws HttpException, IOException { //..get the entity final HttpEntity entity = response.getEntity(); if ( entity == null ) return; //..Get any content encoding headers final Header ceHeader = entity.getContentEncoding(); if ( ceHeader == null ) return; //..Get any entries HeaderElement[] codecs = ceHeader.getElements(); //..See if one is marked as gzip for ( final HeaderElement codec : codecs ) { if ( codec.getName().equalsIgnoreCase( "gzip" )) { //..Hey, it's gzip! decompress the entity response.setEntity( new GzipDecompressingEntity( response.getEntity())); //..Done with this ish. return; } } } }; }
/** * Handles the following {@code Content-Encoding}s by * using the appropriate decompressor to wrap the response Entity: * <ul> * <li>gzip - see {@link GzipDecompressingEntity}</li> * <li>deflate - see {@link DeflateDecompressingEntity}</li> * <li>identity - no action needed</li> * </ul> * * @param response the response which contains the entity * @param context not currently used * * @throws HttpException if the {@code Content-Encoding} is none of the above */ public void process( final HttpResponse response, final HttpContext context) throws HttpException, IOException { final HttpEntity entity = response.getEntity(); // entity can be null in case of 304 Not Modified, 204 No Content or similar // check for zero length entity. if (entity != null && entity.getContentLength() != 0) { final Header ceheader = entity.getContentEncoding(); if (ceheader != null) { final HeaderElement[] codecs = ceheader.getElements(); boolean uncompressed = false; for (final HeaderElement codec : codecs) { final String codecname = codec.getName().toLowerCase(Locale.ENGLISH); if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); uncompressed = true; break; } else if ("deflate".equals(codecname)) { response.setEntity(new DeflateDecompressingEntity(response.getEntity())); uncompressed = true; break; } else if ("identity".equals(codecname)) { /* Don't need to transform the content - no-op */ return; } else { throw new HttpException("Unsupported Content-Coding: " + codec.getName()); } } if (uncompressed) { response.removeHeaders("Content-Length"); response.removeHeaders("Content-Encoding"); response.removeHeaders("Content-MD5"); } } } }
@Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { final HttpEntity entity = response.getEntity(); if (entity == null) { return; // no work to do } final Header ceheader = entity.getContentEncoding(); if (ceheader != null) { final HeaderElement[] codecs = ceheader.getElements(); if (codecs.length == 0) { return; } final HeaderElement codec = codecs[0]; if ("gzip".equalsIgnoreCase(codec.getName())) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); } else if ("deflate".equalsIgnoreCase(codec.getName())) { final InputStreamEntity isEntity = new InputStreamEntity(getTranslatedInputStream(response.getEntity()), -1); isEntity.setContentType(response.getEntity().getContentType()); response.setEntity(isEntity); } else if ("identity".equalsIgnoreCase(codec.getName())) { /* Don't need to transform the content - no-op */ } else { throw new HttpException("Unsupported Content-Coding: " + codec.getName()); } } }
private HttpEntity getHttpEntity(HttpResponse response) { HttpEntity entity = response.getEntity(); if (entity != null) { Header contentEncodingHeader = entity.getContentEncoding(); if (contentEncodingHeader != null) { for (HeaderElement element : contentEncodingHeader.getElements()) { if (element.getName().equalsIgnoreCase("gzip")) { return new GzipDecompressingEntity(response.getEntity()); } } } } return entity; }
private void handleGzip(HttpResponse httpResponse) { Header ceheader = httpResponse.getEntity().getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (HeaderElement codec : codecs) { if (codec.getName().equalsIgnoreCase("gzip")) { httpResponse.setEntity(new GzipDecompressingEntity(httpResponse.getEntity())); } } } }
public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { final HttpEntity entity = response.getEntity(); final Header encodingHeader = entity.getContentEncoding(); if (encodingHeader != null) { final HeaderElement[] codecs = encodingHeader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } }
public String executeGet(String uri) throws ExecutionException, InterruptedException, IOException { System.out.println(Thread.currentThread().getName()); return EntityUtils.toString(new GzipDecompressingEntity(client.execute(new HttpGet(uri), null).get().getEntity())); }
private static HttpClient getHttpClient() { SchemeRegistry schemeRegistry = new SchemeRegistry(); schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry); // Increase max total connection to 200 cm.setMaxTotal(200); // Increase default max connection per route to 20 cm.setDefaultMaxPerRoute(20); // Increase max connections for localhost:80 to 50 HttpHost localhost = new HttpHost("locahost", 80); cm.setMaxPerRoute(new HttpRoute(localhost), 50); DefaultHttpClient httpClient = new DefaultHttpClient(cm); // Use a proxy if it is defined - we need to pass this on to the // HttpClient if (System.getProperties().containsKey("http.proxyHost")) { String host = System.getProperty("http.proxyHost"); String port = System.getProperty("http.proxyPort", "8080"); HttpHost proxy = new HttpHost(host, Integer.parseInt(port)); httpClient.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy); } httpClient.addRequestInterceptor(new HttpRequestInterceptor() { @Override public void process(HttpRequest request, HttpContext arg1) throws HttpException, IOException { if (!request.containsHeader("Accept-Encoding")) { request.addHeader("Accept-Encoding", "gzip"); } } }); httpClient.addResponseInterceptor(new HttpResponseInterceptor() { @Override public void process(final HttpResponse response, final HttpContext context) throws HttpException, IOException { if (log.isTraceEnabled()) { log.trace("Response Headers:"); for (Header h : response.getAllHeaders()) { log.trace(h.getName() + " : " + h.getValue()); } } HttpEntity entity = response.getEntity(); if (entity != null) { Header ceheader = entity.getContentEncoding(); if (ceheader != null) { HeaderElement[] codecs = ceheader.getElements(); for (int i = 0; i < codecs.length; i++) { if (codecs[i].getName().equalsIgnoreCase("gzip")) { response.setEntity(new GzipDecompressingEntity(response.getEntity())); return; } } } } } }); return httpClient; }
/** * <pre> * <b>Description : </b> * Processes the HttpResponse to create a DHisHttpResponse object * * @param requestURL * @param username * @param response * @return * @throws IOException </pre> */ private static DhisHttpResponse processResponse( String requestURL, String username, HttpResponse response ) throws Exception { DhisHttpResponse dhisHttpResponse = null; String output = null; int statusCode = 0; if ( response != null ) { HttpEntity responseEntity = response.getEntity(); if ( responseEntity != null && responseEntity.getContent() != null ) { Header contentType = response.getEntity().getContentType(); if ( contentType != null && checkIfGzipContentType( contentType ) ) { GzipDecompressingEntity gzipDecompressingEntity = new GzipDecompressingEntity( response.getEntity() ); InputStream content = gzipDecompressingEntity.getContent(); output = IOUtils.toString( content, StandardCharsets.UTF_8 ); } else { output = EntityUtils.toString( response.getEntity() ); } statusCode = response.getStatusLine().getStatusCode(); } else { throw new Exception( "No content found in the response received from http POST call to " + requestURL + " with username " + username ); } dhisHttpResponse = new DhisHttpResponse( response, output, statusCode ); } else { throw new Exception( "NULL response received from http POST call to " + requestURL + " with username " + username ); } return dhisHttpResponse; }
/** * 带参数的GET请求 * * @param url * 请求地址 * @param pm * 参数 * @return */ public static String sendGet(String url, BDHttpParam pm) { // 设置通用参数 StringBuilder params = new StringBuilder(); if (pm.hasCommon()) { params.append(url).append("?"); for (Entry<String, String> apm : pm.getCommonParams().entrySet()) { params.append(apm.getKey()).append("=").append(apm.getValue()).append("&"); } url = params.substring(0, params.lastIndexOf("&")); } // 修改适应https请求 @20160804 // HttpClient client = HttpClientBuilder.create().build(); HttpClient client = isHttps(url) ? new DefaultHttpClient() : HttpClientBuilder.create().build(); try { if (isHttps(url)) { SSLContext sslCtx = SSLContext.getInstance("SSL"); sslCtx.init(null, new TrustManager[] { X509_TM }, null); SSLSocketFactory sslSf = new SSLSocketFactory(sslCtx); ClientConnectionManager ccm = client.getConnectionManager(); SchemeRegistry sr = ccm.getSchemeRegistry(); sr.register(new Scheme("https", 443, sslSf)); } HttpGet get = new HttpGet(url); log.debug("get url {}", url); // 设置Cookie if (pm.hasCookie()) { StringBuilder cookies = new StringBuilder(); for (Entry<String, String> acm : pm.getCookieParams().entrySet()) { cookies.append(acm.getKey()).append("=").append(acm.getValue()).append(";"); } get.setHeader("Cookie", cookies.toString()); log.debug("\twith cookie {}", cookies.toString()); } // 设置指定http-头 if (pm.hasHeader()) { for (Entry<String, String> ahm : pm.getHeaderParams().entrySet()) { get.setHeader(ahm.getKey(), ahm.getValue()); log.debug("\twith header {}={}", ahm.getKey(), ahm.getValue()); } } HttpResponse res = client.execute(get); int code = res.getStatusLine().getStatusCode(); if (code == 200) { // 判断是否gzip响应 @20160804 Header hd = res.getLastHeader("Content-Encoding"); if (hd != null && hd.getValue().equals("gzip")) { res.setEntity(new GzipDecompressingEntity(res.getEntity())); } return EntityUtils.toString(res.getEntity(), pm.getCharset()).trim(); } // NetMonitor.log(code, url); } catch (Exception e) { // NetMonitor.log(e, url); } finally { HttpClientUtils.closeQuietly(client); } return ""; }
/** * POST请求 * * @param url * 地址 * @param pm * 参数 */ public static String sendPost(String url, BDHttpParam pm) { HttpClient client = HttpClientBuilder.create().build(); try { HttpPost post = new HttpPost(url); log.debug("post url {}", url); // 设置Cookie内容 if (pm.hasCookie()) { StringBuilder cookies = new StringBuilder(); for (Entry<String, String> acm : pm.getCookieParams().entrySet()) { cookies.append(acm.getKey()).append("=").append(acm.getValue()).append(";"); } post.setHeader("Cookie", cookies.toString()); log.debug("\twith cookie {}", cookies.toString()); } // 设置指定http-头 if (pm.hasHeader()) { for (Entry<String, String> ahm : pm.getHeaderParams().entrySet()) { post.setHeader(ahm.getKey(), ahm.getValue()); log.debug("\twith header {}={}", ahm.getKey(), ahm.getValue()); } } // 设置通用参数 List<NameValuePair> paramList = new ArrayList<NameValuePair>(); if (pm.hasCommon()) { for (Entry<String, String> apm : pm.getCommonParams().entrySet()) { paramList.add(new BasicNameValuePair(apm.getKey(), apm.getValue())); log.debug("\twith param {}={}", apm.getKey(), apm.getValue()); } } post.setEntity(new UrlEncodedFormEntity(paramList, pm.getCharset())); HttpResponse res = client.execute(post); int code = res.getStatusLine().getStatusCode(); if (code == 200) { Header hd = res.getLastHeader("Content-Encoding"); if (hd != null && hd.getValue().equals("gzip")) { res.setEntity(new GzipDecompressingEntity(res.getEntity())); } return EntityUtils.toString(res.getEntity(), pm.getCharset()).trim(); } // NetMonitor.log(code, url); } catch (Exception e) { // NetMonitor.log(e, url); } finally { HttpClientUtils.closeQuietly(client); } return ""; }
/** * Copy response body data (the entity) from the proxy to the servlet client. * * @param proxyResponse * @param servletResponse * @param kibanaFilter * @param email * @param index * @throws java.io.IOException */ protected void copyResponseEntity(HttpResponse proxyResponse, HttpServletResponse servletResponse, KibanaFilter kibanaFilter, String email, String index) throws IOException { if (kibanaFilter == null) { super.copyResponseEntity(proxyResponse, servletResponse); } else { switch (kibanaFilter) { case KIBANA_INDEXPATTERN_SEARCH: HttpEntity entity = proxyResponse.getEntity(); if (entity != null) { GzipDecompressingEntity gzipEntity = new GzipDecompressingEntity( entity); String resp = EntityUtils.toString(gzipEntity); BasicHttpEntity basic = new BasicHttpEntity(); JSONObject indices = new JSONObject(resp); //Remove all projects other than the current one and check //if user is authorizer to access it List<String> projects = projectController.findProjectNamesByUser(email, true); JSONArray hits = indices.getJSONObject("hits").getJSONArray("hits"); for (int i = hits.length() - 1; i >= 0; i--) { String projectName = hits.getJSONObject(i).getString("_id"); if (index != null) { if ((!currentProjects.get(email).equalsIgnoreCase(projectName) || !projects.contains(projectName)) && !projectName.equals(Settings.KIBANA_DEFAULT_INDEX) && !projectName.equals(index)) { hits.remove(i); } } else { if ((!currentProjects.get(email).equalsIgnoreCase(projectName) || !projects.contains(projectName)) && !projectName.equals(Settings.KIBANA_DEFAULT_INDEX)) { hits.remove(i); } } } InputStream in = IOUtils.toInputStream(indices.toString()); OutputStream servletOutputStream = servletResponse.getOutputStream(); basic.setContent(in); GzipCompressingEntity compress = new GzipCompressingEntity(basic); compress.writeTo(servletOutputStream); } break; default: super.copyResponseEntity(proxyResponse, servletResponse); break; } } }
/** * Get the content of a gzip encoded page * @param url url from which to read * @return the String content of the page * @throws Exception */ public static String getGzipedData(String url) throws Exception { NUHttpGet httpGet = new NUHttpGet(url); HttpResponse httpResponse = NUHttpClient.getHttpClient().execute(httpGet); return EntityUtils.toString(new GzipDecompressingEntity(httpResponse.getEntity())); }
/** * Get the content of a gzip encoded page * @param url url from which to read * @param httpContext the httpContext in which to make the request * @return the String content of the page * @throws Exception */ public static String getGzipedData(String url, HttpContext httpContext) throws Exception { NUHttpGet httpGet = new NUHttpGet(url); HttpResponse httpResponse = NUHttpClient.getHttpClient().execute(httpGet, httpContext); return EntityUtils.toString(new GzipDecompressingEntity(httpResponse.getEntity())); }
/** * * 从response返回的实体中读取页面代码 * * @param httpEntity * Http实体 * * @return 页面代码 * * @throws ParseException * * @throws IOException */ private static String readHtmlContentFromEntity(HttpEntity httpEntity) throws ParseException, IOException { String html = ""; Header header = httpEntity.getContentEncoding(); if (httpEntity.getContentLength() < 2147483647L) { // EntityUtils无法处理ContentLength超过2147483647L的Entity if (header != null && "gzip".equals(header.getValue())) { html = EntityUtils.toString(new GzipDecompressingEntity( httpEntity)); } else { html = EntityUtils.toString(httpEntity); } } else { InputStream in = httpEntity.getContent(); if (header != null && "gzip".equals(header.getValue())) { html = unZip(in, ContentType.getOrDefault(httpEntity) .getCharset().toString()); } else { html = readInStreamToString(in, ContentType.getOrDefault(httpEntity).getCharset() .toString()); } if (in != null) { in.close(); } } return html; }