public DatarouterHttpResponse(HttpResponse response, HttpClientContext context, Consumer<HttpEntity> httpEntityConsumer){ this.response = response; this.cookies = context.getCookieStore().getCookies(); if(response != null){ this.statusCode = response.getStatusLine().getStatusCode(); this.entity = ""; HttpEntity httpEntity = response.getEntity(); if(httpEntity == null){ return; } if(httpEntityConsumer != null){ httpEntityConsumer.accept(httpEntity); return; } try{ this.entity = EntityUtils.toString(httpEntity); }catch(IOException e){ logger.error("Exception occurred while reading HTTP response entity", e); }finally{ EntityUtils.consumeQuietly(httpEntity); } } }
public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { AuthState authState = (AuthState) context.getAttribute(HttpClientContext.TARGET_AUTH_STATE); if (authState.getAuthScheme() != null || authState.hasAuthOptions()) { return; } // If no authState has been established and this is a PUT or POST request, add preemptive authorisation String requestMethod = request.getRequestLine().getMethod(); if (alwaysSendAuth || requestMethod.equals(HttpPut.METHOD_NAME) || requestMethod.equals(HttpPost.METHOD_NAME)) { CredentialsProvider credentialsProvider = (CredentialsProvider) context.getAttribute(HttpClientContext.CREDS_PROVIDER); HttpHost targetHost = (HttpHost) context.getAttribute(HttpCoreContext.HTTP_TARGET_HOST); Credentials credentials = credentialsProvider.getCredentials(new AuthScope(targetHost.getHostName(), targetHost.getPort())); if (credentials == null) { throw new HttpException("No credentials for preemptive authentication"); } authState.update(authScheme, credentials); } }
public Map<String, String> connect(String url, Map<String, Object> parameters) throws ManagerResponseException { Map<String, String> response = new HashMap<String, String>(); CloseableHttpClient httpclient = HttpClients.createDefault(); List<NameValuePair> nvps = new ArrayList<>(); nvps.add(new BasicNameValuePair("j_username", (String) parameters.get("login"))); nvps.add(new BasicNameValuePair("j_password", (String) parameters.get("password"))); localContext = HttpClientContext.create(); localContext.setCookieStore(new BasicCookieStore()); HttpPost httpPost = new HttpPost(url); try { httpPost.setEntity(new UrlEncodedFormEntity(nvps)); CloseableHttpResponse httpResponse = httpclient.execute(httpPost, localContext); ResponseHandler<String> handler = new CustomResponseErrorHandler(); String body = handler.handleResponse(httpResponse); response.put(BODY, body); httpResponse.close(); } catch (Exception e) { authentificationUtils.getMap().clear(); throw new ManagerResponseException(e.getMessage(), e); } return response; }
@Override public AsyncClientHttpRequest createAsyncRequest(URI uri, HttpMethod httpMethod) throws IOException { HttpAsyncClient asyncClient = getHttpAsyncClient(); startAsyncClient(); HttpUriRequest httpRequest = createHttpUriRequest(httpMethod, uri); postProcessHttpRequest(httpRequest); HttpContext context = createHttpContext(httpMethod, uri); if (context == null) { context = HttpClientContext.create(); } // Request configuration not set in the context if (context.getAttribute(HttpClientContext.REQUEST_CONFIG) == null) { // Use request configuration given by the user, when available RequestConfig config = null; if (httpRequest instanceof Configurable) { config = ((Configurable) httpRequest).getConfig(); } if (config == null) { config = RequestConfig.DEFAULT; } context.setAttribute(HttpClientContext.REQUEST_CONFIG, config); } return new HttpComponentsAsyncClientHttpRequest(asyncClient, httpRequest, context); }
public ResponseWrap(CloseableHttpClient httpClient, HttpRequestBase request, CloseableHttpResponse response, HttpClientContext context, ObjectMapper _mapper) { this.response = response; this.httpClient = httpClient; this.request = request; this.context = context; mapper = _mapper; try { HttpEntity entity = response.getEntity(); if (entity != null) { this.entity = new BufferedHttpEntity(entity); } else { this.entity = new BasicHttpEntity(); } EntityUtils.consumeQuietly(entity); this.response.close(); } catch (IOException e) { logger.warn(e.getMessage()); } }
@Override public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool, CrawlerSession crawlerSession) { HttpClientContext httpClientContext = HttpClientContext.adapt(context); Proxy proxy = (Proxy) crawlerSession.getExtInfo(VSCRAWLER_AVPROXY_KEY); if (proxy == null) { String accessUrl = null; if (request instanceof HttpRequestWrapper || request instanceof HttpGet) { accessUrl = HttpUriRequest.class.cast(request).getURI().toString(); } if (!PoolUtil.isDungProxyEnabled(httpClientContext)) { log.info("{}不会被代理", accessUrl); return null; } proxy = ipPool.getIP(host.getHostName(), accessUrl); if (proxy == null) { return null; } crawlerSession.setExtInfo(VSCRAWLER_AVPROXY_KEY, proxy); } return proxy; }
@Test public void testPreemptiveAuthentication() throws Exception { final CountingAuthHandler requestHandler = new CountingAuthHandler(); this.serverBootstrap.registerHandler("*", requestHandler); final HttpHost target = start(); final HttpClientContext context = HttpClientContext.create(); final AuthCache authCache = new BasicAuthCache(); authCache.put(target, new BasicScheme()); context.setAuthCache(authCache); final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("test", "test")); context.setCredentialsProvider(credsProvider); final HttpGet httpget = new HttpGet("/"); final HttpResponse response1 = this.httpclient.execute(target, httpget, context); final HttpEntity entity1 = response1.getEntity(); Assert.assertEquals(HttpStatus.SC_OK, response1.getStatusLine().getStatusCode()); Assert.assertNotNull(entity1); EntityUtils.consume(entity1); Assert.assertEquals(1, requestHandler.getCount()); }
protected HttpContext createHttpContext(HttpProxy httpProxy, CookieStore cookieStore) { HttpContext httpContext = new HttpClientContext(); if (cookieStore != null) { httpContext.setAttribute(HttpClientContext.COOKIE_STORE, cookieStore); } if (httpProxy != null && StringUtils.isNotBlank(httpProxy.getUsername())) { CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); credentialsProvider.setCredentials(new AuthScope(httpProxy.getHost(), httpProxy.getPort()), new UsernamePasswordCredentials(httpProxy.getUsername(), httpProxy.getPassword())); httpContext.setAttribute(HttpClientContext.CREDS_PROVIDER, credentialsProvider); } return httpContext; }
@Override public boolean retryRequest( HttpResponse httpResponse, int executionCount, HttpContext httpContext) { int status = httpResponse.getStatusLine().getStatusCode(); if (executionCount < 3 && status >= 500 && status < 600) { HttpClientContext context = HttpClientContext.adapt(httpContext); logger.info( "Retry " + executionCount + " for request for: " + context.getRequest().getRequestLine().getUri()); return true; } return false; }
private void retrieveAccessToken() throws IOException, WeiboClientException { String state = "__MY_STATE__"; String authorizationCallback = "https://api.weibo.com/oauth2/authorize"; String url = this.client.getAuthorizationUrl(ResponseType.Code, DisplayType.Default, state, authorizationCallback); //httpget CloseableHttpResponse response = null; HttpClientContext context = HttpClientContext.create(); HttpGet httpGet = new HttpGet(url); response = httpClient.execute(httpGet, context); // 获取所有的重定向位置 List<URI> redirectLocations = context.getRedirectLocations(); //end System.out.println("Please visit: " + url); System.out.print("Input code: "); in = new BufferedReader(new InputStreamReader(System.in)); String code = in.readLine(); String accessTokenCallback = "https://api.weibo.com/oauth2/authorize"; SinaWeibo2AccessToken accessToken = this.client.getAccessTokenByCode(code, accessTokenCallback); System.out.println(); System.out.println("Access token: " + accessToken.getToken()); System.out.println("Uid: " + accessToken.getUid()); System.out.println("Expires in: " + accessToken.getExpiresIn()); System.out.println("Remind in: " + accessToken.getRemindIn()); accessToken = new SinaWeibo2AccessToken(accessToken.getToken()); this.client.setAccessToken(accessToken); }
@SuppressWarnings({ "unchecked", "rawtypes" }) public void testTooLargeResponse() throws Exception { ContentTooLongException tooLong = new ContentTooLongException("too long!"); CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class); when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class), any(HttpClientContext.class), any(FutureCallback.class))).then(new Answer<Future<HttpResponse>>() { @Override public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable { HeapBufferedAsyncResponseConsumer consumer = (HeapBufferedAsyncResponseConsumer) invocationOnMock.getArguments()[1]; FutureCallback callback = (FutureCallback) invocationOnMock.getArguments()[3]; assertEquals(new ByteSizeValue(100, ByteSizeUnit.MB).bytesAsInt(), consumer.getBufferLimit()); callback.failed(tooLong); return null; } }); RemoteScrollableHitSource source = sourceWithMockedClient(true, httpClient); AtomicBoolean called = new AtomicBoolean(); Consumer<Response> checkResponse = r -> called.set(true); Throwable e = expectThrows(RuntimeException.class, () -> source.doStartNextScroll(FAKE_SCROLL_ID, timeValueMillis(0), checkResponse)); // Unwrap the some artifacts from the test while (e.getMessage().equals("failed")) { e = e.getCause(); } // This next exception is what the user sees assertEquals("Remote responded with a chunk that was too large. Use a smaller batch size.", e.getMessage()); // And that exception is reported as being caused by the underlying exception returned by the client assertSame(tooLong, e.getCause()); assertFalse(called.get()); }
/** * Verifies the content of the {@link HttpRequest} that's internally created and passed through to the http client */ @SuppressWarnings("unchecked") public void testInternalHttpRequest() throws Exception { ArgumentCaptor<HttpAsyncRequestProducer> requestArgumentCaptor = ArgumentCaptor.forClass(HttpAsyncRequestProducer.class); int times = 0; for (String httpMethod : getHttpMethods()) { HttpUriRequest expectedRequest = performRandomRequest(httpMethod); verify(httpClient, times(++times)).<HttpResponse>execute(requestArgumentCaptor.capture(), any(HttpAsyncResponseConsumer.class), any(HttpClientContext.class), any(FutureCallback.class)); HttpUriRequest actualRequest = (HttpUriRequest)requestArgumentCaptor.getValue().generateRequest(); assertEquals(expectedRequest.getURI(), actualRequest.getURI()); assertEquals(expectedRequest.getClass(), actualRequest.getClass()); assertArrayEquals(expectedRequest.getAllHeaders(), actualRequest.getAllHeaders()); if (expectedRequest instanceof HttpEntityEnclosingRequest) { HttpEntity expectedEntity = ((HttpEntityEnclosingRequest) expectedRequest).getEntity(); if (expectedEntity != null) { HttpEntity actualEntity = ((HttpEntityEnclosingRequest) actualRequest).getEntity(); assertEquals(EntityUtils.toString(expectedEntity), EntityUtils.toString(actualEntity)); } } } }
@Before @SuppressWarnings("unchecked") public void createRestClient() throws IOException { CloseableHttpAsyncClient httpClient = mock(CloseableHttpAsyncClient.class); when(httpClient.<HttpResponse>execute(any(HttpAsyncRequestProducer.class), any(HttpAsyncResponseConsumer.class), any(HttpClientContext.class), any(FutureCallback.class))).thenAnswer(new Answer<Future<HttpResponse>>() { @Override public Future<HttpResponse> answer(InvocationOnMock invocationOnMock) throws Throwable { HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0]; HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest(); HttpHost httpHost = requestProducer.getTarget(); HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2]; assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class)); FutureCallback<HttpResponse> futureCallback = (FutureCallback<HttpResponse>) invocationOnMock.getArguments()[3]; //return the desired status code or exception depending on the path if (request.getURI().getPath().equals("/soe")) { futureCallback.failed(new SocketTimeoutException(httpHost.toString())); } else if (request.getURI().getPath().equals("/coe")) { futureCallback.failed(new ConnectTimeoutException(httpHost.toString())); } else if (request.getURI().getPath().equals("/ioe")) { futureCallback.failed(new IOException(httpHost.toString())); } else { int statusCode = Integer.parseInt(request.getURI().getPath().substring(1)); StatusLine statusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 1), statusCode, ""); futureCallback.completed(new BasicHttpResponse(statusLine)); } return null; } }); int numHosts = RandomNumbers.randomIntBetween(getRandom(), 2, 5); httpHosts = new HttpHost[numHosts]; for (int i = 0; i < numHosts; i++) { httpHosts[i] = new HttpHost("localhost", 9200 + i); } failureListener = new HostsTrackingFailureListener(); restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, null, failureListener); }
private HttpClientContext prepareContext() { HttpHost targetHost = new HttpHost(serverConfig.getServerName(), serverConfig.getPort(), serverConfig.isUseHTTPS() ? "https" : "http"); AuthCache authCache = new BasicAuthCache(); authCache.put(targetHost, new BasicScheme()); CredentialsProvider credsProvider = new BasicCredentialsProvider(); UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(serverConfig.getUserName(), serverConfig.getPassword()); credsProvider.setCredentials(AuthScope.ANY, credentials); // Add AuthCache to the execution context HttpClientContext context = HttpClientContext.create(); context.setCredentialsProvider(credsProvider); context.setAuthCache(authCache); return context; }
/** * Remove a Cookie by name and path * * @param name cookie name * @param path cookie path */ @PublicAtsApi public void removeCookie( String name, String path ) { BasicCookieStore cookieStore = (BasicCookieStore) httpContext.getAttribute(HttpClientContext.COOKIE_STORE); if (cookieStore != null) { List<Cookie> cookies = cookieStore.getCookies(); cookieStore.clear(); for (Cookie cookie : cookies) { if (!cookie.getName().equals(name) || !cookie.getPath().equals(path)) { cookieStore.addCookie(cookie); } } } }
private HttpContext httpContext() { BasicCredentialsProvider basicCredentialsProvider = new BasicCredentialsProvider(); if (isNotBlank(this.username) && this.password != null) { UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(this.username, this.password); HttpHost httpHost = HttpHost.create(this.rootConfluenceUrl); AuthScope authScope = new AuthScope(httpHost); basicCredentialsProvider.setCredentials(authScope, credentials); BasicAuthCache basicAuthCache = new BasicAuthCache(); basicAuthCache.put(httpHost, new BasicScheme()); HttpClientContext httpClientContext = HttpClientContext.create(); httpClientContext.setCredentialsProvider(basicCredentialsProvider); httpClientContext.setAuthCache(basicAuthCache); return httpClientContext; } else { return null; } }
static SonarHttpRequester getSonarHttpRequester(GlobalConfigDataForSonarInstance globalConfigDataForSonarInstance) { try { HttpGet request = new HttpGet(getSonarApiServerVersion(globalConfigDataForSonarInstance)); HttpClientContext context = HttpClientContext.create(); CloseableHttpClient client = HttpClientBuilder.create().build(); CloseableHttpResponse response = client.execute(request, context); String sonarVersion = EntityUtils.toString(response.getEntity()); if (majorSonarVersion(sonarVersion) <= 5) { return new SonarHttpRequester5x(); } else if (majorSonarVersion(sonarVersion) >= 6 && minorSonarVersion(sonarVersion) == 0) { return new SonarHttpRequester60(); } else if (majorSonarVersion(sonarVersion) >= 6 && minorSonarVersion(sonarVersion) >= 1) { return new SonarHttpRequester61(); } else { throw new UnsuportedVersionException("Plugin doesn't suport this version of sonar api! Please contact the developer."); } } catch (IOException e) { throw new ApiConnectionException(e.getLocalizedMessage()); } }
private void loginApi(GlobalConfigDataForSonarInstance globalConfigDataForSonarInstance) { context = HttpClientContext.create(); client = HttpClientBuilder.create().build(); if (StringUtils.isNotEmpty(globalConfigDataForSonarInstance.getToken())) { token = globalConfigDataForSonarInstance.getToken(); } else { HttpPost loginHttpPost = new HttpPost(globalConfigDataForSonarInstance.getSonarUrl() + getSonarApiLogin()); List<NameValuePair> nvps = new ArrayList<>(); nvps.add(new BasicNameValuePair("login", globalConfigDataForSonarInstance.getUsername())); nvps.add(new BasicNameValuePair("password", globalConfigDataForSonarInstance.getPass())); nvps.add(new BasicNameValuePair("remember_me", "1")); loginHttpPost.setEntity(createEntity(nvps)); loginHttpPost.addHeader(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded"); executePostRequest(client, loginHttpPost); } logged = true; }
@Override public boolean retryRequest( IOException exception, int executionCount, HttpContext context) { if (executionCount > retryExecutionCount) { return false; } if (exception instanceof InterruptedIOException) { return false; } if (exception instanceof UnknownHostException) { return false; } if (exception instanceof ConnectTimeoutException) { return true; } if (exception instanceof SSLException) { return false; } HttpClientContext clientContext = HttpClientContext.adapt(context); HttpRequest request = clientContext.getRequest(); boolean idempotent = !(request instanceof HttpEntityEnclosingRequest); if (idempotent) { // Retry if the request is considered idempotent return true; } return false; }
public static <T> T execute(HttpUriRequest request,ResponseHandler<T> responseHandler){ String uriId = loggerRequest(request); if(responseHandler instanceof LocalResponseHandler){ LocalResponseHandler lrh = (LocalResponseHandler) responseHandler; lrh.setUriId(uriId); } try { T t = httpClient.execute(request, responseHandler,HttpClientContext.create()); if(resultErrorHandler != null){ resultErrorHandler.doHandle(uriId, request, t); } return t; } catch (Exception e) { e.printStackTrace(); logger.error(e.getMessage()); } return null; }
/** * * @param mch_id mch_id * @param request request * @param clazz clazz * @param sign_type 数据返回验证签名类型 * @param key 数据返回验证签名key * @since 2.8.5 * @return result */ public static <T> T keyStoreExecuteXmlResult(String mch_id,HttpUriRequest request,Class<T> clazz,String sign_type,String key){ String uriId = loggerRequest(request); ResponseHandler<T> responseHandler = XmlResponseHandler.createResponseHandler(clazz,sign_type,key); if(responseHandler instanceof LocalResponseHandler){ LocalResponseHandler lrh = (LocalResponseHandler) responseHandler; lrh.setUriId(uriId); } try { T t = httpClient_mchKeyStore.get(mch_id).execute(request,responseHandler,HttpClientContext.create()); if(resultErrorHandler != null){ resultErrorHandler.doHandle(uriId, request, t); } return t; } catch (Exception e) { e.printStackTrace(); logger.error(e.getMessage()); } return null; }
@Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context){ if(logOnRetry){ HttpClientContext clientContext = HttpClientContext.adapt(context); logger.warn("Request {} failure Nº {}", clientContext.getRequest().getRequestLine(), executionCount, exception); } Object retrySafe = context.getAttribute(RETRY_SAFE_ATTRIBUTE); if(retrySafe == null || !(retrySafe instanceof Boolean) || !(Boolean)retrySafe || executionCount > retryCount){ return false; } return true; }
private HttpClientContext createHttpClientContextFromConfig() { // create context from config HttpClientContext context = HttpClientContext.create(); if (config.getCookieStore() != null) { context.setCookieStore(config.getCookieStore()); } if (config.getCredsProvider() != null) { context.setCredentialsProvider(config.getCredsProvider()); } if (config.getAuthCache() != null) { context.setAuthCache(config.getAuthCache()); } return context; }
public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException { final HttpClientContext clientContext = HttpClientContext.adapt(httpContext); List<Cookie> cookies = clientContext.getCookieStore().getCookies(); boolean set = (null != StickyCookieHolder.getTestStickySessionCookie()); boolean found = false; ListIterator<Cookie> it = cookies.listIterator(); while (it.hasNext()) { Cookie cookie = it.next(); if (cookie.getName().equals(StickyCookieHolder.COOKIE_NAME)) { found = true; if (set) { // set the cookie with the value saved for each thread using the rule it.set(StickyCookieHolder.getTestStickySessionCookie()); } else { // if the cookie is not set in TestStickySessionRule, remove it from here it.remove(); } } } // if the cookie needs to be set from TestStickySessionRule but did not exist in the client cookie list, add it here. if (!found && set) { cookies.add(StickyCookieHolder.getTestStickySessionCookie()); } BasicCookieStore cs = new BasicCookieStore(); cs.addCookies(cookies.toArray(new Cookie[cookies.size()])); httpContext.setAttribute(HttpClientContext.COOKIE_STORE, cs); }
@Override public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException { CloseableHttpClient client = (CloseableHttpClient) getHttpClient(); Assert.state(client != null, "Synchronous execution requires an HttpClient to be set"); HttpUriRequest httpRequest = createHttpUriRequest(httpMethod, uri); postProcessHttpRequest(httpRequest); HttpContext context = createHttpContext(httpMethod, uri); if (context == null) { context = HttpClientContext.create(); } // Request configuration not set in the context if (context.getAttribute(HttpClientContext.REQUEST_CONFIG) == null) { // Use request configuration given by the user, when available RequestConfig config = null; if (httpRequest instanceof Configurable) { config = ((Configurable) httpRequest).getConfig(); } if (config == null) { if (this.socketTimeout > 0 || this.connectTimeout > 0) { config = RequestConfig.custom() .setConnectTimeout(this.connectTimeout) .setSocketTimeout(this.socketTimeout) .build(); } else { config = RequestConfig.DEFAULT; } } context.setAttribute(HttpClientContext.REQUEST_CONFIG, config); } if (this.bufferRequestBody) { return new HttpComponentsClientHttpRequest(client, httpRequest, context); } else { return new HttpComponentsStreamingClientHttpRequest(client, httpRequest, context); } }
@Override public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { if (executionCount >= 3) {// 如果已经重试了3次,就放弃 return false; } if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试 return true; } if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常 return false; } if (exception instanceof InterruptedIOException) {// 超时 return true; } if (exception instanceof UnknownHostException) {// 目标服务器不可达 return false; } if (exception instanceof ConnectTimeoutException) {// 连接被拒绝 return false; } if (exception instanceof SSLException) {// ssl握手异常 return false; } HttpClientContext clientContext = HttpClientContext.adapt(context); HttpRequest request = clientContext.getRequest(); // 如果请求是幂等的,就再次尝试 if (!(request instanceof HttpEntityEnclosingRequest)) { return true; } return false; }
@Override public void authSucceeded(final HttpHost authhost, final AuthScheme authScheme, final HttpContext context) { final HttpClientContext clientContext = HttpClientContext.adapt(context); final Credentials credentials = clientContext.getAttribute(PROXY_CREDENTIALS_INPUT_ID, Credentials.class); if(null != credentials) { clientContext.removeAttribute(PROXY_CREDENTIALS_INPUT_ID); if(log.isInfoEnabled()) { log.info(String.format("Save passphrase for proxy %s", authhost)); } keychain.addCredentials(authhost.getHostName(), credentials.getUsername(), credentials.getPassword()); } super.authSucceeded(authhost, authScheme, context); }
/** * 执行请求 * * @date 2015年7月17日 * @return */ public ResponseWrap execute() { settingRequest(); if (httpClient == null) { httpClient = clientBuilder.build(); } try { HttpClientContext context = HttpClientContext.create(); CloseableHttpResponse response = httpClient.execute(request, context); return new ResponseWrap(httpClient, request, response, context, mapper); } catch (IOException e) { logger.error(e.getMessage(), e); throw new RuntimeException(e.getMessage(), e); } }
private static void addPreemptiveAuthenticationProxy(HttpClientContext clientContext, ProxyConfiguration proxyConfiguration) { if (proxyConfiguration.preemptiveBasicAuthenticationEnabled()) { HttpHost targetHost = new HttpHost(proxyConfiguration.endpoint().getHost(), proxyConfiguration.endpoint().getPort()); final CredentialsProvider credsProvider = newProxyCredentialsProvider(proxyConfiguration); // Create AuthCache instance AuthCache authCache = new BasicAuthCache(); // Generate BASIC scheme object and add it to the local auth cache BasicScheme basicAuth = new BasicScheme(); authCache.put(targetHost, basicAuth); clientContext.setCredentialsProvider(credsProvider); clientContext.setAuthCache(authCache); } }
/** * Returns a new HttpClientContext used for request execution. */ public static HttpClientContext newClientContext(HttpClientSettings settings, Map<String, ? extends Object> attributes) { final HttpClientContext clientContext = new HttpClientContext(); if (attributes != null && !attributes.isEmpty()) { for (Map.Entry<String, ?> entry : attributes.entrySet()) { clientContext.setAttribute(entry.getKey(), entry.getValue()); } } addPreemptiveAuthenticationProxy(clientContext, settings); return clientContext; }
private static void addPreemptiveAuthenticationProxy(HttpClientContext clientContext, HttpClientSettings settings) { if (settings.isPreemptiveBasicProxyAuth()) { HttpHost targetHost = new HttpHost(settings.getProxyHost(), settings .getProxyPort()); final CredentialsProvider credsProvider = newProxyCredentialsProvider(settings); // Create AuthCache instance AuthCache authCache = new BasicAuthCache(); // Generate BASIC scheme object and add it to the local auth cache BasicScheme basicAuth = new BasicScheme(); authCache.put(targetHost, basicAuth); clientContext.setCredentialsProvider(credsProvider); clientContext.setAuthCache(authCache); } }
@Override protected HttpHost determineProxy(HttpHost host, HttpRequest request, HttpContext context) throws HttpException { HttpClientContext httpClientContext = HttpClientContext.adapt(context); Proxy proxy = proxyPlanner.determineProxy(host, request, context, ipPool, crawlerSession); if (proxy == null) { return null; } if (log.isDebugEnabled()) { log.debug("{} 当前使用IP为:{}:{}", host.getHostName(), proxy.getIp(), proxy.getPort()); } context.setAttribute(VSCRAWLER_AVPROXY_KEY, proxy); if (proxy.getAuthenticationHeaders() != null) { for (Header header : proxy.getAuthenticationHeaders()) { request.addHeader(header); } } if (StringUtils.isNotEmpty(proxy.getUsername()) && StringUtils.isNotEmpty(proxy.getPassword())) { BasicCredentialsProvider credsProvider1 = new BasicCredentialsProvider(); httpClientContext.setCredentialsProvider(credsProvider1); credsProvider1.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword())); } return new HttpHost(proxy.getIp(), proxy.getPort()); }
@Override public CloseableHttpResponse execute(HttpRoute route, HttpRequestWrapper request, HttpClientContext clientContext, HttpExecutionAware execAware) throws IOException, HttpException { Proxy proxy = (Proxy) clientContext.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY); if (proxy != null) { proxy.recordUsage(); } try { return delegate.execute(route, request, clientContext, execAware); } catch (IOException ioe) { if (proxy != null) { proxy.recordFailed(); } throw ioe; } }
/** * Open the entry page. * * @param retryTimes retry times of qr scan */ void open(int retryTimes) { final String url = WECHAT_URL_ENTRY; HttpHeaders customHeader = new HttpHeaders(); customHeader.setPragma("no-cache"); customHeader.setCacheControl("no-cache"); customHeader.set("Upgrade-Insecure-Requests", "1"); customHeader.set(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); HeaderUtils.assign(customHeader, getHeader); restTemplate.exchange(url, HttpMethod.GET, new HttpEntity<>(customHeader), String.class); //manually insert two cookies into cookiestore, as they're supposed to be stored in browsers by javascript. CookieStore store = (CookieStore) ((StatefullRestTemplate) restTemplate).getHttpContext().getAttribute(HttpClientContext.COOKIE_STORE); Date maxDate = new Date(Long.MAX_VALUE); String domain = WECHAT_URL_ENTRY.replaceAll("https://", "").replaceAll("/", ""); Map<String, String> cookies = new HashMap<>(3); cookies.put("MM_WX_NOTIFY_STATE", "1"); cookies.put("MM_WX_SOUND_STATE", "1"); if (retryTimes > 0) { cookies.put("refreshTimes", String.valueOf(retryTimes)); } appendAdditionalCookies(store, cookies, domain, "/", maxDate); //It's now at entry page. this.originValue = WECHAT_URL_ENTRY; this.refererValue = WECHAT_URL_ENTRY.replaceAll("/$", ""); }
/** * Initialization * * @param hostUrl hostUrl * @param baseRequest baseRequest * @return current user's information and contact information * @throws IOException if the http response body can't be convert to {@link InitResponse} */ InitResponse init(String hostUrl, BaseRequest baseRequest) throws IOException { String url = String.format(WECHAT_URL_INIT, hostUrl, RandomUtils.generateDateWithBitwiseNot()); CookieStore store = (CookieStore) ((StatefullRestTemplate) restTemplate).getHttpContext().getAttribute(HttpClientContext.COOKIE_STORE); Date maxDate = new Date(Long.MAX_VALUE); String domain = hostUrl.replaceAll("https://", "").replaceAll("/", ""); Map<String, String> cookies = new HashMap<>(3); cookies.put("MM_WX_NOTIFY_STATE", "1"); cookies.put("MM_WX_SOUND_STATE", "1"); appendAdditionalCookies(store, cookies, domain, "/", maxDate); InitRequest request = new InitRequest(); request.setBaseRequest(baseRequest); HttpHeaders customHeader = new HttpHeaders(); customHeader.set(HttpHeaders.REFERER, hostUrl + "/"); customHeader.setOrigin(hostUrl); HeaderUtils.assign(customHeader, postHeader); ResponseEntity<String> responseEntity = restTemplate.exchange(url, HttpMethod.POST, new HttpEntity<>(request, customHeader), String.class); return jsonMapper.readValue(WechatUtils.textDecode(responseEntity.getBody()), InitResponse.class); }
/** * Reads content. * @param httpClient HTTP client * @param since since date * @return content reference * @throws IOException if reading content fails * @throws URISyntaxException if file url is an invalid URI */ public SimpleDataReference readContent(CloseableHttpClient httpClient, Date since) throws IOException, URISyntaxException { HttpGet method = new HttpGet(fileUrl.toExternalForm()); method.setConfig(DEFAULT_REQUEST_CONFIG); method.setHeader("User-Agent", HttpConstants.getUserAgent()); HttpClientContext context = creds!=null && !creds.isEmpty()? createHttpClientContext(fileUrl, creds): null; try (CloseableHttpResponse httpResponse = httpClient.execute(method,context); InputStream input = httpResponse.getEntity().getContent();) { if (httpResponse.getStatusLine().getStatusCode()>=400) { throw new HttpResponseException(httpResponse.getStatusLine().getStatusCode(), httpResponse.getStatusLine().getReasonPhrase()); } Date lastModifiedDate = readLastModifiedDate(httpResponse); MimeType contentType = readContentType(httpResponse); boolean readBody = since==null || lastModifiedDate==null || lastModifiedDate.getTime()>=since.getTime(); SimpleDataReference ref = new SimpleDataReference(broker.getBrokerUri(), broker.getEntityDefinition().getLabel(), fileUrl.toExternalForm(), lastModifiedDate, fileUrl.toURI()); ref.addContext(contentType, readBody? IOUtils.toByteArray(input): null); return ref; } }
@Test public void testMustNotAddMultipartByteRangeContentTypeTo416Response() throws Exception { originResponse = Proxies.enhanceResponse( new BasicHttpResponse(HttpVersion.HTTP_1_1, 416, "Requested Range Not Satisfiable")); EasyMock.expect( mockBackend.execute( EasyMock.isA(HttpRoute.class), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); if (result.getStatusLine().getStatusCode() == 416) { for (final Header h : result.getHeaders("Content-Type")) { for (final HeaderElement elt : h.getElements()) { Assert.assertFalse("multipart/byteranges".equalsIgnoreCase(elt.getName())); } } } }
@Test public void testSelectNoCredentialsProvider() throws Exception { final TargetAuthenticationStrategy authStrategy = new TargetAuthenticationStrategy(); final HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_UNAUTHORIZED, "UNAUTHORIZED"); final HttpHost authhost = new HttpHost("locahost", 80); final HttpClientContext context = HttpClientContext.create(); final Map<String, Header> challenges = new HashMap<String, Header>(); challenges.put("basic", new BasicHeader(AUTH.WWW_AUTH, "Basic realm=\"test\"")); challenges.put("digest", new BasicHeader(AUTH.WWW_AUTH, "Digest realm=\"realm1\", nonce=\"1234\"")); final Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create() .register("basic", new BasicSchemeFactory()) .register("digest", new DigestSchemeFactory()).build(); context.setAuthSchemeRegistry(authSchemeRegistry); final Queue<AuthOption> options = authStrategy.select(challenges, authhost, response, context); Assert.assertNotNull(options); Assert.assertEquals(0, options.size()); }