@Override public Proxy determineProxy(HttpHost host, HttpRequest request, HttpContext context, IPPool ipPool, CrawlerSession crawlerSession) { HttpClientContext httpClientContext = HttpClientContext.adapt(context); Proxy proxy = (Proxy) crawlerSession.getExtInfo(VSCRAWLER_AVPROXY_KEY); if (proxy == null) { String accessUrl = null; if (request instanceof HttpRequestWrapper || request instanceof HttpGet) { accessUrl = HttpUriRequest.class.cast(request).getURI().toString(); } if (!PoolUtil.isDungProxyEnabled(httpClientContext)) { log.info("{}不会被代理", accessUrl); return null; } proxy = ipPool.getIP(host.getHostName(), accessUrl); if (proxy == null) { return null; } crawlerSession.setExtInfo(VSCRAWLER_AVPROXY_KEY, proxy); } return proxy; }
@Override public CloseableHttpResponse execute(HttpRoute route, HttpRequestWrapper request, HttpClientContext clientContext, HttpExecutionAware execAware) throws IOException, HttpException { Proxy proxy = (Proxy) clientContext.getAttribute(VSCrawlerConstant.VSCRAWLER_AVPROXY_KEY); if (proxy != null) { proxy.recordUsage(); } try { return delegate.execute(route, request, clientContext, execAware); } catch (IOException ioe) { if (proxy != null) { proxy.recordFailed(); } throw ioe; } }
@Override public HttpUriRequest getRedirect(HttpRequest request, HttpResponse response, HttpContext context) throws ProtocolException { URI uri = getLocationURI(request, response, context); String method = request.getRequestLine().getMethod(); if ("post".equalsIgnoreCase(method)) { try { HttpRequestWrapper httpRequestWrapper = (HttpRequestWrapper) request; httpRequestWrapper.setURI(uri); httpRequestWrapper.removeHeaders("Content-Length"); return httpRequestWrapper; } catch (Exception e) { logger.error("强转为HttpRequestWrapper出错"); } return new HttpPost(uri); } else { return new HttpGet(uri); } }
@Test public void testResponsesToTRACEsAreNotCached() throws Exception { emptyMockCacheExpectsNoPuts(); request = HttpRequestWrapper.wrap(new BasicHttpRequest("TRACE", "/", HttpVersion.HTTP_1_1)); originResponse.setHeader("Cache-Control", "max-age=3600"); EasyMock.expect( mockBackend.execute( EasyMock.isA(HttpRoute.class), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); impl.execute(route, request, context, null); verifyMocks(); }
@Test(expected = IOException.class) public void testPostProcessIOException() throws Exception { final HttpRoute route = new HttpRoute(target); final HttpRequestWrapper request = HttpRequestWrapper.wrap(new HttpGet("/test")); final HttpClientContext context = HttpClientContext.create(); final CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class); Mockito.when(requestExecutor.execute( Mockito.eq(route), Mockito.<HttpRequestWrapper>any(), Mockito.<HttpClientContext>any(), Mockito.<HttpExecutionAware>any())).thenReturn(response); Mockito.doThrow(new IOException("Ooopsie")).when(httpProcessor).process( Mockito.same(response), Mockito.<HttpContext>any()); try { protocolExec.execute(route, request, context, execAware); } catch (final Exception ex) { Mockito.verify(response).close(); throw ex; } }
@Test public void testNoEntityForIfNoneMatchRequestNotYetInCache() throws Exception { final Date now = new Date(); final Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L); impl = createCachingExecChain(mockBackend, new BasicHttpCache(), CacheConfig.DEFAULT); final HttpRequestWrapper req1 = HttpRequestWrapper.wrap(new HttpGet( "http://foo.example.com/")); req1.addHeader("If-None-Match", "\"etag\""); final HttpResponse resp1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_NOT_MODIFIED, "Not modified"); resp1.setHeader("Content-Length", "128"); resp1.setHeader("ETag", "\"etag\""); resp1.setHeader("Date", DateUtils.formatDate(tenSecondsAgo)); resp1.setHeader("Cache-Control", "public, max-age=5"); backendExpectsAnyRequestAndReturn(resp1); replayMocks(); final HttpResponse result = impl.execute(route, req1, context, null); verifyMocks(); assertEquals(HttpStatus.SC_NOT_MODIFIED, result.getStatusLine().getStatusCode()); assertNull("The 304 response messages MUST NOT contain a message-body", result.getEntity()); }
@Test public void testFundamentals() throws Exception { final HttpRoute route = new HttpRoute(target); final HttpRequestWrapper request = HttpRequestWrapper.wrap(new HttpGet("/test")); final HttpClientContext context = HttpClientContext.create(); final CloseableHttpResponse response = Mockito.mock(CloseableHttpResponse.class); Mockito.when(requestExecutor.execute( Mockito.eq(route), Mockito.<HttpRequestWrapper>any(), Mockito.<HttpClientContext>any(), Mockito.<HttpExecutionAware>any())).thenReturn(response); protocolExec.execute(route, request, context, execAware); Mockito.verify(httpProcessor).process(request, context); Mockito.verify(requestExecutor).execute(route, request, context, execAware); Mockito.verify(httpProcessor).process(response, context); Assert.assertEquals(new HttpHost("foo", 80), context.getTargetHost()); Assert.assertEquals(target, context.getTargetHost()); Assert.assertEquals(route, context.getHttpRoute()); Assert.assertSame(request, context.getRequest()); Assert.assertSame(response, context.getResponse()); }
private HttpResponse handleCacheHit(final HttpHost target, final HttpRequestWrapper request, final HttpContext context, final HttpCacheEntry entry) throws ClientProtocolException, IOException { recordCacheHit(target, request); HttpResponse out = null; final Date now = getCurrentDate(); if (suitabilityChecker.canCachedResponseBeUsed(target, request, entry, now)) { log.debug("Cache hit"); out = generateCachedResponse(request, context, entry, now); } else if (!mayCallBackend(request)) { log.debug("Cache entry not suitable but only-if-cached requested"); out = generateGatewayTimeout(context); } else { log.debug("Revalidating cache entry"); return revalidateCacheEntry(target, request, context, entry, now); } if (context != null) { context.setAttribute(ExecutionContext.HTTP_TARGET_HOST, target); context.setAttribute(ExecutionContext.HTTP_REQUEST, request); context.setAttribute(ExecutionContext.HTTP_RESPONSE, out); context.setAttribute(ExecutionContext.HTTP_REQ_SENT, Boolean.TRUE); } return out; }
@Test public void testStaleIfErrorInRequestIsFalseReturnsError() throws Exception{ final Date now = new Date(); final Date tenSecondsAgo = new Date(now.getTime() - 10 * 1000L); final HttpRequestWrapper req1 = HttpRequestWrapper.wrap(HttpTestUtils.makeDefaultRequest()); final HttpResponse resp1 = HttpTestUtils.make200Response(tenSecondsAgo, "public, max-age=5"); backendExpectsAnyRequestAndReturn(resp1); final HttpRequestWrapper req2 = HttpRequestWrapper.wrap(HttpTestUtils.makeDefaultRequest()); req2.setHeader("Cache-Control","stale-if-error=2"); final HttpResponse resp2 = HttpTestUtils.make500Response(); backendExpectsAnyRequestAndReturn(resp2); replayMocks(); impl.execute(route, req1, context, null); final HttpResponse result = impl.execute(route, req2, context, null); verifyMocks(); assertEquals(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.getStatusLine().getStatusCode()); }
@Test public void testExpiresHeaderEqualToDateHeaderIsTreatedAsStale() throws Exception { final HttpRequestWrapper req1 = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1)); final HttpResponse resp1 = HttpTestUtils.make200Response(); resp1.setHeader("Cache-Control","public"); resp1.setHeader("ETag","\"etag\""); resp1.setHeader("Expires", resp1.getFirstHeader("Date").getValue()); final HttpRequestWrapper req2 = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1)); final HttpResponse resp2 = HttpTestUtils.make200Response(); backendExpectsAnyRequestAndReturn(resp1); // second request to origin MUST happen backendExpectsAnyRequestAndReturn(resp2); replayMocks(); impl.execute(route, req1, context, null); impl.execute(route, req2, context, null); verifyMocks(); }
@Test public void testPassesOnOrigin401ResponseWithoutWWWAuthenticateHeader() throws Exception { originResponse = Proxies.enhanceResponse( new BasicHttpResponse(HTTP_1_1, 401, "Unauthorized")); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, HttpRequestWrapper.wrap(request), context, null); verifyMocks(); Assert.assertSame(originResponse, result); }
@Test public void testPassesOnOrigin405WithoutAllowHeader() throws Exception { originResponse = Proxies.enhanceResponse( new BasicHttpResponse(HTTP_1_1, 405, "Method Not Allowed")); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, HttpRequestWrapper.wrap(request), context, null); verifyMocks(); Assert.assertSame(originResponse, result); }
@Test public void cacheHitOkWithOnlyIfCached() throws Exception { final HttpRequestWrapper req1 = HttpRequestWrapper.wrap(HttpTestUtils.makeDefaultRequest()); final HttpResponse resp1 = HttpTestUtils.make200Response(); resp1.setHeader("Cache-Control","max-age=3600"); backendExpectsAnyRequestAndReturn(resp1); final HttpRequestWrapper req2 = HttpRequestWrapper.wrap(HttpTestUtils.makeDefaultRequest()); req2.setHeader("Cache-Control", "only-if-cached"); replayMocks(); impl.execute(route, req1, context, null); final HttpResponse result = impl.execute(route, req2, context, null); verifyMocks(); assertTrue(HttpTestUtils.semanticallyTransparent(resp1, result)); }
@Test public void consumesPartialContentFromOriginEvenIfNotRequested() throws Exception { final HttpRequestWrapper wrapper = HttpRequestWrapper.wrap(new HttpGet("http://foo.example.com/")); final int nbytes = 128; final HttpResponse resp = makePartialResponse(nbytes); final Flag closed = new Flag(); final ByteArrayInputStream bais = makeTrackableBody(nbytes, closed); resp.setEntity(new InputStreamEntity(bais, -1)); try { impl.ensureProtocolCompliance(wrapper, resp); } catch (final ClientProtocolException expected) { } assertTrue(closed.set || bais.read() == -1); }
@Test public void testUnknownHeadersOnResponsesAreForwarded() throws Exception { originResponse.addHeader("X-Unknown-Header", "blahblah"); EasyMock.expect( mockBackend.execute( EasyMock.isA(HttpRoute.class), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); final Header[] hdrs = result.getHeaders("X-Unknown-Header"); Assert.assertEquals(1, hdrs.length); Assert.assertEquals("blahblah", hdrs[0].getValue()); }
@Test public void testSetsCacheMissContextIfRequestNotServableFromCache() throws Exception { impl = createCachingExecChain(mockBackend, new BasicHttpCache(), CacheConfig.DEFAULT); final HttpRequestWrapper req = HttpRequestWrapper.wrap(new HttpGet( "http://foo.example.com/")); req.setHeader("Cache-Control", "no-cache"); final HttpResponse resp = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_NO_CONTENT, "No Content"); backendExpectsAnyRequestAndReturn(resp); replayMocks(); impl.execute(route, req, context, null); verifyMocks(); Assert.assertEquals(CacheResponseStatus.CACHE_MISS, context.getCacheResponseStatus()); }
@Test public void test204ResponsesDoNotContainMessageBodies() throws Exception { originResponse = Proxies.enhanceResponse( new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_NO_CONTENT, "No Content")); originResponse.setEntity(HttpTestUtils.makeBody(entityLength)); EasyMock.expect( mockBackend.execute( EasyMock.isA(HttpRoute.class), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); Assert.assertTrue(result.getEntity() == null || result.getEntity().getContentLength() == 0); }
@Test public void testNonCacheableResponseIsNotCachedAndIsReturnedAsIs() throws Exception { final CacheConfig configDefault = CacheConfig.DEFAULT; impl = createCachingExecChain(mockBackend, new BasicHttpCache(new HeapResourceFactory(), mockStorage, configDefault), configDefault); final HttpRequestWrapper req1 = HttpRequestWrapper.wrap(HttpTestUtils.makeDefaultRequest()); final HttpResponse resp1 = HttpTestUtils.make200Response(); resp1.setHeader("Cache-Control", "no-cache"); expect(mockStorage.getEntry(isA(String.class))).andReturn(null).anyTimes(); mockStorage.removeEntry(isA(String.class)); expectLastCall().anyTimes(); backendExpectsAnyRequestAndReturn(resp1); replayMocks(); final HttpResponse result = impl.execute(route, req1, context, null); verifyMocks(); assertTrue(HttpTestUtils.semanticallyTransparent(resp1, result)); }
private CloseableHttpResponse handleCacheMiss( final HttpRoute route, final HttpRequestWrapper request, final HttpClientContext context, final HttpExecutionAware execAware) throws IOException, HttpException { final HttpHost target = context.getTargetHost(); recordCacheMiss(target, request); if (!mayCallBackend(request)) { return Proxies.enhanceResponse( new BasicHttpResponse( HttpVersion.HTTP_1_1, HttpStatus.SC_GATEWAY_TIMEOUT, "Gateway Timeout")); } final Map<String, Variant> variants = getExistingCacheVariants(target, request); if (variants != null && !variants.isEmpty()) { return negotiateResponseFromVariants(route, request, context, execAware, variants); } return callBackend(route, request, context, execAware); }
private void testDoesNotModifyHeaderOnRequest(final String header, final String value) throws Exception { final BasicHttpEntityEnclosingRequest req = new BasicHttpEntityEnclosingRequest("POST","/",HttpVersion.HTTP_1_1); req.setEntity(HttpTestUtils.makeBody(128)); req.setHeader("Content-Length","128"); req.setHeader(header,value); final Capture<HttpRequestWrapper> cap = new Capture<HttpRequestWrapper>(); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), EasyMock.capture(cap), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); impl.execute(route, HttpRequestWrapper.wrap(req), context, null); verifyMocks(); final HttpRequest captured = cap.getValue(); Assert.assertEquals(value, captured.getFirstHeader(header).getValue()); }
private void testOrderOfMultipleHeadersIsPreservedOnRequests(final String h, final HttpRequestWrapper request) throws Exception { final Capture<HttpRequestWrapper> reqCapture = new Capture<HttpRequestWrapper>(); EasyMock.expect( mockBackend.execute( EasyMock.isA(HttpRoute.class), EasyMock.capture(reqCapture), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); impl.execute(route, request, context, null); verifyMocks(); final HttpRequest forwarded = reqCapture.getValue(); Assert.assertNotNull(forwarded); final String expected = HttpTestUtils.getCanonicalHeaderValue(request, h); final String actual = HttpTestUtils.getCanonicalHeaderValue(forwarded, h); if (!actual.contains(expected)) { Assert.assertEquals(expected, actual); } }
@Test(expected=HttpException.class) public void testExecHttpException() throws Exception { final HttpRoute route = new HttpRoute(target); final HttpClientContext context = new HttpClientContext(); final HttpRequestWrapper request = HttpRequestWrapper.wrap(new HttpGet("http://bar/test")); Mockito.when(requestExecutor.execute( Mockito.<HttpRequest>any(), Mockito.<HttpClientConnection>any(), Mockito.<HttpClientContext>any())).thenThrow(new HttpException("Ka-boom")); try { minimalClientExec.execute(route, request, context, execAware); } catch (final Exception ex) { Mockito.verify(connManager).releaseConnection(managedConn, null, 0, TimeUnit.MILLISECONDS); throw ex; } }
private HttpResponse testRequestWithWeakETagValidatorIsNotAllowed(final String header) throws Exception { final Capture<HttpRequestWrapper> cap = new Capture<HttpRequestWrapper>(); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), EasyMock.capture(cap), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse).times(0, 1); replayMocks(); final HttpResponse response = impl.execute(route, request, context, null); verifyMocks(); // it's probably ok to return a 400 (Bad Request) to this client if (cap.hasCaptured()) { final HttpRequest forwarded = cap.getValue(); final Header h = forwarded.getFirstHeader(header); if (h != null) { Assert.assertFalse(h.getValue().startsWith("W/")); } } return response; }
@Test(expected=RuntimeException.class) public void testExecRuntimeException() throws Exception { final HttpRoute route = new HttpRoute(target); final HttpClientContext context = new HttpClientContext(); final HttpRequestWrapper request = HttpRequestWrapper.wrap(new HttpGet("http://bar/test")); Mockito.when(requestExecutor.execute( Mockito.<HttpRequest>any(), Mockito.<HttpClientConnection>any(), Mockito.<HttpClientContext>any())).thenThrow(new RuntimeException("Ka-boom")); try { minimalClientExec.execute(route, request, context, execAware); } catch (final Exception ex) { Mockito.verify(connManager).releaseConnection(managedConn, null, 0, TimeUnit.MILLISECONDS); throw ex; } }
@Test public void testExecute() throws Exception { final HttpGet httpget = new HttpGet("http://somehost/stuff"); final HttpRoute route = new HttpRoute(new HttpHost("somehost", 80)); final ArgumentCaptor<HttpRequestWrapper> argcap = ArgumentCaptor.forClass(HttpRequestWrapper.class); Mockito.when(routePlanner.determineRoute( Mockito.eq(new HttpHost("somehost")), argcap.capture(), Mockito.<HttpClientContext>any())).thenReturn(route); client.execute(httpget); Assert.assertNotNull(argcap.getValue()); Assert.assertSame(httpget, argcap.getValue().getOriginal()); Mockito.verify(execChain).execute( Mockito.same(route), Mockito.<HttpRequestWrapper>any(), Mockito.<HttpClientContext>any(), Mockito.same(httpget)); }
/** * If the {@link HttpRequest} is non-compliant but 'fixable' we go ahead and * fix the request here. * * @param request the request to check for compliance * @throws ClientProtocolException when we have trouble making the request compliant */ public void makeRequestCompliant(final HttpRequestWrapper request) throws ClientProtocolException { if (requestMustNotHaveEntity(request)) { ((HttpEntityEnclosingRequest) request).setEntity(null); } verifyRequestWithExpectContinueFlagHas100continueHeader(request); verifyOPTIONSRequestWithBodyHasContentType(request); decrementOPTIONSMaxForwardsIfGreaterThen0(request); stripOtherFreshnessDirectivesWithNoCache(request); if (requestVersionIsTooLow(request) || requestMinorVersionIsTooHighMajorVersionsMatch(request)) { request.setProtocolVersion(HttpVersion.HTTP_1_1); } }
@Test public void testHigherMajorProtocolVersionsOnRequestSwitchToTunnelBehavior() throws Exception { // tunnel behavior: I don't muck with request or response in // any way request = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/foo", new ProtocolVersion("HTTP", 2, 13))); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), eqRequest(request), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); Assert.assertSame(originResponse, result); }
@Test public void testHigher1_XProtocolVersionsDowngradeTo1_1() throws Exception { request = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/foo", new ProtocolVersion("HTTP", 1, 2))); final HttpRequestWrapper downgraded = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/foo", HttpVersion.HTTP_1_1)); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), eqRequest(downgraded), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); Assert.assertTrue(HttpTestUtils.semanticallyTransparent(originResponse, result)); }
@Test public void testResponseToAHEADRequestMustNotHaveABody() throws Exception { request = HttpRequestWrapper.wrap(new BasicHttpRequest("HEAD", "/", HttpVersion.HTTP_1_1)); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); Assert.assertTrue(result.getEntity() == null || result.getEntity().getContentLength() == 0); }
@Test public void testRequestsWithLowerProtocolVersionsGetUpgradedTo1_1() throws Exception { request = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/foo", new ProtocolVersion("HTTP", 1, 0))); final HttpRequestWrapper upgraded = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/foo", HttpVersion.HTTP_1_1)); EasyMock.expect( mockBackend.execute( EasyMock.eq(route), eqRequest(upgraded), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); Assert.assertTrue(HttpTestUtils.semanticallyTransparent(originResponse, result)); }
@Test public void testTransferCodingsAreNotSentToAnHTTP_1_0Client() throws Exception { originResponse.setHeader("Transfer-Encoding", "identity"); request = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/foo", new ProtocolVersion("HTTP", 1, 0))); EasyMock.expect( mockBackend.execute( EasyMock.isA(HttpRoute.class), EasyMock.isA(HttpRequestWrapper.class), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn(originResponse); replayMocks(); final HttpResponse result = impl.execute(route, request, context, null); verifyMocks(); Assert.assertNull(result.getFirstHeader("TE")); Assert.assertNull(result.getFirstHeader("Transfer-Encoding")); }
@Test public void responseToGetWithQueryFrom1_0OriginVia1_1ProxyAndNoExpiresIsNotCached() throws Exception { final HttpRequestWrapper req2 = HttpRequestWrapper.wrap( new HttpGet("http://foo.example.com/bar?baz=quux")); final HttpResponse resp2 = new BasicHttpResponse(HttpVersion.HTTP_1_0, HttpStatus.SC_OK, "OK"); resp2.setEntity(HttpTestUtils.makeBody(200)); resp2.setHeader("Content-Length","200"); resp2.setHeader("Date", DateUtils.formatDate(now)); resp2.setHeader("Via","1.0 someproxy"); backendExpectsAnyRequestAndReturn(resp2); replayMocks(); impl.execute(route, req2, context, null); verifyMocks(); }
@Override public void onRequest(HttpRequestWrapper request, HttpContext httpContext, BaseSpan<?> span) { Tags.HTTP_METHOD.set(span, request.getRequestLine().getMethod()); URI uri = request.getURI(); Tags.HTTP_URL.set(span, request.getRequestLine().getUri()); Tags.PEER_PORT.set(span, uri.getPort() == -1 ? 80 : uri.getPort()); Tags.PEER_HOSTNAME.set(span, uri.getHost()); }
@Override public CloseableHttpResponse execute( HttpRoute route, HttpRequestWrapper request, HttpClientContext clientContext, HttpExecutionAware execAware) throws IOException, HttpException { ActiveSpan localSpan = clientContext.getAttribute(ACTIVE_SPAN, ActiveSpan.class); CloseableHttpResponse response = null; try { if (localSpan == null) { localSpan = handleLocalSpan(request, clientContext); } return (response = handleNetworkProcessing(localSpan, route, request, clientContext, execAware)); } catch (Exception e) { localSpan.deactivate(); throw e; } finally { if (response != null) { /** * This exec runs after {@link org.apache.http.impl.execchain.RedirectExec} which loops * until there is no redirect or reaches max redirect count. * {@link RedirectStrategy} is used to decide whether localSpan should be finished or not. * If there is a redirect localSpan is not finished and redirect is logged. */ Integer redirectCount = clientContext.getAttribute(REDIRECT_COUNT, Integer.class); if (!redirectHandlingDisabled && clientContext.getRequestConfig().isRedirectsEnabled() && redirectStrategy.isRedirected(request, response, clientContext) && ++redirectCount < clientContext.getRequestConfig().getMaxRedirects()) { clientContext.setAttribute(REDIRECT_COUNT, redirectCount); } else { localSpan.deactivate(); } } } }
@Test public void testReturns200ForIfNoneMatchHeaderFails() throws Exception { impl = createCachingExecChain(mockBackend, new BasicHttpCache(), CacheConfig.DEFAULT); final HttpRequestWrapper req1 = HttpRequestWrapper.wrap(new HttpGet( "http://foo.example.com/")); final HttpRequestWrapper req2 = HttpRequestWrapper.wrap(new HttpGet( "http://foo.example.com/")); final HttpResponse resp1 = new BasicHttpResponse(HttpVersion.HTTP_1_1, HttpStatus.SC_OK, "OK"); resp1.setEntity(HttpTestUtils.makeBody(128)); resp1.setHeader("Content-Length", "128"); resp1.setHeader("ETag", "\"etag\""); resp1.setHeader("Date", DateUtils.formatDate(new Date())); resp1.setHeader("Cache-Control", "public, max-age=3600"); req2.addHeader("If-None-Match", "\"abc\""); final HttpResponse resp2 = HttpTestUtils.make200Response(); backendExpectsAnyRequestAndReturn(resp1); backendExpectsAnyRequestAndReturn(resp2); replayMocks(); impl.execute(route, req1, context, null); final HttpResponse result = impl.execute(route, req2, context, null); verifyMocks(); Assert.assertEquals(200, result.getStatusLine().getStatusCode()); }
@Test public void testExecRequestNonPersistentConnection() throws Exception { final HttpRoute route = new HttpRoute(target); final HttpClientContext context = new HttpClientContext(); final RequestConfig config = RequestConfig.custom() .setConnectTimeout(123) .setSocketTimeout(234) .setConnectionRequestTimeout(345) .build(); context.setRequestConfig(config); final HttpRequestWrapper request = HttpRequestWrapper.wrap(new HttpGet("http://bar/test")); final HttpResponse response = new BasicHttpResponse(HttpVersion.HTTP_1_1, 200, "OK"); Mockito.when(requestExecutor.execute( Mockito.same(request), Mockito.<HttpClientConnection>any(), Mockito.<HttpClientContext>any())).thenReturn(response); final CloseableHttpResponse finalResponse = minimalClientExec.execute( route, request, context, execAware); Mockito.verify(connManager).requestConnection(route, null); Mockito.verify(connRequest).get(345, TimeUnit.MILLISECONDS); Mockito.verify(execAware, Mockito.times(1)).setCancellable(connRequest); Mockito.verify(execAware, Mockito.times(2)).setCancellable(Mockito.<Cancellable>any()); Mockito.verify(connManager).connect(managedConn, route, 123, context); Mockito.verify(connManager).routeComplete(managedConn, route, context); Mockito.verify(managedConn).setSocketTimeout(234); Mockito.verify(requestExecutor, Mockito.times(1)).execute(request, managedConn, context); Mockito.verify(managedConn, Mockito.times(1)).close(); Mockito.verify(connManager).releaseConnection(managedConn, null, 0, TimeUnit.MILLISECONDS); Assert.assertSame(managedConn, context.getConnection()); Assert.assertNotNull(finalResponse); Assert.assertTrue(finalResponse instanceof HttpResponseProxy); }
@Test public void testDoesNotModifyIfMatchOnRequests() throws Exception { request = HttpRequestWrapper.wrap(HttpRequestWrapper.wrap( new BasicHttpRequest("DELETE", "/", HttpVersion.HTTP_1_1))); request.setHeader("If-Match", "\"etag\""); testDoesNotModifyHeaderOnRequests("If-Match"); }
@Override public <T> T execute(HttpUriRequest request, ResponseHandler<? extends T> responseHandler) throws IOException, ClientProtocolException { HttpRequestWrapper wrap = HttpRequestWrapper.wrap(request); adviseRobotsTxt(wrap.getURI()); wrap.setURI(applyPHP(wrap.getURI())); return client.execute(wrap, responseHandler); }
protected void testCacheIsNotUsedWhenRespondingToRequest(final HttpRequestWrapper req) throws Exception { final HttpRequestWrapper req1 = HttpRequestWrapper.wrap( new BasicHttpRequest("GET", "/", HttpVersion.HTTP_1_1)); final HttpResponse resp1 = HttpTestUtils.make200Response(); resp1.setHeader("Etag","\"etag\""); resp1.setHeader("Cache-Control","max-age=3600"); backendExpectsAnyRequestAndReturn(resp1); final HttpResponse resp2 = HttpTestUtils.make200Response(); resp2.setHeader("Etag","\"etag2\""); resp2.setHeader("Cache-Control","max-age=1200"); final Capture<HttpRequestWrapper> cap = new Capture<HttpRequestWrapper>(); EasyMock.expect(mockBackend.execute( EasyMock.eq(route), EasyMock.capture(cap), EasyMock.isA(HttpClientContext.class), EasyMock.<HttpExecutionAware>isNull())).andReturn( Proxies.enhanceResponse(resp2)); replayMocks(); impl.execute(route, req1, context, null); final HttpResponse result = impl.execute(route, req, context, null); verifyMocks(); Assert.assertTrue(HttpTestUtils.semanticallyTransparent(resp2, result)); final HttpRequest captured = cap.getValue(); Assert.assertTrue(HttpTestUtils.equivalent(req, captured)); }
@Test(expected=ClientProtocolException.class) public void throwsExceptionIfOriginReturnsPartialResponseWhenNotRequested() throws Exception { final HttpRequestWrapper wrapper = HttpRequestWrapper.wrap(new HttpGet("http://foo.example.com/")); final int nbytes = 128; final HttpResponse resp = makePartialResponse(nbytes); resp.setEntity(HttpTestUtils.makeBody(nbytes)); impl.ensureProtocolCompliance(wrapper, resp); }