@Test(expected=ClientProtocolException.class) public void testMaxRedirectCheck() throws Exception { this.serverBootstrap.registerHandler("*", new CircularRedirectService()); final HttpHost target = start(); final RequestConfig config = RequestConfig.custom() .setCircularRedirectsAllowed(true) .setMaxRedirects(5) .build(); final HttpGet httpget = new HttpGet("/circular-oldlocation/"); httpget.setConfig(config); try { this.httpclient.execute(target, httpget); } catch (final ClientProtocolException e) { Assert.assertTrue(e.getCause() instanceof RedirectException); throw e; } }
@Test(expected = RedirectException.class) public void testMaxRedirect() throws Exception { final HttpRoute route = new HttpRoute(target); final HttpGet get = new HttpGet("/test"); final HttpRequestWrapper request = HttpRequestWrapper.wrap(get); final HttpClientContext context = HttpClientContext.create(); final RequestConfig config = RequestConfig.custom() .setRedirectsEnabled(true) .setMaxRedirects(3) .build(); context.setRequestConfig(config); final CloseableHttpResponse response1 = Mockito.mock(CloseableHttpResponse.class); final HttpGet redirect = new HttpGet("http://localhost:80/redirect"); Mockito.when(requestExecutor.execute( Mockito.eq(route), Mockito.<HttpRequestWrapper>any(), Mockito.<HttpClientContext>any(), Mockito.<HttpExecutionAware>any())).thenReturn(response1); Mockito.when(redirectStrategy.isRedirected( Mockito.<HttpRequestWrapper>any(), Mockito.<HttpResponse>any(), Mockito.<HttpClientContext>any())).thenReturn(Boolean.TRUE); Mockito.when(redirectStrategy.getRedirect( Mockito.<HttpRequestWrapper>any(), Mockito.<HttpResponse>any(), Mockito.<HttpClientContext>any())).thenReturn(redirect); Mockito.when(httpRoutePlanner.determineRoute( Mockito.eq(target), Mockito.<HttpRequestWrapper>any(), Mockito.<HttpClientContext>any())).thenReturn(route); redirectExec.execute(route, request, context, execAware); }
@Test public void testRedirectSync() throws Exception { proxy = new SimpleFixedHttpProxy(); URI robotsURL0 = URI.create("http://foo.bar/robots.txt"); URI robotsURL1 = URI.create("http://foo.bar/fubar/robots.txt"); proxy.addNon200(robotsURL0, "HTTP/1.1 301 Moved Permanently\nLocation: " + robotsURL1 + "\n", ""); proxy.add200(robotsURL1, "", "# goodguy can do anything\n" + "User-agent: goodguy\n" + "Disallow:\n\n" + "# every other guy can do nothing\n" + "User-agent: *\n" + "Disallow: /\n" ); URI url = URI.create("http://foo.bar/goo/zoo.html"); // Disallowed proxy.add200(url, "", "Should not be crawled..."); proxy.addNon200(URI.create("http://too.many/robots.txt"), "HTTP/1.1 301 Moved Permanently\nLocation: http://too.many/0\n", ""); for(int i = 0; i < 5; i++) proxy.addNon200(URI.create("http://too.many/" + i), "HTTP/1.1 301 Moved Permanently\nLocation: http://too.many/" + (i + 1) + "\n", ""); proxy.start(); HttpClient httpClient = FetchDataTest.getHttpClient(new HttpHost("localhost", proxy.port()), true); FetchData fetchData = new FetchData(Helpers.getTestConfiguration(this)); fetchData.fetch(URI.create(BURL.schemeAndAuthority(url) + "/robots.txt"), httpClient, null, null, true); char[][] filter = URLRespectsRobots.parseRobotsResponse(fetchData, "goodGuy"); assertTrue(URLRespectsRobots.apply(filter, url)); filter = URLRespectsRobots.parseRobotsResponse(fetchData, "badGuy"); assertFalse(URLRespectsRobots.apply(filter, url)); filter = URLRespectsRobots.parseRobotsResponse(fetchData, "goodGuy foo"); assertTrue(URLRespectsRobots.apply(filter, url)); filter = URLRespectsRobots.parseRobotsResponse(fetchData, "badGuy foo"); assertFalse(URLRespectsRobots.apply(filter, url)); fetchData = new FetchData(Helpers.getTestConfiguration(this)); fetchData.fetch(URI.create("http://too.many/robots.txt"), httpClient, null, null, true); assertTrue(fetchData.exception instanceof RedirectException); fetchData.close(); }