public static void sendConnectionErrorMessage(IDiscordClient client, IChannel channel, String command, @Nullable String message, @NotNull HttpStatusException httpe) throws RateLimitException, DiscordException, MissingPermissionsException { @NotNull String problem = message != null ? message + "\n" : ""; if (httpe.getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE) { problem += "Service unavailable, please try again latter."; } else if (httpe.getStatusCode() == HttpStatus.SC_FORBIDDEN) { problem += "Acess dennied."; } else if (httpe.getStatusCode() == HttpStatus.SC_NOT_FOUND) { problem += "Not Found"; } else { problem += httpe.getStatusCode() + SPACE + httpe.getMessage(); } new MessageBuilder(client) .appendContent("Error during HTTP Connection ", MessageBuilder.Styles.BOLD) .appendContent("\n") .appendContent(EventManager.MAIN_COMMAND_NAME, MessageBuilder.Styles.BOLD) .appendContent(SPACE) .appendContent(command, MessageBuilder.Styles.BOLD) .appendContent("\n") .appendContent(problem, MessageBuilder.Styles.BOLD) .withChannel(channel) .send(); }
static void Wikipedia(String dico) { Document significatowikipedia = null; String cercowikipedia = dico.substring((dico.indexOf("'")) + 1, (dico.lastIndexOf("'"))); try { significatowikipedia = Jsoup.connect("https://it.wikipedia.org/wiki/" + cercowikipedia.replace(" ", "_")).userAgent("Mozilla").get(); String divs = significatowikipedia.select("p").text(); if (!divs.equals("")) { new GUI().giveResponse("La ricerca di " + cercowikipedia + " su wikipedia ha restituito il seguente risultato:" + '\n' + divs); } else { new GUI().giveResponse("Mi dispiace, non ho trovato informazioni su " + cercowikipedia + " su Wikipedia..."); } } catch (HttpStatusException e) { new GUI().giveResponse("Mi dispiace, Wikipedia sembra non avere una voce per '" + cercowikipedia +"'..."); } catch (java.io.IOException f) { f.printStackTrace(); } catch (StringIndexOutOfBoundsException g) { new GUI().giveResponse("Ricorda che, perché io cerchi informazioni riguardo a qualcosa, occorre che tu la definisca fra due virgolette!"); } }
@Override public void parse(String url, DataProcessor dataProcessor) throws Exception { log.info("Parsing BandListPage - " + url); try { Document document = Jsoup.parse(new URL(url), TIMEOUT_MILLIS); for (Element element : document.select("div#mainarea > a")) { bandPage.parse(element.attr("href"), dataProcessor); } } catch (Exception e) { if (e instanceof HttpStatusException) { log.warn("Unable to fetch url - " + url + " - " + ((HttpStatusException)e).getStatusCode()); throw e; } else { log.warn("Unable to fetch url - " + url + " - " + e.getMessage()); } } }
@Override public void parse(String url, DataProcessor dataProcessor) throws Exception { log.info("Parsing IndexPage - " + url); try { Document document = Jsoup.parse(new URL(url), TIMEOUT_MILLIS); for (Element element : document.select("div#menu > a")) { bandListPage.parse(url + "?letter=" + parseQueryString(element.attr("href"), "&").get("letter"), dataProcessor); } } catch (Exception e) { if (e instanceof HttpStatusException) { log.warn("Unable to fetch url - " + url + " - " + ((HttpStatusException)e).getStatusCode()); throw e; } else { log.warn("Unable to fetch url - " + url + " - " + e.getMessage()); } } }
public static void manageSilentlyIOException(Exception e){ ClientConfig.setSentryContext(null, null, null, null); // First we try parsing the exception message to see if it contains the response code Matcher exMsgStatusCodeMatcher = Pattern.compile("^Server returned HTTP response code: (\\d+)") .matcher(e.getMessage()); if(exMsgStatusCodeMatcher.find()) { int statusCode = Integer.parseInt(exMsgStatusCodeMatcher.group(1)); if (statusCode >= 500 && statusCode < 600) LOG.warn("manageSilentlyIOException", e); else LOG.error("manageSilentlyIOException", e); } else if (e instanceof UnknownHostException || e instanceof SocketTimeoutException || e instanceof FileNotFoundException || e instanceof HttpStatusException || e instanceof NoRouteToHostException) LOG.warn("manageSilentlyIOException", e); else LOG.error("manageSilentlyIOException", e); }
private List<EarningsCalendar> parseYahoo(String link, Date lastLoadedEarningsCalendarDate) { List<EarningsCalendar> ret = new ArrayList<EarningsCalendar>(); Document document; try { document = downladPage(link); ret = newParser(lastLoadedEarningsCalendarDate, document); } catch (MalformedURLException e) { log.error("Unexpected IO error while getting list of earnings calendars " + link, e); } catch (IOException e1) { if (e1 instanceof HttpStatusException) { HttpStatusException se = (HttpStatusException) e1; if (se.getStatusCode() == 404) { throw new RuntimeException(se); } } log.error("Unexpected IO error while getting list of earnings calendars " + link, e1); } catch (Throwable t) { log.error("Unexpected error while getting list of earnings calendars " + link, t); } return ret; }
/** * * @param url = https://query1.finance.yahoo.com/v10/finance/quoteSummary/IQE.L?formatted=true&crumb=lXTeLpxBk2G&lang=en-US®ion=US&modules=incomeStatementHistory%2CcashflowStatementHistory%2CbalanceSheetHistory%2CincomeStatementHistoryQuarterly%2CcashflowStatementHistoryQuarterly%2CbalanceSheetHistoryQuarterly%2Cearnings&corsDomain=finance.yahoo.com * @returnurl = https://query1.finance.yahoo.com/v10/finance/quoteSummary/AAIF.L?formatted=true&modules=incomeStatementHistory%2CcashflowStatementHistory%2CbalanceSheetHistory%2CincomeStatementHistoryQuarterly%2CcashflowStatementHistoryQuarterly%2CbalanceSheetHistoryQuarterly%2Cearnings * @throws IOException * @throws JsonParseException * @throws JsonMappingException */ private YahooFinancialJsonDataModel downloadFinancialData(String url,String ticker) throws IOException, JsonParseException, JsonMappingException { String json = null; YahooFinancialJsonDataModel financial = null; try { json = Jsoup.connect(url).header("Accept-Encoding", "gzip, deflate") .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0") .maxBodySize(0).timeout(600000).ignoreContentType(true).execute().body(); } catch (HttpStatusException e) { String message = "Cannot find data at: "+url; log.error(message,e); financial = YahooFinancialJsonDataModel.errorData(ticker, new Date(), message+" due to: "+e.getMessage()); } if (json != null && financial == null) { if (json.indexOf("preferred") > 0) { System.out.println(json); log.info("\n" + json); } ObjectMapper mapper = new ObjectMapper(); financial = mapper.readValue(json, YahooFinancialJsonDataModel.class); } return financial; }
public SearchResult execute() throws IOException, ParserConfigurationException, SAXException { // System.out.println(getResponse().body().string().toString()); Response response = getResponse(); try { if (response.code() == 200) { //String str = response.body().string(); SAXParser saxParser = factory.newSAXParser(); ResultHandler handler = new ResultHandler(); try { saxParser.parse(new InputSource(response.body().byteStream()), handler); } catch (SAXException e) { //System.err.println(str); throw e; } return new SearchResult(this, handler.judgments); } else { throw new HttpStatusException("Code was not 200 but " + response.code(), response.code(), getRequest().url().toString()); } } finally { response.body().close(); // Make sure it's closed } }
@Override public Object call() throws Exception { try { DynamicHTMLExtractor wrapper = new DynamicHTMLExtractor(page, ChromeDriverPath); document = (Document) wrapper.browser_emulator.getHTMLDocument(); if (tableSelector != null && sfields != null && cfields != null) { return wrapper.extractTable(tableSelector, cfields, sfields); } else if (tableSelector != null && sfields != null) { return wrapper.extractTable(tableSelector, sfields); } else if (cfields != null && sfields != null) { return wrapper.extractFields(cfields, sfields); } else if (sfields != null) { return wrapper.extractFields(sfields); } } catch (HttpStatusException ex) { return null; } return null; }
@Override public Object call() throws URISyntaxException, IOException { try { StaticHTMLExtractor wrapper = new StaticHTMLExtractor(page); document = (Document) wrapper.fetcher.getHTMLDocument(); if (tableSelector != null && sfields != null && cfields != null) { return wrapper.extractTable(tableSelector, cfields, sfields); } else if (tableSelector != null && sfields != null) { return wrapper.extractTable(tableSelector, sfields); } else if (cfields != null && sfields != null) { return wrapper.extractFields(cfields, sfields); } else if (sfields != null) { return wrapper.extractFields(sfields); } } catch (HttpStatusException ex) { System.out.println(ex.fillInStackTrace()); return null; } return null; }
protected Document httpGet(Connection conn) { try { // TODO: execute network request in a separate thread pool return conn.get(); } catch (IOException e) { if(e instanceof HttpStatusException) { HttpStatusException statusException = (HttpStatusException) e; // sleep for backoffTime if we're asked to slow down if(statusException.getStatusCode() == 503) { try { Thread.sleep(backoffTime); } catch (InterruptedException e1) { throw new RuntimeException(e1); } } } throw new RuntimeException(e); } }
@SuppressLint("DefaultLocale") public static void sendException(Context c, Exception e, boolean fatal, String additionalData) { boolean send = true; if (e instanceof java.net.UnknownHostException || e instanceof NoRouteToHostException) { fatal = false; } else if (e instanceof HttpStatusException) { if (TextUtils.isEmpty(additionalData)) { additionalData = String.format("%d: %s", ((HttpStatusException) e).getStatusCode(), ((HttpStatusException) e).getUrl()); } send = (((HttpStatusException) e).getStatusCode() != 503); } else if (e instanceof SSLException) { if (!TextUtils.isEmpty(additionalData)) { additionalData = additionalData + ", "; } else { additionalData = ""; } additionalData = additionalData + getAnalytics().getPsStatus().toString(); } if (send) { getAnalytics().sendException(c, e, fatal, additionalData); } }
@Override public void exe(MessageEvent event, String cmdName, String[] args) throws Exception { String channel = event.getChannel().getName(); if(args.length == 1) { try { Jsoup.connect("http://www.twitch.tv/" + args[0]).get(); Utilities.sendMessage(channel, "http://www.twitch.tv/" + args[0]); } catch(HttpStatusException e) { if(e.getStatusCode() == 404) Utilities.sendMessage(channel, l10n.translate("error", channel)); } } else Utilities.sendHelp(module, event.getUser().getNick(), channel); }
@Override public void exe(MessageEvent event, String cmdName, String[] args) throws Exception { String channel = event.getChannel().getName(); if(args.length == 1) { try { Jsoup.connect("http://www.youtube.com/" + args[0]).get(); Utilities.sendMessage(channel, "http://www.youtube.com/" + args[0]); } catch(HttpStatusException e) { if(e.getStatusCode() == 404) Utilities.sendMessage(channel, l10n.translate("error", channel)); } } else Utilities.sendHelp(module, event.getUser().getNick(), channel); }
@Override public void exe(MessageEvent event, String cmdName, String[] args) throws Exception { String channel = event.getChannel().getName(); if(args.length == 1) { try { Jsoup.connect("http://www.twitter.com/" + args[0]).get(); Utilities.sendMessage(channel, "http://www.twitter.com/" + args[0]); } catch(HttpStatusException e) { if(e.getStatusCode() == 404) Utilities.sendMessage(channel, l10n.translate("error", channel)); } } else Utilities.sendHelp(module, event.getUser().getNick(), channel); }
/** * Loads a GET request from a specified path and query info * @param path the path (beginning with a forward slash). * @param params parameter information to pass as query. * @return the body string, or null if error occurs. * @throws IOException */ public String request(String path, Map<String, String> params) throws IOException { if (checkExpiration() != 1) return null; Connection conn = Jsoup.connect(Session.GRADEBOOK_ROOT + path) .timeout(60000).cookies(getCookies()); if (params != null) conn.data(params); try { Connection.Response resp = conn.execute(); return resp.body(); } catch (HttpStatusException e) { e.printStackTrace(); return null; } }
@Override public void onCommand(String command, User user, PircBotX network, String prefix, Channel channel, boolean isPrivate, int userPermLevel, String... args) throws Exception { if(command.equalsIgnoreCase("showerthought")){ args = new String[1]; args[0] = "showerthoughts"; } try { JsonArray results = GeneralUtils.getJsonObject("https://api.reddit.com/r/" + args[0] + "/?limit=25").get("data").getAsJsonObject().get("children").getAsJsonArray(); if (results.size() < 1) { IRCUtils.sendError(user, network, channel, "Search returned no results", prefix); } else { JsonObject result = results.get(new Random().nextInt(results.size() - 1)).getAsJsonObject().get("data").getAsJsonObject(); IRCUtils.sendMessage(user, network, channel, result.get("title").getAsString() + " - by " + IRCUtils.noPing(result.get("author").getAsString()) + " - " + GeneralUtils.shortenURL(result.get("url").getAsString()), prefix); } }catch(HttpStatusException e){ if(e.getStatusCode() == 403){ IRCUtils.sendError(user,network,channel, "Private Subreddit!", prefix); } } }
@Override public void onCommand(String command, User user, PircBotX network, String prefix, Channel channel, boolean isPrivate, int userPermLevel, String... args) throws Exception { if (!args[0].startsWith(".")) { args[0] = "." + args[0]; } try { Document doc = Jsoup.connect("http://www.iana.org/domains/root/db/" + args[0]).userAgent(Registry.USER_AGENT).get(); Elements titles = doc.select("#main_right").select("h2"); Elements names = doc.select("#main_right").select("b"); String[] organization = doc.select("#main_right").after("br").html().split("\n"); Set<String> results = new HashSet<>(); results.add("Sponsoring Organization: "+ names.get(0).text()); results.add("Administrative Contact: " + names.get(1).text() + ", " + GeneralUtils.stripHTML(organization[10])); results.add("Technical Contact: " + names.get(5).text() + ", " + GeneralUtils.stripHTML(organization[23])); IRCUtils.sendMessage(user, network, channel, StringUtils.join(results, " - "), prefix); }catch(HttpStatusException e){ if(e.getStatusCode() == 404){ IRCUtils.sendError(user, network, channel, "Invalid TLD", prefix); }else{ IRCUtils.sendError(user, network, channel, "Failed to execute command, please make sure you are using the correct syntax (" + getSyntax() + ")", prefix); } } }
public void parseHtml() throws NoAccessException, HttpStatusException, IOException { doc = Jsoup.connect(url).timeout(15000).get(); if (!doc.getElementsByClass("access-link").isEmpty()) { throw new NoAccessException(); } Integer totalPages; try { totalPages = Integer.parseInt(doc .getElementsByClass("number-of-pages").first().text()); } catch (NullPointerException e) { totalPages = 1; } getChapters(chapters); for (int i = 2; i <= totalPages; i++) { url = urlBase + i; System.out.println(url); doc = Jsoup.connect(url).timeout(15000).get(); chapters.putAll(getChapters(chapters)); } }
public static void handle(String action, Context context, Throwable err) { String msg; if(err instanceof HttpStatusException && ((HttpStatusException) err).getStatusCode() == 503) { msg = "Mmmh... This service is currently unavailable... This is not my fault, I promise ! Try again later."; } else if(err instanceof SocketTimeoutException) { msg = String.format("Mmmh... %s takes too long... This is not my fault, I promise ! Try again later.", StringUtils.capitalize(action)); } else { msg = String.format("Sorry, something went wrong while %s... My developer has been warned.", action); } LogUtils.errorf(context.getContent(), context.getChannel(), err, msg); }
@Override public void performAction() throws IOException { try { document = Jsoup.connect(uRLString).get(); } catch (HttpStatusException e) { e.printStackTrace(); if (e.getStatusCode() == 404) { document = null; } else { throw e; } } }
private static void runOnBackground(final HttpRequestCallbackListener listener, final BackgroundLogic backgroundLogic) { new Thread() { @Override public void run() { int count = 0, count2 = 0; while (true) { try { backgroundLogic.run(); break; } catch (Exception e) { e.printStackTrace(); if (e instanceof ConnectException) { count2++; if (count2 > 3) { listener.onFailure(e, REASON_INTERNET_NO_GOOD); break; } continue; } if (e instanceof SocketException || e instanceof UnknownHostException || e instanceof SocketTimeoutException) { count++; if (count > 3) { String reason = REASON_CONNECT_SERVER_FAILURE; if (e instanceof SocketTimeoutException) reason = REASON_INTERNET_NO_GOOD; listener.onFailure(e, reason); break; } continue; } listener.onFailure(e, e instanceof HttpStatusException ? REASON_SERVER_404 : REASON_CONNECT_SERVER_FAILURE); break; } } } }.start(); }
public static void manageIOException(Exception e, IMessage message, Command command, Language lg, DiscordException notFound){ // First we try parsing the exception message to see if it contains the response code Matcher exMsgStatusCodeMatcher = Pattern.compile("^Server returned HTTP response code: (\\d+)") .matcher(e.getMessage()); if(exMsgStatusCodeMatcher.find()) { int statusCode = Integer.parseInt(exMsgStatusCodeMatcher.group(1)); if (statusCode >= 500 && statusCode < 600) { LOG.warn("manageIOException", e); gameWebsite503.throwException(message, command, lg); } else { ClientConfig.setSentryContext(message.getGuild(), message.getAuthor(), message.getChannel(), message); LOG.error("manageIOException", e); unknown.throwException(message, command, lg); } } else if (e instanceof UnknownHostException || e instanceof SocketTimeoutException) { gameWebsite503.throwException(message, command, lg); } else if (e instanceof FileNotFoundException || e instanceof HttpStatusException || e instanceof NoRouteToHostException){ notFound.throwException(message, command, lg); } else { ClientConfig.setSentryContext(message.getGuild(), message.getAuthor(), message.getChannel(), message); LOG.error("manageIOException", e); unknown.throwException(message, command, lg); } }
public static String getContent(String url) { LOGGER.debug("url:" + url); Connection conn = Jsoup.connect(url) .header("Accept", ACCEPT) .header("Accept-Encoding", ENCODING) .header("Accept-Language", LANGUAGE) .header("Connection", CONNECTION) .header("Referer", REFERER) .header("Host", HOST) .header("User-Agent", USER_AGENTS.get(uac.incrementAndGet() % USER_AGENTS.size())) .header("X-Forwarded-For", getRandomIp()) .header("Proxy-Client-IP", getRandomIp()) .header("WL-Proxy-Client-IP", getRandomIp()) .ignoreContentType(true); String html = ""; try { html = conn.post().html(); }catch (Exception e){ if(e instanceof HttpStatusException) { HttpStatusException ex = (HttpStatusException) e; LOGGER.error("error code:"+ex.getStatusCode()); if(ex.getStatusCode()==404){ return "404"; } } LOGGER.error("获取URL:"+url+" 页面出错", e); } return html; }
@Test public void impossibleEcliTest() { try { parseXml(requestXmlForEcli("I-should-not-exist").body().byteStream()); assertEquals("apples", "pears"); } catch (HttpStatusException ignored) { } catch (IOException | JAXBException | XPathExpressionException e) { throw new Error(e); } }
public OutputType processResearcher() throws Exception { if (allowSkip() && workVerifiedDT != null && workVerifiedDT.getTimeInMillis() > new DateTime().minusDays(daysConsideredOld).getMillis()) { return OutputType.SKIPPED; } else { Researcher researcher = createResearcher(); researcher.setAffiliation(affiliation); try { reader.getPageItems(researcher); } catch (HttpStatusException e) { if (404 == e.getStatusCode()) { message = e.toString(); deleteResearcher(); return OutputType.DELETED; } else { throw e; } } List<String> preStatements = new ArrayList<String>(); preStatements.add(String.format(DELETE_RESEARCHER_THUMBNAIL, getResearcherURI())); if (generateThumbnail(researcher)) { // just drop the triple, not the image or thumbnail preStatements.addAll(Arrays.asList(String.format(REMOVE_EXISTING_IMAGES, getResearcherURI()), String.format(ADD_THUMBNAIL, getResearcherURI(), researcher.getThumbnailURL()))); } store.startTransaction(); store.execute(preStatements); store.execute(String.format(DELETE_PRIOR_PROCESS_LOG, getResearcherURI(), getCrawler().getURI())); store.update(researcher); store.endTransaction(); return OutputType.PROCESSED; } }
private void loadStudentPicture() throws IOException { try { byte[] data = Jsoup.connect("https://gradebook.pisd.edu/Pinnacle/Gradebook/common/picture.ashx") .timeout(60000).method(Connection.Method.GET).data("studentId", photoId) .cookies(session.getCookies()).ignoreContentType(true).execute().bodyAsBytes(); studentPictureBitmap = BitmapFactory.decodeByteArray(data, 0, data.length); } catch (HttpStatusException ex) { ex.printStackTrace(); } }
private static Document requestGooglePlayJSON(String url) { synchronized (cache) { if (cache.containsKey(url) && cache.get(url) != null) { return cache.get(url); } int tries = 0; Document tmp; while (tries < TRIES) { tries++; try { tmp = Jsoup.connect(url).get(); if (tmp != null) { cache.put(url, tmp); } return tmp; } catch (HttpStatusException hse) { System.out.println("Received incorrect HTTP status. Try: " + tries); hse.printStackTrace(); } catch (IOException ex) { System.out.println("Could not get Google Play information. Try: " + tries); ex.printStackTrace();; } } return null; } }
/** * Compute stats for given user (student). * * @param username Passport York ID * @param password Password for Passport York ID * @return Student with stats attributes set * @throws HttpStatusException HTTP related issues on login and scraping * @throws Exception HTML format related issues */ public Student compute(String username, String password) throws Exception, HttpStatusException { Student student = null; ArrayList<Document> docs = scraper.getDocs(username, password); student = parser.processStudentDocs(docs.get(0), docs.get(1)); student = calculator.statsGeneral(student); return student; }
private static void runOnBackground(final HttpRequestCallbackListener listener, final BackgroundLogic backgroundLogic) { mThreadPool.execute(new Thread() { @Override public void run() { int count = 0, count2 = 0; while (true) { try { backgroundLogic.run(); break; } catch (Exception e) { e.printStackTrace(); if (e instanceof ConnectException) { if (!isNetWorkAvailable()) { listener.onFailure(e, REASON_NO_INTERNET_CONNECT); break; } else { count2++; if (count2 > 3) { listener.onFailure(e, REASON_INTERNET_NO_GOOD); break; } continue; } } if (e instanceof SocketException || e instanceof UnknownHostException || e instanceof SocketTimeoutException) { count++; if (count > 3) { String reason = REASON_CONNECT_SERVER_FAILURE; if (e instanceof SocketTimeoutException) reason = REASON_INTERNET_NO_GOOD; if (!isNetWorkAvailable()) reason = REASON_NO_INTERNET_CONNECT; listener.onFailure(e, reason); break; } continue; } listener.onFailure(e, e instanceof HttpStatusException ? REASON_SERVER_404 : !isNetWorkAvailable() ? REASON_NO_INTERNET_CONNECT : REASON_CONNECT_SERVER_FAILURE); break; } } } }); }
@Override protected UpdaterStatus parseUrl( String url ) { try { Document doc = Jsoup.connect(url).get(); for (int i = 0; i < 20; i++) { // Get the url for the first app returned Elements elements = doc.getElementsByClass("cardlink_1_" + i); if (elements.size() < 1) { return UpdaterStatus.STATUS_UPDATE_NOT_FOUND; } String app_url = elements.get(0).attr("href"); // Get package name from app url Document doc2 = Jsoup.connect(app_url).get(); elements = doc2.getElementsByClass("packagename"); if (elements.size() < 1) { return UpdaterStatus.STATUS_UPDATE_NOT_FOUND; } String pname = elements.get(0).getElementsByClass("right").get(0).html(); // Check if it's the same app if (!pname.equals(mPname)) { continue; } // Get version String version = doc2.getElementsByAttributeValue("itemprop", "softwareVersion").get(0).html(); // Compare versions if (compareVersions(mCurrentVersion, version) == -1) { mResultUrl = app_url; mResultVersion = version; return UpdaterStatus.STATUS_UPDATE_FOUND; } } return UpdaterStatus.STATUS_UPDATE_NOT_FOUND; } catch (HttpStatusException status) { if (status.getStatusCode() == 404 || status.getStatusCode() == 403) { return UpdaterStatus.STATUS_UPDATE_NOT_FOUND; } else { mError = addCommonInfoToError(status); return UpdaterStatus.STATUS_ERROR; } } catch (Exception e) { mError = addCommonInfoToError(e); return UpdaterStatus.STATUS_ERROR; } }
public Response execute() throws IOException { Response response = getResponse(); if (response.code() != 200) throw new HttpStatusException("Did not get HTTP code 200", response.code(), request.url().toString()); return response; }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) throw new MalformedURLException("Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); Response res; try { conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER || status == HTTP_TEMP_REDIR) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString()); } res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method(Method.GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); String location = res.header("Location"); if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php location = location.substring(6); req.url(new URL(req.url(), encodeUrl(location))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; // check that we can handle the returned content type; if not, abort before fetching it String contentType = res.contentType(); if (contentType != null && !req.ignoreContentType() && !contentType.startsWith("text/") && !contentType.startsWith("application/xml") && !xmlContentTypeRxp.matcher(contentType).matches() ) throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml", contentType, req.url().toString()); InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize()); res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } } finally { // per Java's documentation, this is not necessary, and precludes keepalives. However in practise, // connection errors will not be released quickly enough and can cause a too many open files error. conn.disconnect(); } res.executed = true; return res; }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) throw new MalformedURLException("Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); Response res; try { conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString()); } res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method(Method.GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); String location = res.header("Location"); if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php location = location.substring(6); req.url(new URL(req.url(), encodeUrl(location))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; // check that we can handle the returned content type; if not, abort before fetching it String contentType = res.contentType(); if (contentType != null && !req.ignoreContentType() && (!(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml")))) throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml", contentType, req.url().toString()); InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize()); res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } } finally { // per Java's documentation, this is not necessary, and precludes keepalives. However in practise, // connection errors will not be released quickly enough and can cause a too many open files error. conn.disconnect(); } res.executed = true; return res; }
@ResponseStatus(value = HttpStatus.NOT_FOUND, reason = "Error fetching snapshot") @ExceptionHandler(HttpStatusException.class) public void httpStatusExceptionHandler() { }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) throw new MalformedURLException("Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); Response res; try { conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString()); } res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method(Method.GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); req.url(new URL(req.url(), res.header("Location"))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; // check that we can handle the returned content type; if not, abort before fetching it String contentType = res.contentType(); if (contentType != null && !req.ignoreContentType() && (!(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml")))) throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml", contentType, req.url().toString()); InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize()); res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } } finally { // per Java's documentation, this is not necessary, and precludes keepalives. However in practise, // connection errors will not be released quickly enough and can cause a too many open files error. conn.disconnect(); } res.executed = true; return res; }