protected Document doInBackground(String... urls) { Log.d(LOG_TAG, "doInBackground, url=" + urls[0]); try { Response resp = Jsoup.connect(urls[0]) .header("Authorization", "Basic " + base64login).timeout(30 * 1000) .method(Method.GET).execute(); if (resp.contentType().contains("text/html")) { Log.d(LOG_TAG, "New directory"); currentUrl = urls[0]; return resp.parse(); } else { Log.d(LOG_TAG, "UnsupportedContentType"); return null; } } catch (UnsupportedMimeTypeException me) { Log.d(LOG_TAG, "UnsupportedMimeTypeException"); return null; } catch (Exception e) { Log.d(LOG_TAG, "Other Exception"); this.exception = e; return null; } }
protected Document doInBackground(String... urls) { try { Response resp = Jsoup .connect(urls[0]) .timeout(10 * 1000) .cookie("pianhao", "%7B%22qing%22%3A%22super%22%2C%22qtudou%22%3A%22null%22%2C%22qyouku%22%3A%22null%22%2C%22q56%22%3A%22null%22%2C%22qcntv%22%3A%22null%22%2C%22qletv%22%3A%22null%22%2C%22qqiyi%22%3A%22null%22%2C%22qsohu%22%3A%22null%22%2C%22qqq%22%3A%22null%22%2C%22qku6%22%3A%22null%22%2C%22qyinyuetai%22%3A%22null%22%2C%22qtangdou%22%3A%22null%22%2C%22qxunlei%22%3A%22null%22%2C%22qfunshion%22%3A%22null%22%2C%22qsina%22%3A%22null%22%2C%22qpptv%22%3A%22null%22%2C%22xia%22%3A%22ask%22%2C%22pop%22%3A%22no%22%2C%22open%22%3A%22no%22%7D") .method(Method.GET).execute(); return resp.parse(); } catch (UnsupportedMimeTypeException me) { return null; } catch (Exception e) { this.exception = e; return null; } }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) throw new MalformedURLException("Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); Response res; try { conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER || status == HTTP_TEMP_REDIR) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString()); } res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method(Method.GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); String location = res.header("Location"); if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php location = location.substring(6); req.url(new URL(req.url(), encodeUrl(location))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; // check that we can handle the returned content type; if not, abort before fetching it String contentType = res.contentType(); if (contentType != null && !req.ignoreContentType() && !contentType.startsWith("text/") && !contentType.startsWith("application/xml") && !xmlContentTypeRxp.matcher(contentType).matches() ) throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml", contentType, req.url().toString()); InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize()); res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } } finally { // per Java's documentation, this is not necessary, and precludes keepalives. However in practise, // connection errors will not be released quickly enough and can cause a too many open files error. conn.disconnect(); } res.executed = true; return res; }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) throw new MalformedURLException("Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); Response res; try { conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString()); } res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method(Method.GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); String location = res.header("Location"); if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php location = location.substring(6); req.url(new URL(req.url(), encodeUrl(location))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; // check that we can handle the returned content type; if not, abort before fetching it String contentType = res.contentType(); if (contentType != null && !req.ignoreContentType() && (!(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml")))) throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml", contentType, req.url().toString()); InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize()); res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } } finally { // per Java's documentation, this is not necessary, and precludes keepalives. However in practise, // connection errors will not be released quickly enough and can cause a too many open files error. conn.disconnect(); } res.executed = true; return res; }
static Response execute(Connection.Request req, Response previousResponse) throws IOException { Validate.notNull(req, "Request must not be null"); String protocol = req.url().getProtocol(); if (!protocol.equals("http") && !protocol.equals("https")) throw new MalformedURLException("Only http & https protocols supported"); // set up the request for execution if (req.method() == Connection.Method.GET && req.data().size() > 0) serialiseRequestUrl(req); // appends query string HttpURLConnection conn = createConnection(req); Response res; try { conn.connect(); if (req.method() == Connection.Method.POST) writePost(req.data(), conn.getOutputStream()); int status = conn.getResponseCode(); boolean needsRedirect = false; if (status != HttpURLConnection.HTTP_OK) { if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) needsRedirect = true; else if (!req.ignoreHttpErrors()) throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString()); } res = new Response(previousResponse); res.setupFromConnection(conn, previousResponse); if (needsRedirect && req.followRedirects()) { req.method(Method.GET); // always redirect with a get. any data param from original req are dropped. req.data().clear(); req.url(new URL(req.url(), res.header("Location"))); for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts) req.cookie(cookie.getKey(), cookie.getValue()); } return execute(req, res); } res.req = req; // check that we can handle the returned content type; if not, abort before fetching it String contentType = res.contentType(); if (contentType != null && !req.ignoreContentType() && (!(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml")))) throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml", contentType, req.url().toString()); InputStream bodyStream = null; InputStream dataStream = null; try { dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream(); bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ? new BufferedInputStream(new GZIPInputStream(dataStream)) : new BufferedInputStream(dataStream); res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize()); res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it } finally { if (bodyStream != null) bodyStream.close(); if (dataStream != null) dataStream.close(); } } finally { // per Java's documentation, this is not necessary, and precludes keepalives. However in practise, // connection errors will not be released quickly enough and can cause a too many open files error. conn.disconnect(); } res.executed = true; return res; }