Java Code Examples for org.jsoup.HttpStatusException

The following examples show how to use org.jsoup.HttpStatusException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: KaellyBot   Source File: ExceptionManager.java    License: GNU General Public License v3.0 6 votes vote down vote up
public static void manageIOException(Exception e, Message message, Command command, Language lg, DiscordException notFound){
    // First we try parsing the exception message to see if it contains the response code
    Matcher exMsgStatusCodeMatcher = Pattern.compile("^Server returned HTTP response code: (\\d+)")
            .matcher(e.getMessage());
    if(exMsgStatusCodeMatcher.find()) {
        int statusCode = Integer.parseInt(exMsgStatusCodeMatcher.group(1));
        if (statusCode >= 500 && statusCode < 600) {
            LOG.warn("manageIOException", e);
            gameWebsite503.throwException(message, command, lg);
        }
        else {
            LOG.error("manageIOException", e);
            BasicDiscordException.UNKNOWN_ERROR.throwException(message, command, lg);
        }
    } else if (e instanceof UnknownHostException || e instanceof SocketTimeoutException) {
        gameWebsite503.throwException(message, command, lg);
    } else if (e instanceof FileNotFoundException
            || e instanceof HttpStatusException
            || e instanceof NoRouteToHostException){
        notFound.throwException(message, command, lg);
    }
    else {
        LOG.error("manageIOException", e);
        BasicDiscordException.UNKNOWN_ERROR.throwException(message, command, lg);
    }
}
 
Example 2
Source Project: KaellyBot   Source File: ExceptionManager.java    License: GNU General Public License v3.0 6 votes vote down vote up
public static void manageSilentlyIOException(Exception e){
    Reporter.report(e);
    // First we try parsing the exception message to see if it contains the response code
    Matcher exMsgStatusCodeMatcher = Pattern.compile("^Server returned HTTP response code: (\\d+)")
            .matcher(e.getMessage());
    if(exMsgStatusCodeMatcher.find()) {
        int statusCode = Integer.parseInt(exMsgStatusCodeMatcher.group(1));
        if (statusCode >= 500 && statusCode < 600)
            LOG.warn("manageSilentlyIOException", e);
        else
            LOG.error("manageSilentlyIOException", e);
    } else if (e instanceof UnknownHostException
            || e instanceof SocketTimeoutException
            || e instanceof FileNotFoundException
            || e instanceof HttpStatusException
            || e instanceof NoRouteToHostException)
        LOG.warn("manageSilentlyIOException", e);
    else
        LOG.error("manageSilentlyIOException", e);

}
 
Example 3
@Override
public boolean login() throws IOException {
    // TODO: we could set some account parameters from here like default_private_links
    String url = new URL(this.mAccount.getUrlShaarli() + INFO_URL).toExternalForm();
    try {
        Log.d("Login", this.mAccount.getRestAPIKey());
        String body = this.newConnection(url, Connection.Method.GET)
                .execute()
                .body();
        Log.i("Login", body);
    } catch (HttpStatusException e) {
        Log.w("Login", e);
        Log.w("Login", e.getMessage());
        return false;
    }
    return true;
}
 
Example 4
Source Project: petscii-bbs   Source File: InternetBrowser.java    License: Mozilla Public License 2.0 5 votes vote down vote up
void loadWebPage(String url) throws Exception{
    loading();
    clearBrowserWindow();
    Document webpage;
    try {
        webpage = getWebpage(url);
    } catch (HttpStatusException | UnknownHostException ex) {
        webpage = Jsoup.parseBodyFragment("HTTP connection error");
    }
    displayPage(webpage, url);
}
 
Example 5
Source Project: superword   Source File: ChineseSynonymAntonymExtractor.java    License: Apache License 2.0 5 votes vote down vote up
public static String getContent(String url) {
    LOGGER.debug("url:" + url);
    Connection conn = Jsoup.connect(url)
            .header("Accept", ACCEPT)
            .header("Accept-Encoding", ENCODING)
            .header("Accept-Language", LANGUAGE)
            .header("Connection", CONNECTION)
            .header("Referer", REFERER)
            .header("Host", HOST)
            .header("User-Agent", USER_AGENTS.get(uac.incrementAndGet() % USER_AGENTS.size()))
            .header("X-Forwarded-For", getRandomIp())
            .header("Proxy-Client-IP", getRandomIp())
            .header("WL-Proxy-Client-IP", getRandomIp())
            .ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
    }catch (Exception e){
        if(e instanceof  HttpStatusException) {
            HttpStatusException ex = (HttpStatusException) e;
            LOGGER.error("error code:"+ex.getStatusCode());
            if(ex.getStatusCode()==404){
                return "404";
            }
        }
        LOGGER.error("获取URL:"+url+" 页面出错", e);
    }
    return html;
}
 
Example 6
Source Project: tutorials   Source File: JsoupParserIntegrationTest.java    License: MIT License 5 votes vote down vote up
@Test
public void loadDocument404() throws IOException {
    try {
        doc = Jsoup.connect("https://spring.io/will-not-be-found")
            .get();
    } catch (HttpStatusException ex) {
        assertEquals(404, ex.getStatusCode());
    }
}
 
Example 7
Source Project: J-Kinopoisk2IMDB   Source File: Exchangeable.java    License: Apache License 2.0 4 votes vote down vote up
public OUT getProcessedResponse() throws HttpStatusException {
    return responseProcessor.process(getResponse());
}
 
Example 8
Source Project: astor   Source File: HttpConnection.java    License: GNU General Public License v2.0 4 votes vote down vote up
static Response execute(Connection.Request req, Response previousResponse) throws IOException {
    Validate.notNull(req, "Request must not be null");
    String protocol = req.url().getProtocol();
    if (!protocol.equals("http") && !protocol.equals("https"))
        throw new MalformedURLException("Only http & https protocols supported");
    final boolean methodHasBody = req.method().hasBody();
    final boolean hasRequestBody = req.requestBody() != null;
    if (!methodHasBody)
        Validate.isFalse(hasRequestBody, "Cannot set a request body for HTTP method " + req.method());

    // set up the request for execution
    String mimeBoundary = null;
    if (req.data().size() > 0 && (!methodHasBody || hasRequestBody))
        serialiseRequestUrl(req);
    else if (methodHasBody)
        mimeBoundary = setOutputContentType(req);

    HttpURLConnection conn = createConnection(req);
    Response res;
    try {
        conn.connect();
        if (conn.getDoOutput())
            writePost(req, conn.getOutputStream(), mimeBoundary);

        int status = conn.getResponseCode();
        res = new Response(previousResponse);
        res.setupFromConnection(conn, previousResponse);
        res.req = req;

        // redirect if there's a location header (from 3xx, or 201 etc)
        if (res.hasHeader(LOCATION) && req.followRedirects()) {
            if (status != HTTP_TEMP_REDIR) {
                req.method(Method.GET); // always redirect with a get. any data param from original req are dropped.
                req.data().clear();
                req.requestBody(null);
                req.removeHeader(CONTENT_TYPE);
            }

            String location = res.header(LOCATION);
            if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php
                location = location.substring(6);
            URL redir = StringUtil.resolve(req.url(), location);
            req.url(encodeUrl(redir));

            for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
                req.cookie(cookie.getKey(), cookie.getValue());
            }
            return execute(req, res);
        }
        if ((status < 200 || status >= 400) && !req.ignoreHttpErrors())
                throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString());

        // check that we can handle the returned content type; if not, abort before fetching it
        String contentType = res.contentType();
        if (contentType != null
                && !req.ignoreContentType()
                && !contentType.startsWith("text/")
                && !xmlContentTypeRxp.matcher(contentType).matches()
                )
            throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml",
                    contentType, req.url().toString());

        // switch to the XML parser if content type is xml and not parser not explicitly set
        if (contentType != null && xmlContentTypeRxp.matcher(contentType).matches()) {
            // only flip it if a HttpConnection.Request (i.e. don't presume other impls want it):
            if (req instanceof HttpConnection.Request && !((Request) req).parserDefined) {
                req.parser(Parser.xmlParser());
            }
        }

        res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
        if (conn.getContentLength() != 0 && req.method() != HEAD) { // -1 means unknown, chunked. sun throws an IO exception on 500 response with no content when trying to read body
            res.bodyStream = null;
            res.bodyStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
            if (res.hasHeaderWithValue(CONTENT_ENCODING, "gzip"))
                res.bodyStream = new GZIPInputStream(res.bodyStream);
            res.bodyStream = new ConstrainableInputStream(res.bodyStream, DataUtil.bufferSize, req.maxBodySize());
        } else {
            res.byteData = DataUtil.emptyByteBuffer();
        }
    } catch (IOException e){
        // per Java's documentation, this is not necessary, and precludes keepalives. However in practise,
        // connection errors will not be released quickly enough and can cause a too many open files error.
        conn.disconnect();
        throw e;
    }

    res.executed = true;
    return res;
}
 
Example 9
Source Project: astor   Source File: HttpConnection.java    License: GNU General Public License v2.0 4 votes vote down vote up
static Response execute(Connection.Request req, Response previousResponse) throws IOException {
    Validate.notNull(req, "Request must not be null");
    String protocol = req.url().getProtocol();
    if (!protocol.equals("http") && !protocol.equals("https"))
        throw new MalformedURLException("Only http & https protocols supported");
    final boolean methodHasBody = req.method().hasBody();
    final boolean hasRequestBody = req.requestBody() != null;
    if (!methodHasBody)
        Validate.isFalse(hasRequestBody, "Cannot set a request body for HTTP method " + req.method());

    // set up the request for execution
    String mimeBoundary = null;
    if (req.data().size() > 0 && (!methodHasBody || hasRequestBody))
        serialiseRequestUrl(req);
    else if (methodHasBody)
        mimeBoundary = setOutputContentType(req);

    HttpURLConnection conn = createConnection(req);
    Response res;
    try {
        conn.connect();
        if (conn.getDoOutput())
            writePost(req, conn.getOutputStream(), mimeBoundary);

        int status = conn.getResponseCode();
        res = new Response(previousResponse);
        res.setupFromConnection(conn, previousResponse);
        res.req = req;

        // redirect if there's a location header (from 3xx, or 201 etc)
        if (res.hasHeader(LOCATION) && req.followRedirects()) {
            if (status != HTTP_TEMP_REDIR) {
                req.method(Method.GET); // always redirect with a get. any data param from original req are dropped.
                req.data().clear();
                req.requestBody(null);
                req.removeHeader(CONTENT_TYPE);
            }

            String location = res.header(LOCATION);
            if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php
                location = location.substring(6);
            URL redir = StringUtil.resolve(req.url(), location);
            req.url(encodeUrl(redir));

            for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
                req.cookie(cookie.getKey(), cookie.getValue());
            }
            return execute(req, res);
        }
        if ((status < 200 || status >= 400) && !req.ignoreHttpErrors())
                throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString());

        // check that we can handle the returned content type; if not, abort before fetching it
        String contentType = res.contentType();
        if (contentType != null
                && !req.ignoreContentType()
                && !contentType.startsWith("text/")
                && !xmlContentTypeRxp.matcher(contentType).matches()
                )
            throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml",
                    contentType, req.url().toString());

        // switch to the XML parser if content type is xml and not parser not explicitly set
        if (contentType != null && xmlContentTypeRxp.matcher(contentType).matches()) {
            // only flip it if a HttpConnection.Request (i.e. don't presume other impls want it):
            if (req instanceof HttpConnection.Request && !((Request) req).parserDefined) {
                req.parser(Parser.xmlParser());
            }
        }

        res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
        if (conn.getContentLength() != 0 && req.method() != HEAD) { // -1 means unknown, chunked. sun throws an IO exception on 500 response with no content when trying to read body
            res.bodyStream = null;
            res.bodyStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
            if (res.hasHeaderWithValue(CONTENT_ENCODING, "gzip"))
                res.bodyStream = new GZIPInputStream(res.bodyStream);
            res.bodyStream = new ConstrainableInputStream(res.bodyStream, DataUtil.bufferSize, req.maxBodySize());
        } else {
            res.byteData = DataUtil.emptyByteBuffer();
        }
    } catch (IOException e){
        // per Java's documentation, this is not necessary, and precludes keepalives. However in practise,
        // connection errors will not be released quickly enough and can cause a too many open files error.
        conn.disconnect();
        throw e;
    }

    res.executed = true;
    return res;
}
 
Example 10
Source Project: jsoup-learning   Source File: HttpConnection.java    License: MIT License 4 votes vote down vote up
static Response execute(Connection.Request req, Response previousResponse) throws IOException {
    Validate.notNull(req, "Request must not be null");
    String protocol = req.url().getProtocol();
    if (!protocol.equals("http") && !protocol.equals("https"))
        throw new MalformedURLException("Only http & https protocols supported");

    // set up the request for execution
    if (req.method() == Connection.Method.GET && req.data().size() > 0)
        serialiseRequestUrl(req); // appends query string
    HttpURLConnection conn = createConnection(req);
    Response res;
    try {
        conn.connect();
        if (req.method() == Connection.Method.POST)
            writePost(req.data(), conn.getOutputStream());

        int status = conn.getResponseCode();
        boolean needsRedirect = false;
        if (status != HttpURLConnection.HTTP_OK) {
            if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER)
                needsRedirect = true;
            else if (!req.ignoreHttpErrors())
                throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString());
        }
        res = new Response(previousResponse);
        res.setupFromConnection(conn, previousResponse);
        if (needsRedirect && req.followRedirects()) {
            req.method(Method.GET); // always redirect with a get. any data param from original req are dropped.
            req.data().clear();
            req.url(new URL(req.url(), res.header("Location")));
            for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
                req.cookie(cookie.getKey(), cookie.getValue());
            }
            return execute(req, res);
        }
        res.req = req;

        // check that we can handle the returned content type; if not, abort before fetching it
        String contentType = res.contentType();
        if (contentType != null && !req.ignoreContentType() && (!(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml"))))
            throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml",
                    contentType, req.url().toString());

        InputStream bodyStream = null;
        InputStream dataStream = null;
        try {
            dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
            bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ?
                    new BufferedInputStream(new GZIPInputStream(dataStream)) :
                    new BufferedInputStream(dataStream);

            res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize());
            res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
        } finally {
            if (bodyStream != null) bodyStream.close();
            if (dataStream != null) dataStream.close();
        }
    } finally {
        // per Java's documentation, this is not necessary, and precludes keepalives. However in practise,
        // connection errors will not be released quickly enough and can cause a too many open files error.
        conn.disconnect();
    }

    res.executed = true;
    return res;
}