Java Code Examples for org.jsoup.HttpStatusException

public static void manageIOException(Exception e, Message message, Command command, Language lg, DiscordException notFound){
    // First we try parsing the exception message to see if it contains the response code
    Matcher exMsgStatusCodeMatcher = Pattern.compile("^Server returned HTTP response code: (\\d+)")
            .matcher(e.getMessage());
    if(exMsgStatusCodeMatcher.find()) {
        int statusCode = Integer.parseInt(exMsgStatusCodeMatcher.group(1));
        if (statusCode >= 500 && statusCode < 600) {
            LOG.warn("manageIOException", e);
            gameWebsite503.throwException(message, command, lg);
        }
        else {
            LOG.error("manageIOException", e);
            BasicDiscordException.UNKNOWN_ERROR.throwException(message, command, lg);
        }
    } else if (e instanceof UnknownHostException || e instanceof SocketTimeoutException) {
        gameWebsite503.throwException(message, command, lg);
    } else if (e instanceof FileNotFoundException
            || e instanceof HttpStatusException
            || e instanceof NoRouteToHostException){
        notFound.throwException(message, command, lg);
    }
    else {
        LOG.error("manageIOException", e);
        BasicDiscordException.UNKNOWN_ERROR.throwException(message, command, lg);
    }
}
 

public static void manageSilentlyIOException(Exception e){
    Reporter.report(e);
    // First we try parsing the exception message to see if it contains the response code
    Matcher exMsgStatusCodeMatcher = Pattern.compile("^Server returned HTTP response code: (\\d+)")
            .matcher(e.getMessage());
    if(exMsgStatusCodeMatcher.find()) {
        int statusCode = Integer.parseInt(exMsgStatusCodeMatcher.group(1));
        if (statusCode >= 500 && statusCode < 600)
            LOG.warn("manageSilentlyIOException", e);
        else
            LOG.error("manageSilentlyIOException", e);
    } else if (e instanceof UnknownHostException
            || e instanceof SocketTimeoutException
            || e instanceof FileNotFoundException
            || e instanceof HttpStatusException
            || e instanceof NoRouteToHostException)
        LOG.warn("manageSilentlyIOException", e);
    else
        LOG.error("manageSilentlyIOException", e);

}
 

@Override
public boolean login() throws IOException {
    // TODO: we could set some account parameters from here like default_private_links
    String url = new URL(this.mAccount.getUrlShaarli() + INFO_URL).toExternalForm();
    try {
        Log.d("Login", this.mAccount.getRestAPIKey());
        String body = this.newConnection(url, Connection.Method.GET)
                .execute()
                .body();
        Log.i("Login", body);
    } catch (HttpStatusException e) {
        Log.w("Login", e);
        Log.w("Login", e.getMessage());
        return false;
    }
    return true;
}
 

void loadWebPage(String url) throws Exception{
    loading();
    clearBrowserWindow();
    Document webpage;
    try {
        webpage = getWebpage(url);
    } catch (HttpStatusException | UnknownHostException ex) {
        webpage = Jsoup.parseBodyFragment("HTTP connection error");
    }
    displayPage(webpage, url);
}
 

public static String getContent(String url) {
    LOGGER.debug("url:" + url);
    Connection conn = Jsoup.connect(url)
            .header("Accept", ACCEPT)
            .header("Accept-Encoding", ENCODING)
            .header("Accept-Language", LANGUAGE)
            .header("Connection", CONNECTION)
            .header("Referer", REFERER)
            .header("Host", HOST)
            .header("User-Agent", USER_AGENTS.get(uac.incrementAndGet() % USER_AGENTS.size()))
            .header("X-Forwarded-For", getRandomIp())
            .header("Proxy-Client-IP", getRandomIp())
            .header("WL-Proxy-Client-IP", getRandomIp())
            .ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
    }catch (Exception e){
        if(e instanceof  HttpStatusException) {
            HttpStatusException ex = (HttpStatusException) e;
            LOGGER.error("error code:"+ex.getStatusCode());
            if(ex.getStatusCode()==404){
                return "404";
            }
        }
        LOGGER.error("获取URL："+url+" 页面出错", e);
    }
    return html;
}
 

@Test
public void loadDocument404() throws IOException {
    try {
        doc = Jsoup.connect("https://spring.io/will-not-be-found")
            .get();
    } catch (HttpStatusException ex) {
        assertEquals(404, ex.getStatusCode());
    }
}
 

public OUT getProcessedResponse() throws HttpStatusException {
    return responseProcessor.process(getResponse());
}
 

static Response execute(Connection.Request req, Response previousResponse) throws IOException {
    Validate.notNull(req, "Request must not be null");
    String protocol = req.url().getProtocol();
    if (!protocol.equals("http") && !protocol.equals("https"))
        throw new MalformedURLException("Only http & https protocols supported");
    final boolean methodHasBody = req.method().hasBody();
    final boolean hasRequestBody = req.requestBody() != null;
    if (!methodHasBody)
        Validate.isFalse(hasRequestBody, "Cannot set a request body for HTTP method " + req.method());

    // set up the request for execution
    String mimeBoundary = null;
    if (req.data().size() > 0 && (!methodHasBody || hasRequestBody))
        serialiseRequestUrl(req);
    else if (methodHasBody)
        mimeBoundary = setOutputContentType(req);

    HttpURLConnection conn = createConnection(req);
    Response res;
    try {
        conn.connect();
        if (conn.getDoOutput())
            writePost(req, conn.getOutputStream(), mimeBoundary);

        int status = conn.getResponseCode();
        res = new Response(previousResponse);
        res.setupFromConnection(conn, previousResponse);
        res.req = req;

        // redirect if there's a location header (from 3xx, or 201 etc)
        if (res.hasHeader(LOCATION) && req.followRedirects()) {
            if (status != HTTP_TEMP_REDIR) {
                req.method(Method.GET); // always redirect with a get. any data param from original req are dropped.
                req.data().clear();
                req.requestBody(null);
                req.removeHeader(CONTENT_TYPE);
            }

            String location = res.header(LOCATION);
            if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php
                location = location.substring(6);
            URL redir = StringUtil.resolve(req.url(), location);
            req.url(encodeUrl(redir));

            for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
                req.cookie(cookie.getKey(), cookie.getValue());
            }
            return execute(req, res);
        }
        if ((status < 200 || status >= 400) && !req.ignoreHttpErrors())
                throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString());

        // check that we can handle the returned content type; if not, abort before fetching it
        String contentType = res.contentType();
        if (contentType != null
                && !req.ignoreContentType()
                && !contentType.startsWith("text/")
                && !xmlContentTypeRxp.matcher(contentType).matches()
                )
            throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml",
                    contentType, req.url().toString());

        // switch to the XML parser if content type is xml and not parser not explicitly set
        if (contentType != null && xmlContentTypeRxp.matcher(contentType).matches()) {
            // only flip it if a HttpConnection.Request (i.e. don't presume other impls want it):
            if (req instanceof HttpConnection.Request && !((Request) req).parserDefined) {
                req.parser(Parser.xmlParser());
            }
        }

        res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
        if (conn.getContentLength() != 0 && req.method() != HEAD) { // -1 means unknown, chunked. sun throws an IO exception on 500 response with no content when trying to read body
            res.bodyStream = null;
            res.bodyStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
            if (res.hasHeaderWithValue(CONTENT_ENCODING, "gzip"))
                res.bodyStream = new GZIPInputStream(res.bodyStream);
            res.bodyStream = new ConstrainableInputStream(res.bodyStream, DataUtil.bufferSize, req.maxBodySize());
        } else {
            res.byteData = DataUtil.emptyByteBuffer();
        }
    } catch (IOException e){
        // per Java's documentation, this is not necessary, and precludes keepalives. However in practise,
        // connection errors will not be released quickly enough and can cause a too many open files error.
        conn.disconnect();
        throw e;
    }

    res.executed = true;
    return res;
}
 

static Response execute(Connection.Request req, Response previousResponse) throws IOException {
    Validate.notNull(req, "Request must not be null");
    String protocol = req.url().getProtocol();
    if (!protocol.equals("http") && !protocol.equals("https"))
        throw new MalformedURLException("Only http & https protocols supported");
    final boolean methodHasBody = req.method().hasBody();
    final boolean hasRequestBody = req.requestBody() != null;
    if (!methodHasBody)
        Validate.isFalse(hasRequestBody, "Cannot set a request body for HTTP method " + req.method());

    // set up the request for execution
    String mimeBoundary = null;
    if (req.data().size() > 0 && (!methodHasBody || hasRequestBody))
        serialiseRequestUrl(req);
    else if (methodHasBody)
        mimeBoundary = setOutputContentType(req);

    HttpURLConnection conn = createConnection(req);
    Response res;
    try {
        conn.connect();
        if (conn.getDoOutput())
            writePost(req, conn.getOutputStream(), mimeBoundary);

        int status = conn.getResponseCode();
        res = new Response(previousResponse);
        res.setupFromConnection(conn, previousResponse);
        res.req = req;

        // redirect if there's a location header (from 3xx, or 201 etc)
        if (res.hasHeader(LOCATION) && req.followRedirects()) {
            if (status != HTTP_TEMP_REDIR) {
                req.method(Method.GET); // always redirect with a get. any data param from original req are dropped.
                req.data().clear();
                req.requestBody(null);
                req.removeHeader(CONTENT_TYPE);
            }

            String location = res.header(LOCATION);
            if (location != null && location.startsWith("http:/") && location.charAt(6) != '/') // fix broken Location: http:/temp/AAG_New/en/index.php
                location = location.substring(6);
            URL redir = StringUtil.resolve(req.url(), location);
            req.url(encodeUrl(redir));

            for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
                req.cookie(cookie.getKey(), cookie.getValue());
            }
            return execute(req, res);
        }
        if ((status < 200 || status >= 400) && !req.ignoreHttpErrors())
                throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString());

        // check that we can handle the returned content type; if not, abort before fetching it
        String contentType = res.contentType();
        if (contentType != null
                && !req.ignoreContentType()
                && !contentType.startsWith("text/")
                && !xmlContentTypeRxp.matcher(contentType).matches()
                )
            throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml",
                    contentType, req.url().toString());

        // switch to the XML parser if content type is xml and not parser not explicitly set
        if (contentType != null && xmlContentTypeRxp.matcher(contentType).matches()) {
            // only flip it if a HttpConnection.Request (i.e. don't presume other impls want it):
            if (req instanceof HttpConnection.Request && !((Request) req).parserDefined) {
                req.parser(Parser.xmlParser());
            }
        }

        res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
        if (conn.getContentLength() != 0 && req.method() != HEAD) { // -1 means unknown, chunked. sun throws an IO exception on 500 response with no content when trying to read body
            res.bodyStream = null;
            res.bodyStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
            if (res.hasHeaderWithValue(CONTENT_ENCODING, "gzip"))
                res.bodyStream = new GZIPInputStream(res.bodyStream);
            res.bodyStream = new ConstrainableInputStream(res.bodyStream, DataUtil.bufferSize, req.maxBodySize());
        } else {
            res.byteData = DataUtil.emptyByteBuffer();
        }
    } catch (IOException e){
        // per Java's documentation, this is not necessary, and precludes keepalives. However in practise,
        // connection errors will not be released quickly enough and can cause a too many open files error.
        conn.disconnect();
        throw e;
    }

    res.executed = true;
    return res;
}
 

static Response execute(Connection.Request req, Response previousResponse) throws IOException {
    Validate.notNull(req, "Request must not be null");
    String protocol = req.url().getProtocol();
    if (!protocol.equals("http") && !protocol.equals("https"))
        throw new MalformedURLException("Only http & https protocols supported");

    // set up the request for execution
    if (req.method() == Connection.Method.GET && req.data().size() > 0)
        serialiseRequestUrl(req); // appends query string
    HttpURLConnection conn = createConnection(req);
    Response res;
    try {
        conn.connect();
        if (req.method() == Connection.Method.POST)
            writePost(req.data(), conn.getOutputStream());

        int status = conn.getResponseCode();
        boolean needsRedirect = false;
        if (status != HttpURLConnection.HTTP_OK) {
            if (status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER)
                needsRedirect = true;
            else if (!req.ignoreHttpErrors())
                throw new HttpStatusException("HTTP error fetching URL", status, req.url().toString());
        }
        res = new Response(previousResponse);
        res.setupFromConnection(conn, previousResponse);
        if (needsRedirect && req.followRedirects()) {
            req.method(Method.GET); // always redirect with a get. any data param from original req are dropped.
            req.data().clear();
            req.url(new URL(req.url(), res.header("Location")));
            for (Map.Entry<String, String> cookie : res.cookies.entrySet()) { // add response cookies to request (for e.g. login posts)
                req.cookie(cookie.getKey(), cookie.getValue());
            }
            return execute(req, res);
        }
        res.req = req;

        // check that we can handle the returned content type; if not, abort before fetching it
        String contentType = res.contentType();
        if (contentType != null && !req.ignoreContentType() && (!(contentType.startsWith("text/") || contentType.startsWith("application/xml") || contentType.startsWith("application/xhtml+xml"))))
            throw new UnsupportedMimeTypeException("Unhandled content type. Must be text/*, application/xml, or application/xhtml+xml",
                    contentType, req.url().toString());

        InputStream bodyStream = null;
        InputStream dataStream = null;
        try {
            dataStream = conn.getErrorStream() != null ? conn.getErrorStream() : conn.getInputStream();
            bodyStream = res.hasHeader("Content-Encoding") && res.header("Content-Encoding").equalsIgnoreCase("gzip") ?
                    new BufferedInputStream(new GZIPInputStream(dataStream)) :
                    new BufferedInputStream(dataStream);

            res.byteData = DataUtil.readToByteBuffer(bodyStream, req.maxBodySize());
            res.charset = DataUtil.getCharsetFromContentType(res.contentType); // may be null, readInputStream deals with it
        } finally {
            if (bodyStream != null) bodyStream.close();
            if (dataStream != null) dataStream.close();
        }
    } finally {
        // per Java's documentation, this is not necessary, and precludes keepalives. However in practise,
        // connection errors will not be released quickly enough and can cause a too many open files error.
        conn.disconnect();
    }

    res.executed = true;
    return res;
}