org.jsoup.Connection Java Examples

private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse) throws IOException {
    method = Method.valueOf(conn.getRequestMethod());
    url = conn.getURL();
    statusCode = conn.getResponseCode();
    statusMessage = conn.getResponseMessage();
    contentType = conn.getContentType();

    Map<String, List<String>> resHeaders = createHeaderMap(conn);
    processResponseHeaders(resHeaders);

    // if from a redirect, map previous response cookies into this response
    if (previousResponse != null) {
        for (Map.Entry<String, String> prevCookie : previousResponse.cookies().entrySet()) {
            if (!hasCookie(prevCookie.getKey()))
                cookie(prevCookie.getKey(), prevCookie.getValue());
        }
    }
}
 

private static String _getContent(String url) {
    Connection conn = Jsoup.connect(url)
            .header("Accept", ACCEPT)
            .header("Accept-Encoding", ENCODING)
            .header("Accept-Language", LANGUAGE)
            .header("Connection", CONNECTION)
            .header("Referer", REFERER)
            .header("Host", HOST)
            .header("User-Agent", USER_AGENT)
            .timeout(1000)
            .ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
        html = html.replaceAll("[\n\r]", "");
    }catch (Exception e){
        LOGGER.error("获取URL：" + url + "页面出错", e);
    }
    return html;
}
 

public static String get(String url, Map<String, String> headers,
		Map<String, String> params) {
	try {
		Connection connect = Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		if (params != null) {
			connect.data(params);
		}
		if (headers != null) {
			connect.headers(headers);
		}
		return connect.execute().body();
	} catch (Throwable e) {
	}
	return null;
}
 

/**
 * 根据av号获取相关信息
 */
private void downloadByAv(String avNumber) throws Exception {
    if (avNumber.isEmpty()){
        sendToastMessage("请输入av号");
    }else {
        sendLogMessage("开始转换URL...\n");
        String url = "https://www.bilibili.com/video/av"+avNumber;
        Connection.Response response = Jsoup.connect(url).timeout(10000).execute();
        //普通视频不会重定向，番剧会进行重定向
        if (!response.url().toString().equals(url)){
            url = response.url().toString();
        }
        sendLogMessage("转换URL成功\n");
        downloadByUrl(url);
    }
}
 

@Test
public void handles200WithNoContent() throws IOException {
    Connection con = Jsoup
        .connect("http://direct.infohound.net/tools/200-no-content.pl")
        .userAgent(browserUa);
    Connection.Response res = con.execute();
    Document doc = res.parse();
    assertEquals(200, res.statusCode());

    con = Jsoup
        .connect("http://direct.infohound.net/tools/200-no-content.pl")
        .parser(Parser.xmlParser())
        .userAgent(browserUa);
    res = con.execute();
    doc = res.parse();
    assertEquals(200, res.statusCode());
}
 

@Override
public Document getNextPage(Document page) throws IOException {
    this.currAlbum.pageIndex++;
    boolean endOfAlbum = currAlbum.pageIndex > currAlbum.numPages;
    boolean noMoreSubalbums = albums.isEmpty();
    if (endOfAlbum && noMoreSubalbums){
        throw new IOException("No more pages");
    }
    try {
        Thread.sleep(WAIT_BEFORE_NEXT_PAGE);
    } catch (InterruptedException e) {
        LOGGER.info("Interrupted while waiting before getting next page");
    }
    if (endOfAlbum){
        LOGGER.info("Turning to next album " + albums.get(0).baseURL);
        return getFirstPage();
    } else {
        LOGGER.info("Turning to page " + currAlbum.pageIndex +
                " of album " + currAlbum.baseURL);
        Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response();
        currAlbum.cookies = resp.cookies();
        currAlbum.currPage = resp.parse();
        return currAlbum.currPage;
    }
}
 

@Override
public void run() {
    String errorMsg;
    try {
        Connection.Response response = doPostFileRequest(URL, KEY, mStream);
        if (response != null && response.statusCode() == ConstantUtil.HTTP_STATUS_200) {
            Gson gson = new Gson();
            UploadImageResponse entity = gson.fromJson(response.body(), UploadImageResponse.class);
            if (UploadImageResponse.CODE_SUCCESS.equals(entity.getCode())) {
                if (entity.getData() != null && entity.getData().getUrl() != null && !TextUtils.isEmpty(entity.getData().getUrl().getDistribute())) {
                    successOnUI(entity.getData().getUrl().getDistribute());
                    return;
                }
                errorMsg = "图片上传返回数据有误";
            } else {
                errorMsg = entity.getMsg();
            }
        } else {
            errorMsg = "图片上传失败";
        }
    } catch (Exception e) {
        e.printStackTrace();
        errorMsg = e.getMessage();
    }
    failedOnUI(errorMsg);
}
 

public static Connection.Response doPostFileRequest(String url, String key, InputStream inputStream) throws IOException {
    // Https请求
    if (url.startsWith("https")) {
        trustEveryone();
    }

    Map<String, String> data = new HashMap<>();
    data.put("apiType", "ali,juejin,Huluxia,Imgbb");

    String filename = UUID.randomUUID() + ".jpg";

    return Jsoup.connect(url)
            .method(Connection.Method.POST)
            .ignoreContentType(true)
            .timeout(120000)
            .data(key, filename, inputStream)
            .data(data)
            .execute();
}
 

@Override
protected void onPostExecute(Connection.Response response) {
    if (response != null && response.statusCode() == 200) {
        try {
            Log.v(TAG, "Response to JSON request: " + response.body());
            JSONObject root = new JSONObject(response.body());

            if ("success".equals(root.getString("type"))) {
                Activity activity = getFragment().getActivity();
                activity.setResult(CommonActivity.RESPONSE_SYNC_SUCCESSFUL);
                activity.finish();
                return;
            } else {
                String message = root.getString("msg");
                if (message != null) {
                    Toast.makeText(getFragment().getContext(), message, Toast.LENGTH_SHORT).show();
                    return;
                }
            }

        } catch (JSONException e) {
            Log.e(TAG, "Failed to parse JSON object", e);
        }
    }
    Toast.makeText(getFragment().getContext(), "Could not sync.", Toast.LENGTH_SHORT).show();
}
 

@Test public void canInterruptDocumentRead() throws IOException, InterruptedException {
    // todo - implement in interruptable channels, so it's immediate
    final String[] body = new String[1];
    Thread runner = new Thread(new Runnable() {
        public void run() {
            try {
                Connection.Response res = Jsoup.connect("http://jsscxml.org/serverload.stream")
                    .timeout(15 * 1000)
                    .execute();
                body[0] = res.parse().text();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }

        }
    });

    runner.start();
    Thread.sleep(1000 * 7);
    runner.interrupt();
    assertTrue(runner.isInterrupted());
    runner.join();

    assertTrue(body[0].length() > 0);
}
 

private void attemptLogin() {
    try {
        ConnectionProvider connectionProvider = Inject.connectionProvider();
        Connection.Response response = connectionProvider
                .loginConnection(username, password)
                .execute();

        String cookie = response.cookie("user");
        String cfduid = response.cookie("_cfduid");

        if (!TextUtils.isEmpty(cookie)) {
            subscriber.onNext(new Login(username, cookie, Login.Status.SUCCESSFUL));
        } else {
            subscriber.onNext(new Login(username, null, Login.Status.WRONG_CREDENTIALS));
        }

    } catch (IOException e) {
        subscriber.onError(e);
    }
}
 

@Test
public void maxBodySize() throws IOException {
    String url = "http://direct.infohound.net/tools/large.html"; // 280 K

    Connection.Response defaultRes = Jsoup.connect(url).execute();
    Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops
    Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops
    Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop
    Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();

    int actualString = 280735;
    assertEquals(actualString, defaultRes.body().length());
    assertEquals(50 * 1024, smallRes.body().length());
    assertEquals(200 * 1024, mediumRes.body().length());
    assertEquals(actualString, largeRes.body().length());
    assertEquals(actualString, unlimitedRes.body().length());

    int actualDocText = 269541;
    assertEquals(actualDocText, defaultRes.parse().text().length());
    assertEquals(49165, smallRes.parse().text().length());
    assertEquals(196577, mediumRes.parse().text().length());
    assertEquals(actualDocText, largeRes.parse().text().length());
    assertEquals(actualDocText, unlimitedRes.parse().text().length());
}
 

@Test public void handlesEscapedRedirectUrls() throws IOException {
    String url = "http://www.altalex.com/documents/news/2016/12/06/questioni-civilistiche-conseguenti-alla-depenalizzazione";
    // sends: Location:http://shop.wki.it/shared/sso/sso.aspx?sso=&url=http%3a%2f%2fwww.altalex.com%2fsession%2fset%2f%3freturnurl%3dhttp%253a%252f%252fwww.altalex.com%253a80%252fdocuments%252fnews%252f2016%252f12%252f06%252fquestioni-civilistiche-conseguenti-alla-depenalizzazione
    // then to: http://www.altalex.com/session/set/?returnurl=http%3a%2f%2fwww.altalex.com%3a80%2fdocuments%2fnews%2f2016%2f12%2f06%2fquestioni-civilistiche-conseguenti-alla-depenalizzazione&sso=RDRG6T684G4AK2E7U591UGR923
    // then : http://www.altalex.com:80/documents/news/2016/12/06/questioni-civilistiche-conseguenti-alla-depenalizzazione

    // bug is that jsoup goes to
    // 	GET /shared/sso/sso.aspx?sso=&url=http%253a%252f%252fwww.altalex.com%252fsession%252fset%252f%253freturnurl%253dhttp%25253a%25252f%25252fwww.altalex.com%25253a80%25252fdocuments%25252fnews%25252f2016%25252f12%25252f06%25252fquestioni-civilistiche-conseguenti-alla-depenalizzazione HTTP/1.1
    // i.e. double escaped

    Connection.Response res = Jsoup.connect(url)
            .proxy("localhost", 8888)
            .execute();
    Document doc = res.parse();
    assertEquals(200, res.statusCode());
}
 

@Test
public void redirectsResponseCookieToNextResponse() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-cookie.pl");
    Connection.Response res = con.execute();
    assertEquals("asdfg123", res.cookie("token")); // confirms that cookies set on 1st hit are presented in final result
    Document doc = res.parse();
    assertEquals("token=asdfg123; uid=jhy", ihVal("HTTP_COOKIE", doc)); // confirms that redirected hit saw cookie
}
 

@Test
public void inWildUtfRedirect() throws IOException {
    Connection.Response res = Jsoup.connect("http://brabantn.ws/Q4F").execute();
    Document doc = res.parse();
    assertEquals(
        "http://www.omroepbrabant.nl/?news/2474781303/Gestrande+ree+in+Oss+niet+verdoofd,+maar+doodgeschoten+%E2%80%98Dit+kan+gewoon+niet,+bizar%E2%80%99+[VIDEO].aspx",
        doc.location()
        );
}
 

@Test
public void fetchBaidu() throws IOException {
    Connection.Response res = Jsoup.connect("http://www.baidu.com/").timeout(10*1000).execute();
    Document doc = res.parse();

    assertEquals("GBK", doc.outputSettings().charset().displayName());
    assertEquals("GBK", res.charset());
    assert(res.hasCookie("BAIDUID"));
    assertEquals("text/html;charset=gbk", res.contentType());
}
 

public static void download(String url, String file) {
	try {
		Connection connect = org.jsoup.Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		FileUtil.cpyStream(connect.execute().bodyStream(),
				new FileOutputStream(file));
	} catch (Throwable e) {
	}
}
 

public AiPaUtil(Charset charset, Map<String, String> header, Connection.Method method, int timeout, String userAgent, Map<String,String> cookies) {
    this.charset = charset;
    this.header = header;
    this.method = method;
    this.timeout = timeout;
    this.userAgent = userAgent;
    this.cookies = cookies;
}
 

@Test
public void doesPut() throws IOException {
    Connection.Response res = Jsoup.connect(echoURL)
            .data("uname", "Jsoup", "uname", "Jonathan", "百", "度一下")
            .cookie("auth", "token")
            .method(Connection.Method.PUT)
            .execute();

    Document doc = res.parse();
    assertEquals("PUT", ihVal("REQUEST_METHOD", doc));
    //assertEquals("gzip", ihVal("HTTP_ACCEPT_ENCODING", doc)); // current proxy removes gzip on post
    assertEquals("auth=token", ihVal("HTTP_COOKIE", doc));
}
 

@Test
public void combinesSameHeadersWithComma() throws IOException {
    // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
    String url = "http://direct.infohound.net/tools/q.pl";
    Connection con = Jsoup.connect(url);
    con.get();

    assertEquals("text/html", con.response().header("Content-Type"));
    assertEquals("no-cache, no-store", con.response().header("Cache-Control"));
}
 

@Override
public boolean isCompatibleShaarli() throws IOException {
    final String loginFormUrl = this.mShaarliUrl + "?do=login";
    try {
        Connection.Response loginFormPage = this.newConnection(loginFormUrl, Connection.Method.GET).execute();
        this.mCookies = loginFormPage.cookies();
        this.mToken = loginFormPage.parse().body().select("input[name=token]").first().attr("value");
    } catch (NullPointerException | IllegalArgumentException e) {
        return false;
    }
    return true;
}
 

@Test
public void sendHeadRequest() throws IOException {
    String url = "http://direct.infohound.net/tools/parse-xml.xml";
    Connection con = Jsoup.connect(url).method(Connection.Method.HEAD);
    final Connection.Response response = con.execute();
    assertEquals("text/xml", response.header("Content-Type"));
    assertEquals("", response.body()); // head ought to have no body
    Document doc = response.parse();
    assertEquals("", doc.text());
}
 

@Override
protected void onPostExecute(Connection.Response response) {
    Activity activity = getFragment();
    if (response != null && response.statusCode() == 200) {
        try {
            Log.v(TAG, "Response to JSON request: " + response.body());
            JSONObject root = new JSONObject(response.body());

            boolean success = "success".equals(root.getString("type"));
            if (success) {
                Document commentHtml = Jsoup.parse(root.getString("comment"));

                // Save the content of the edit state for a bit & remove the edit state from being rendered.
                Element editState = commentHtml.select(".comment__edit-state.is-hidden textarea[name=description]").first();
                commentHtml.select(".comment__edit-state").html("");
                Element desc = commentHtml.select(".comment__description").first();

                if (editState == null)
                    Log.d(TAG, "edit state is null?");
                comment.setEditableContent(editState == null ? null : editState.text());
                comment.setContent(desc.html());

                Intent data = new Intent();
                data.putExtra("edited-comment", comment);
                activity.setResult(WriteCommentActivity.COMMENT_EDIT_SENT, data);
                activity.finish();

                return;
            }
        } catch (JSONException e) {
            Log.e(TAG, "Failed to parse JSON object", e);
        }
    }
    onFail();
}
 

private static Connection defaultConnection(String baseUrlExtension) {
    Connection conn = Jsoup.connect(BASE_URL + baseUrlExtension)
            .timeout(TIMEOUT_MILLIS)
            .userAgent(USER_AGENT);
    conn.header("Accept-Encoding", "gzip");

    return conn;
}
 

@Test
public void redirectsResponseCookieToNextResponse() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-cookie.pl");
    Connection.Response res = con.execute();
    assertEquals("asdfg123", res.cookie("token")); // confirms that cookies set on 1st hit are presented in final result
    Document doc = res.parse();
    assertEquals("uid=jhy; token=asdfg123", ihVal("HTTP_COOKIE", doc)); // confirms that redirected hit saw cookie
}
 

@Override
protected List<BasicUser> doInBackground(Void... params) {
    try {
        // Fetch the Giveaway page
        String url = "https://www.steamgifts.com/account/manage/" + what.name().toLowerCase(Locale.ENGLISH) + "/search";
        Log.d(TAG, "Fetching URL " + url);

        Connection jsoup = Jsoup.connect(url)
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT)
                .followRedirects(false);
        jsoup.data("page", Integer.toString(page));

        if (searchQuery != null)
            jsoup.data("q", searchQuery);

        jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());

        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        // Fetch the xsrf token
        Element xsrfToken = document.select("input[name=xsrf_token]").first();
        if (xsrfToken != null)
            foundXsrfToken = xsrfToken.attr("value");

        // Do away with pinned giveaways.
        document.select(".pinned-giveaways__outer-wrap").html("");

        // Parse all rows of giveaways
        return loadAll(document);
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 

private static boolean needsMultipart(Connection.Request req) {
    // multipart mode, for files. add the header if we see something with an inputstream, and return a non-null boundary
    boolean needsMulti = false;
    for (Connection.KeyVal keyVal : req.data()) {
        if (keyVal.hasInputStream()) {
            needsMulti = true;
            break;
        }
    }
    return needsMulti;
}
 

@Test
public void postRedirectsFetchWithGet() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302.pl")
            .data("Argument", "Riposte")
            .method(Connection.Method.POST);
    Connection.Response res = con.execute();
    assertEquals("http://jsoup.org", res.url().toExternalForm());
    assertEquals(Connection.Method.GET, res.method());
}
 

@Override
public void run() {

    String xsrf = getXsrf();

    Map<String, String> headers = new HashMap<>();
    headers.put("Accept", "application/json, text/javascript, */*; q=0.01");
    headers.put("X-Requested-With", "XMLHttpRequest");
    headers.put("Content-Type", "application/x-www-form-urlencoded");

    Map<String, String> cookies = getCookies();
    if (!cookies.containsKey(ConstantUtil.KEY_XSRF)) {
        cookies.put(ConstantUtil.KEY_XSRF, xsrf);
    }

    try {
        Connection.Response res = Jsoup.connect(mUrl).cookies(cookies).headers(headers).method
                (Connection.Method.GET).execute();
        if (res.statusCode() == ConstantUtil.HTTP_STATUS_200 || res.statusCode() == ConstantUtil.HTTP_STATUS_302) {
            Gson gson = new Gson();
            BaseResponse response = gson.fromJson(res.body(), BaseResponse.class);
            if (response.getSuccess() == 1) {
                // 赞成功
                successOnUI(true);
            } else {
                // 以前已经赞过该评论
                successOnUI(false);
            }
            return;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

    failedOnUI("失败");
}
 

@Test
public void handlesDodgyCharset() throws IOException {
    // tests that when we get back "UFT8", that it is recognised as unsupported, and falls back to default instead
    String url = "http://direct.infohound.net/tools/bad-charset.pl";
    Connection.Response res = Jsoup.connect(url).execute();
    assertEquals("text/html; charset=UFT8", res.header("Content-Type")); // from the header
    assertEquals(null, res.charset()); // tried to get from header, not supported, so returns null
    Document doc = res.parse(); // would throw an error if charset unsupported
    assertTrue(doc.text().contains("Hello!"));
    assertEquals("UTF-8", res.charset()); // set from default on parse
}