Java Code Examples for org.jsoup.Connection

The following examples show how to use org.jsoup.Connection. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: astor   Source File: HttpConnection.java    License: GNU General Public License v2.0 6 votes vote down vote up
private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse) throws IOException {
    method = Method.valueOf(conn.getRequestMethod());
    url = conn.getURL();
    statusCode = conn.getResponseCode();
    statusMessage = conn.getResponseMessage();
    contentType = conn.getContentType();

    Map<String, List<String>> resHeaders = createHeaderMap(conn);
    processResponseHeaders(resHeaders);

    // if from a redirect, map previous response cookies into this response
    if (previousResponse != null) {
        for (Map.Entry<String, String> prevCookie : previousResponse.cookies().entrySet()) {
            if (!hasCookie(prevCookie.getKey()))
                cookie(prevCookie.getKey(), prevCookie.getValue());
        }
    }
}
 
Example 2
Source Project: superword   Source File: Definition.java    License: Apache License 2.0 6 votes vote down vote up
private static String _getContent(String url) {
    Connection conn = Jsoup.connect(url)
            .header("Accept", ACCEPT)
            .header("Accept-Encoding", ENCODING)
            .header("Accept-Language", LANGUAGE)
            .header("Connection", CONNECTION)
            .header("Referer", REFERER)
            .header("Host", HOST)
            .header("User-Agent", USER_AGENT)
            .timeout(1000)
            .ignoreContentType(true);
    String html = "";
    try {
        html = conn.post().html();
        html = html.replaceAll("[\n\r]", "");
    }catch (Exception e){
        LOGGER.error("获取URL:" + url + "页面出错", e);
    }
    return html;
}
 
Example 3
Source Project: JsDroidCmd   Source File: HttpUtil.java    License: Mozilla Public License 2.0 6 votes vote down vote up
public static String get(String url, Map<String, String> headers,
		Map<String, String> params) {
	try {
		Connection connect = Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		if (params != null) {
			connect.data(params);
		}
		if (headers != null) {
			connect.headers(headers);
		}
		return connect.execute().body();
	} catch (Throwable e) {
	}
	return null;
}
 
Example 4
Source Project: DanDanPlayForAndroid   Source File: BiliBiliDownloadDialog.java    License: MIT License 6 votes vote down vote up
/**
 * 根据av号获取相关信息
 */
private void downloadByAv(String avNumber) throws Exception {
    if (avNumber.isEmpty()){
        sendToastMessage("请输入av号");
    }else {
        sendLogMessage("开始转换URL...\n");
        String url = "https://www.bilibili.com/video/av"+avNumber;
        Connection.Response response = Jsoup.connect(url).timeout(10000).execute();
        //普通视频不会重定向,番剧会进行重定向
        if (!response.url().toString().equals(url)){
            url = response.url().toString();
        }
        sendLogMessage("转换URL成功\n");
        downloadByUrl(url);
    }
}
 
Example 5
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 6 votes vote down vote up
@Test
public void handles200WithNoContent() throws IOException {
    Connection con = Jsoup
        .connect("http://direct.infohound.net/tools/200-no-content.pl")
        .userAgent(browserUa);
    Connection.Response res = con.execute();
    Document doc = res.parse();
    assertEquals(200, res.statusCode());

    con = Jsoup
        .connect("http://direct.infohound.net/tools/200-no-content.pl")
        .parser(Parser.xmlParser())
        .userAgent(browserUa);
    res = con.execute();
    doc = res.parse();
    assertEquals(200, res.statusCode());
}
 
Example 6
Source Project: ripme   Source File: PhotobucketRipper.java    License: MIT License 6 votes vote down vote up
@Override
public Document getNextPage(Document page) throws IOException {
    this.currAlbum.pageIndex++;
    boolean endOfAlbum = currAlbum.pageIndex > currAlbum.numPages;
    boolean noMoreSubalbums = albums.isEmpty();
    if (endOfAlbum && noMoreSubalbums){
        throw new IOException("No more pages");
    }
    try {
        Thread.sleep(WAIT_BEFORE_NEXT_PAGE);
    } catch (InterruptedException e) {
        LOGGER.info("Interrupted while waiting before getting next page");
    }
    if (endOfAlbum){
        LOGGER.info("Turning to next album " + albums.get(0).baseURL);
        return getFirstPage();
    } else {
        LOGGER.info("Turning to page " + currAlbum.pageIndex +
                " of album " + currAlbum.baseURL);
        Connection.Response resp = Http.url(currAlbum.getCurrPageURL()).response();
        currAlbum.cookies = resp.cookies();
        currAlbum.currPage = resp.parse();
        return currAlbum.currPage;
    }
}
 
Example 7
Source Project: guanggoo-android   Source File: UploadImageTask.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
    String errorMsg;
    try {
        Connection.Response response = doPostFileRequest(URL, KEY, mStream);
        if (response != null && response.statusCode() == ConstantUtil.HTTP_STATUS_200) {
            Gson gson = new Gson();
            UploadImageResponse entity = gson.fromJson(response.body(), UploadImageResponse.class);
            if (UploadImageResponse.CODE_SUCCESS.equals(entity.getCode())) {
                if (entity.getData() != null && entity.getData().getUrl() != null && !TextUtils.isEmpty(entity.getData().getUrl().getDistribute())) {
                    successOnUI(entity.getData().getUrl().getDistribute());
                    return;
                }
                errorMsg = "图片上传返回数据有误";
            } else {
                errorMsg = entity.getMsg();
            }
        } else {
            errorMsg = "图片上传失败";
        }
    } catch (Exception e) {
        e.printStackTrace();
        errorMsg = e.getMessage();
    }
    failedOnUI(errorMsg);
}
 
Example 8
Source Project: guanggoo-android   Source File: UploadImageTask.java    License: Apache License 2.0 6 votes vote down vote up
public static Connection.Response doPostFileRequest(String url, String key, InputStream inputStream) throws IOException {
    // Https请求
    if (url.startsWith("https")) {
        trustEveryone();
    }

    Map<String, String> data = new HashMap<>();
    data.put("apiType", "ali,juejin,Huluxia,Imgbb");

    String filename = UUID.randomUUID() + ".jpg";

    return Jsoup.connect(url)
            .method(Connection.Method.POST)
            .ignoreContentType(true)
            .timeout(120000)
            .data(key, filename, inputStream)
            .data(data)
            .execute();
}
 
Example 9
Source Project: SteamGifts   Source File: SyncFragment.java    License: MIT License 6 votes vote down vote up
@Override
protected void onPostExecute(Connection.Response response) {
    if (response != null && response.statusCode() == 200) {
        try {
            Log.v(TAG, "Response to JSON request: " + response.body());
            JSONObject root = new JSONObject(response.body());

            if ("success".equals(root.getString("type"))) {
                Activity activity = getFragment().getActivity();
                activity.setResult(CommonActivity.RESPONSE_SYNC_SUCCESSFUL);
                activity.finish();
                return;
            } else {
                String message = root.getString("msg");
                if (message != null) {
                    Toast.makeText(getFragment().getContext(), message, Toast.LENGTH_SHORT).show();
                    return;
                }
            }

        } catch (JSONException e) {
            Log.e(TAG, "Failed to parse JSON object", e);
        }
    }
    Toast.makeText(getFragment().getContext(), "Could not sync.", Toast.LENGTH_SHORT).show();
}
 
Example 10
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 6 votes vote down vote up
@Test public void canInterruptDocumentRead() throws IOException, InterruptedException {
    // todo - implement in interruptable channels, so it's immediate
    final String[] body = new String[1];
    Thread runner = new Thread(new Runnable() {
        public void run() {
            try {
                Connection.Response res = Jsoup.connect("http://jsscxml.org/serverload.stream")
                    .timeout(15 * 1000)
                    .execute();
                body[0] = res.parse().text();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }

        }
    });

    runner.start();
    Thread.sleep(1000 * 7);
    runner.interrupt();
    assertTrue(runner.isInterrupted());
    runner.join();

    assertTrue(body[0].length() > 0);
}
 
Example 11
Source Project: yahnac   Source File: HNewsApi.java    License: Apache License 2.0 6 votes vote down vote up
private void attemptLogin() {
    try {
        ConnectionProvider connectionProvider = Inject.connectionProvider();
        Connection.Response response = connectionProvider
                .loginConnection(username, password)
                .execute();

        String cookie = response.cookie("user");
        String cfduid = response.cookie("_cfduid");

        if (!TextUtils.isEmpty(cookie)) {
            subscriber.onNext(new Login(username, cookie, Login.Status.SUCCESSFUL));
        } else {
            subscriber.onNext(new Login(username, null, Login.Status.WRONG_CREDENTIALS));
        }

    } catch (IOException e) {
        subscriber.onError(e);
    }
}
 
Example 12
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 6 votes vote down vote up
@Test
public void maxBodySize() throws IOException {
    String url = "http://direct.infohound.net/tools/large.html"; // 280 K

    Connection.Response defaultRes = Jsoup.connect(url).execute();
    Connection.Response smallRes = Jsoup.connect(url).maxBodySize(50 * 1024).execute(); // crops
    Connection.Response mediumRes = Jsoup.connect(url).maxBodySize(200 * 1024).execute(); // crops
    Connection.Response largeRes = Jsoup.connect(url).maxBodySize(300 * 1024).execute(); // does not crop
    Connection.Response unlimitedRes = Jsoup.connect(url).maxBodySize(0).execute();

    int actualString = 280735;
    assertEquals(actualString, defaultRes.body().length());
    assertEquals(50 * 1024, smallRes.body().length());
    assertEquals(200 * 1024, mediumRes.body().length());
    assertEquals(actualString, largeRes.body().length());
    assertEquals(actualString, unlimitedRes.body().length());

    int actualDocText = 269541;
    assertEquals(actualDocText, defaultRes.parse().text().length());
    assertEquals(49165, smallRes.parse().text().length());
    assertEquals(196577, mediumRes.parse().text().length());
    assertEquals(actualDocText, largeRes.parse().text().length());
    assertEquals(actualDocText, unlimitedRes.parse().text().length());
}
 
Example 13
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 6 votes vote down vote up
@Test public void handlesEscapedRedirectUrls() throws IOException {
    String url = "http://www.altalex.com/documents/news/2016/12/06/questioni-civilistiche-conseguenti-alla-depenalizzazione";
    // sends: Location:http://shop.wki.it/shared/sso/sso.aspx?sso=&url=http%3a%2f%2fwww.altalex.com%2fsession%2fset%2f%3freturnurl%3dhttp%253a%252f%252fwww.altalex.com%253a80%252fdocuments%252fnews%252f2016%252f12%252f06%252fquestioni-civilistiche-conseguenti-alla-depenalizzazione
    // then to: http://www.altalex.com/session/set/?returnurl=http%3a%2f%2fwww.altalex.com%3a80%2fdocuments%2fnews%2f2016%2f12%2f06%2fquestioni-civilistiche-conseguenti-alla-depenalizzazione&sso=RDRG6T684G4AK2E7U591UGR923
    // then : http://www.altalex.com:80/documents/news/2016/12/06/questioni-civilistiche-conseguenti-alla-depenalizzazione

    // bug is that jsoup goes to
    // 	GET /shared/sso/sso.aspx?sso=&url=http%253a%252f%252fwww.altalex.com%252fsession%252fset%252f%253freturnurl%253dhttp%25253a%25252f%25252fwww.altalex.com%25253a80%25252fdocuments%25252fnews%25252f2016%25252f12%25252f06%25252fquestioni-civilistiche-conseguenti-alla-depenalizzazione HTTP/1.1
    // i.e. double escaped

    Connection.Response res = Jsoup.connect(url)
            .proxy("localhost", 8888)
            .execute();
    Document doc = res.parse();
    assertEquals(200, res.statusCode());
}
 
Example 14
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void redirectsResponseCookieToNextResponse() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-cookie.pl");
    Connection.Response res = con.execute();
    assertEquals("asdfg123", res.cookie("token")); // confirms that cookies set on 1st hit are presented in final result
    Document doc = res.parse();
    assertEquals("token=asdfg123; uid=jhy", ihVal("HTTP_COOKIE", doc)); // confirms that redirected hit saw cookie
}
 
Example 15
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void inWildUtfRedirect() throws IOException {
    Connection.Response res = Jsoup.connect("http://brabantn.ws/Q4F").execute();
    Document doc = res.parse();
    assertEquals(
        "http://www.omroepbrabant.nl/?news/2474781303/Gestrande+ree+in+Oss+niet+verdoofd,+maar+doodgeschoten+%E2%80%98Dit+kan+gewoon+niet,+bizar%E2%80%99+[VIDEO].aspx",
        doc.location()
        );
}
 
Example 16
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void fetchBaidu() throws IOException {
    Connection.Response res = Jsoup.connect("http://www.baidu.com/").timeout(10*1000).execute();
    Document doc = res.parse();

    assertEquals("GBK", doc.outputSettings().charset().displayName());
    assertEquals("GBK", res.charset());
    assert(res.hasCookie("BAIDUID"));
    assertEquals("text/html;charset=gbk", res.contentType());
}
 
Example 17
Source Project: JsDroidCmd   Source File: HttpUtil.java    License: Mozilla Public License 2.0 5 votes vote down vote up
public static void download(String url, String file) {
	try {
		Connection connect = org.jsoup.Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		FileUtil.cpyStream(connect.execute().bodyStream(),
				new FileOutputStream(file));
	} catch (Throwable e) {
	}
}
 
Example 18
Source Project: AIPa   Source File: AiPaUtil.java    License: Apache License 2.0 5 votes vote down vote up
public AiPaUtil(Charset charset, Map<String, String> header, Connection.Method method, int timeout, String userAgent, Map<String,String> cookies) {
    this.charset = charset;
    this.header = header;
    this.method = method;
    this.timeout = timeout;
    this.userAgent = userAgent;
    this.cookies = cookies;
}
 
Example 19
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void doesPut() throws IOException {
    Connection.Response res = Jsoup.connect(echoURL)
            .data("uname", "Jsoup", "uname", "Jonathan", "百", "度一下")
            .cookie("auth", "token")
            .method(Connection.Method.PUT)
            .execute();

    Document doc = res.parse();
    assertEquals("PUT", ihVal("REQUEST_METHOD", doc));
    //assertEquals("gzip", ihVal("HTTP_ACCEPT_ENCODING", doc)); // current proxy removes gzip on post
    assertEquals("auth=token", ihVal("HTTP_COOKIE", doc));
}
 
Example 20
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void combinesSameHeadersWithComma() throws IOException {
    // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
    String url = "http://direct.infohound.net/tools/q.pl";
    Connection con = Jsoup.connect(url);
    con.get();

    assertEquals("text/html", con.response().header("Content-Type"));
    assertEquals("no-cache, no-store", con.response().header("Cache-Control"));
}
 
Example 21
@Override
public boolean isCompatibleShaarli() throws IOException {
    final String loginFormUrl = this.mShaarliUrl + "?do=login";
    try {
        Connection.Response loginFormPage = this.newConnection(loginFormUrl, Connection.Method.GET).execute();
        this.mCookies = loginFormPage.cookies();
        this.mToken = loginFormPage.parse().body().select("input[name=token]").first().attr("value");
    } catch (NullPointerException | IllegalArgumentException e) {
        return false;
    }
    return true;
}
 
Example 22
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void sendHeadRequest() throws IOException {
    String url = "http://direct.infohound.net/tools/parse-xml.xml";
    Connection con = Jsoup.connect(url).method(Connection.Method.HEAD);
    final Connection.Response response = con.execute();
    assertEquals("text/xml", response.header("Content-Type"));
    assertEquals("", response.body()); // head ought to have no body
    Document doc = response.parse();
    assertEquals("", doc.text());
}
 
Example 23
Source Project: SteamGifts   Source File: EditCommentTask.java    License: MIT License 5 votes vote down vote up
@Override
protected void onPostExecute(Connection.Response response) {
    Activity activity = getFragment();
    if (response != null && response.statusCode() == 200) {
        try {
            Log.v(TAG, "Response to JSON request: " + response.body());
            JSONObject root = new JSONObject(response.body());

            boolean success = "success".equals(root.getString("type"));
            if (success) {
                Document commentHtml = Jsoup.parse(root.getString("comment"));

                // Save the content of the edit state for a bit & remove the edit state from being rendered.
                Element editState = commentHtml.select(".comment__edit-state.is-hidden textarea[name=description]").first();
                commentHtml.select(".comment__edit-state").html("");
                Element desc = commentHtml.select(".comment__description").first();

                if (editState == null)
                    Log.d(TAG, "edit state is null?");
                comment.setEditableContent(editState == null ? null : editState.text());
                comment.setContent(desc.html());

                Intent data = new Intent();
                data.putExtra("edited-comment", comment);
                activity.setResult(WriteCommentActivity.COMMENT_EDIT_SENT, data);
                activity.finish();

                return;
            }
        } catch (JSONException e) {
            Log.e(TAG, "Failed to parse JSON object", e);
        }
    }
    onFail();
}
 
Example 24
Source Project: yahnac   Source File: ConnectionProvider.java    License: Apache License 2.0 5 votes vote down vote up
private static Connection defaultConnection(String baseUrlExtension) {
    Connection conn = Jsoup.connect(BASE_URL + baseUrlExtension)
            .timeout(TIMEOUT_MILLIS)
            .userAgent(USER_AGENT);
    conn.header("Accept-Encoding", "gzip");

    return conn;
}
 
Example 25
Source Project: jsoup-learning   Source File: UrlConnectTest.java    License: MIT License 5 votes vote down vote up
@Test
public void redirectsResponseCookieToNextResponse() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-cookie.pl");
    Connection.Response res = con.execute();
    assertEquals("asdfg123", res.cookie("token")); // confirms that cookies set on 1st hit are presented in final result
    Document doc = res.parse();
    assertEquals("uid=jhy; token=asdfg123", ihVal("HTTP_COOKIE", doc)); // confirms that redirected hit saw cookie
}
 
Example 26
Source Project: SteamGifts   Source File: LoadWhitelistBlacklistTask.java    License: MIT License 5 votes vote down vote up
@Override
protected List<BasicUser> doInBackground(Void... params) {
    try {
        // Fetch the Giveaway page
        String url = "https://www.steamgifts.com/account/manage/" + what.name().toLowerCase(Locale.ENGLISH) + "/search";
        Log.d(TAG, "Fetching URL " + url);

        Connection jsoup = Jsoup.connect(url)
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT)
                .followRedirects(false);
        jsoup.data("page", Integer.toString(page));

        if (searchQuery != null)
            jsoup.data("q", searchQuery);

        jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());

        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        // Fetch the xsrf token
        Element xsrfToken = document.select("input[name=xsrf_token]").first();
        if (xsrfToken != null)
            foundXsrfToken = xsrfToken.attr("value");

        // Do away with pinned giveaways.
        document.select(".pinned-giveaways__outer-wrap").html("");

        // Parse all rows of giveaways
        return loadAll(document);
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 
Example 27
Source Project: astor   Source File: HttpConnection.java    License: GNU General Public License v2.0 5 votes vote down vote up
private static boolean needsMultipart(Connection.Request req) {
    // multipart mode, for files. add the header if we see something with an inputstream, and return a non-null boundary
    boolean needsMulti = false;
    for (Connection.KeyVal keyVal : req.data()) {
        if (keyVal.hasInputStream()) {
            needsMulti = true;
            break;
        }
    }
    return needsMulti;
}
 
Example 28
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void postRedirectsFetchWithGet() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302.pl")
            .data("Argument", "Riposte")
            .method(Connection.Method.POST);
    Connection.Response res = con.execute();
    assertEquals("http://jsoup.org", res.url().toExternalForm());
    assertEquals(Connection.Method.GET, res.method());
}
 
Example 29
Source Project: guanggoo-android   Source File: VoteCommentTask.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

    String xsrf = getXsrf();

    Map<String, String> headers = new HashMap<>();
    headers.put("Accept", "application/json, text/javascript, */*; q=0.01");
    headers.put("X-Requested-With", "XMLHttpRequest");
    headers.put("Content-Type", "application/x-www-form-urlencoded");

    Map<String, String> cookies = getCookies();
    if (!cookies.containsKey(ConstantUtil.KEY_XSRF)) {
        cookies.put(ConstantUtil.KEY_XSRF, xsrf);
    }

    try {
        Connection.Response res = Jsoup.connect(mUrl).cookies(cookies).headers(headers).method
                (Connection.Method.GET).execute();
        if (res.statusCode() == ConstantUtil.HTTP_STATUS_200 || res.statusCode() == ConstantUtil.HTTP_STATUS_302) {
            Gson gson = new Gson();
            BaseResponse response = gson.fromJson(res.body(), BaseResponse.class);
            if (response.getSuccess() == 1) {
                // 赞成功
                successOnUI(true);
            } else {
                // 以前已经赞过该评论
                successOnUI(false);
            }
            return;
        }
    } catch (IOException e) {
        e.printStackTrace();
    }

    failedOnUI("失败");
}
 
Example 30
Source Project: astor   Source File: UrlConnectTest.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Test
public void handlesDodgyCharset() throws IOException {
    // tests that when we get back "UFT8", that it is recognised as unsupported, and falls back to default instead
    String url = "http://direct.infohound.net/tools/bad-charset.pl";
    Connection.Response res = Jsoup.connect(url).execute();
    assertEquals("text/html; charset=UFT8", res.header("Content-Type")); // from the header
    assertEquals(null, res.charset()); // tried to get from header, not supported, so returns null
    Document doc = res.parse(); // would throw an error if charset unsupported
    assertTrue(doc.text().contains("Hello!"));
    assertEquals("UTF-8", res.charset()); // set from default on parse
}