Java Code Examples for org.jsoup.Connection#data()

The following examples show how to use org.jsoup.Connection#data() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HttpUtil.java    From JsDroidCmd with Mozilla Public License 2.0 6 votes vote down vote up
public static String post(String url, Map<String, String> headers,
		Map<String, String> params) {
	try {
		Connection connect = Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		if (params != null) {
			connect.data(params);
		}
		if (headers != null) {
			connect.headers(headers);
		}
		connect.method(Method.POST);
		return connect.execute().body();
	} catch (Throwable e) {
	}
	return null;
}
 
Example 2
Source File: LoadGiveawayWinnersTask.java    From SteamGifts with MIT License 6 votes vote down vote up
@Override
protected List<Winner> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for page " + page);

    try {
        // Fetch the Giveaway page

        Connection jsoup = Jsoup.connect("https://www.steamgifts.com/giveaway/" + path + "/winners/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn())
            jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        return loadAll(document);
    } catch (IOException e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 
Example 3
Source File: PasswordNetworkManager.java    From Shaarlier with GNU General Public License v3.0 6 votes vote down vote up
/**
 * Method which publishes a link to shaarli
 * Assume being logged in
 * TODO: use the prefetch function
 */
@Override
public void pushLink(Link link) throws IOException {
    String encodedShareUrl = URLEncoder.encode(link.getUrl(), "UTF-8");
    retrievePostLinkToken(encodedShareUrl);

    if (NetworkUtils.isUrl(link.getUrl())) { // In case the url isn't really one, just post the one chosen by the server.
        this.mSharedUrl = link.getUrl();
    }

    final String postUrl = this.mShaarliUrl + "?post=" + encodedShareUrl;

    Connection postPageConn = this.newConnection(postUrl, Connection.Method.POST)
            .data("save_edit", "Save")
            .data("token", this.mToken)
            .data("lf_tags", link.getTags())
            .data("lf_linkdate", this.mDatePostLink)
            .data("lf_url", this.mSharedUrl)
            .data("lf_title", link.getTitle())
            .data("lf_description", link.getDescription());
    if (link.isPrivate()) postPageConn.data("lf_private", "on");
    if (link.isTweet()) postPageConn.data("tweet", "on");
    if (link.isToot()) postPageConn.data("toot", "on");
    postPageConn.execute(); // Then we post
}
 
Example 4
Source File: UrlConnectTest.java    From astor with GNU General Public License v2.0 6 votes vote down vote up
/**
 * Test fetching a form, and submitting it with a file attached.
 */
@Test
public void postHtmlFile() throws IOException {
    Document index = Jsoup.connect("http://direct.infohound.net/tidy/").get();
    FormElement form = index.select("[name=tidy]").forms().get(0);
    Connection post = form.submit();

    File uploadFile = ParseTest.getFile("/htmltests/google-ipod.html");
    FileInputStream stream = new FileInputStream(uploadFile);
    
    Connection.KeyVal fileData = post.data("_file");
    fileData.value("check.html");
    fileData.inputStream(stream);

    Connection.Response res;
    try {
        res = post.execute();
    } finally {
        stream.close();
    }

    Document out = res.parse();
    assertTrue(out.text().contains("HTML Tidy Complete"));
}
 
Example 5
Source File: UrlConnectTest.java    From jsoup-learning with MIT License 5 votes vote down vote up
@Test
public void followsRedirectToHttps() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-secure.pl"); // https://www.google.com
    con.data("id", "5");
    Document doc = con.get();
    assertTrue(doc.title().contains("Google"));
}
 
Example 6
Source File: JspToHtml.java    From DWSurvey with GNU Affero General Public License v3.0 5 votes vote down vote up
public void postJspToHtml(String postUrl, String filePath,String fileName) throws Exception{
	HttpServletRequest request=Struts2Utils.getRequest();
	//${pageContext.request.scheme}://${pageContext.request.serverName }:${pageContext.request.serverPort} pageContext.request.contextPath
	String reqTarget = request.getScheme()+"://"+request.getServerName()+(request.getServerPort()==80?"":":"+request.getServerPort())+request.getContextPath();
	reqTarget =reqTarget+"/toHtml";
	//?url="+postUrl+"&filePath="+filePath+"&fileName="+fileName;
	Map<String, String> map=new HashMap<String, String>();
	map.put("url", postUrl);
	map.put("filePath", filePath);
	map.put("fileName", fileName);
	Connection connection = Jsoup.connect(reqTarget);
	connection.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31");
	connection.data(map);
	Document doc=connection.timeout(8000).get();
}
 
Example 7
Source File: WebConnector.java    From JavaSkype with MIT License 5 votes vote down vote up
private Response sendRequest(Method method, String apiPath, boolean absoluteApiPath, String... keyval) throws IOException {
  String url = absoluteApiPath ? apiPath : SERVER_HOSTNAME + apiPath;
  Connection conn = Jsoup.connect(url).maxBodySize(100 * 1024 * 1024).timeout(10000).method(method).ignoreContentType(true).ignoreHttpErrors(true);
  logger.finest("Sending " + method + " request at " + url);
  if (skypeToken != null) {
    conn.header("X-Skypetoken", skypeToken);
  } else {
    logger.fine("No token sent for the request at: " + url);
  }
  conn.data(keyval);
  return conn.execute();
}
 
Example 8
Source File: LoadGameListTask.java    From SteamGifts with MIT License 5 votes vote down vote up
@Override
protected List<IEndlessAdaptable> doInBackground(Void... params) {
    try {
        // Fetch the Giveaway page

        Connection jsoup = Jsoup.connect("https://www.steamgifts.com/" + pathSegment + "/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (searchQuery != null)
            jsoup.data("q", searchQuery);

        jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(context).getSessionId());

        Document document = jsoup.get();

        SteamGiftsUserData.extract(context, document);

        // Fetch the xsrf token
        Element xsrfToken = document.select("input[name=xsrf_token]").first();
        if (xsrfToken != null)
            foundXsrfToken = xsrfToken.attr("value");

        // Do away with pinned giveaways.
        document.select(".pinned-giveaways__outer-wrap").html("");

        // Parse all rows of giveaways
        return loadAll(document);
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 
Example 9
Source File: HttpConnectionTest.java    From jsoup-learning with MIT License 4 votes vote down vote up
@Test(expected=IllegalArgumentException.class) public void throwsOnOdddData() {
    Connection con = HttpConnection.connect("http://example.com/");
    con.data("Name", "val", "what");
}
 
Example 10
Source File: LoadGiveawayGroupsTask.java    From SteamGifts with MIT License 4 votes vote down vote up
@Override
protected List<GiveawayGroup> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for page " + page);

    try {
        // Fetch the Giveaway page

        Connection jsoup = Jsoup.connect("https://www.steamgifts.com/giveaway/" + path + "/groups/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn())
            jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        // Parse all rows of groups
        Elements groups = document.select(".table__row-inner-wrap");
        Log.d(TAG, "Found inner " + groups.size() + " elements");

        List<GiveawayGroup> groupList = new ArrayList<>();
        for (Element element : groups) {
            Element link = element.select(".table__column__heading").first();

            // Basic information
            String title = link.text();
            String id = link.attr("href").substring(7, 12);

            String avatar = null;
            Element avatarNode = element.select(".global__image-inner-wrap").first();
            if (avatarNode != null)
                avatar = Utils.extractAvatar(avatarNode.attr("style"));

            GiveawayGroup group = new GiveawayGroup(id, title, avatar);
            groupList.add(group);
        }

        return groupList;
    } catch (IOException e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 
Example 11
Source File: UpdateGiveawayFilterTask.java    From SteamGifts with MIT License 4 votes vote down vote up
@Override
protected void addExtraParameters(Connection connection) {
    connection.data("game_id", String.valueOf(internalGameId));
}
 
Example 12
Source File: LoadUserTradeFeedbackTask.java    From SteamGifts with MIT License 4 votes vote down vote up
@Override
protected List<Comment> doInBackground(Void... params) {
    Log.d(TAG, "Fetching giveaways for user " + steamID64 + " (" + rating + ") on page " + page);

    try {
        // Fetch the Giveaway page
        Connection connection = Jsoup.connect("https://www.steamtrades.com/user/" + steamID64 + "/search")
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        connection.data("page", Integer.toString(page));
        connection.data("rating", rating);

        /* FIXME broken with the split of steamtrades & steamgifts
        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn()) {
            connection.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
            connection.followRedirects(false);
        }
        */

        Connection.Response response = connection.execute();
        Document document = response.parse();

        if (response.statusCode() == 200) {
            // FIXME SteamGiftsUserData.extract(fragment.getContext(), document);

            //if (!user.isLoaded())
                //foundXsrfToken = Utils.loadUserProfile(user, document);

            user.setPositiveFeedback(Utils.parseInt(document.select(".increment_positive_review_count").text()));
            user.setNegativeFeedback(Utils.parseInt(document.select(".increment_negative_review_count").text()));


            Element rootCommentNode = document.select(".reviews").first();
            if (rootCommentNode != null) {
                // Parse all rows of giveaways
                ICommentHolder holder = new ICommentHolder() {
                    private List<Comment> list = new ArrayList<>();

                    @Override
                    public List<Comment> getComments() {
                        return list;
                    }

                    @Override
                    public void addComment(Comment comment) {
                        list.add(comment);
                    }
                };
                Utils.loadComments(rootCommentNode, holder, 0, false, true, Comment.Type.TRADE_FEEDBACK);
                return holder.getComments();
            } else
                return new ArrayList<>();
        } else {
            Log.w(TAG, "Got status code " + response.statusCode());
            return null;
        }
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 
Example 13
Source File: LoadGiveawayListTask.java    From SteamGifts with MIT License 4 votes vote down vote up
private void addFilterParameter(Connection jsoup, String parameterName, int value) {
    if (value >= 0)
        jsoup.data(parameterName, String.valueOf(value));
}
 
Example 14
Source File: LoadDiscussionListTask.java    From SteamGifts with MIT License 4 votes vote down vote up
@Override
protected List<Discussion> doInBackground(Void... params) {
    try {
        // Fetch the Giveaway page
        String segment = "";
        if (type != DiscussionListFragment.Type.ALL)
            segment = type.name().replace("_", "-").toLowerCase(Locale.ENGLISH) + "/";
        String url = "https://www.steamgifts.com/discussions/" + segment + "search";

        Log.d(TAG, "Fetching discussions for page " + page + " and URL " + url);

        Connection jsoup = Jsoup.connect(url)
                .userAgent(Constants.JSOUP_USER_AGENT)
                .timeout(Constants.JSOUP_TIMEOUT);
        jsoup.data("page", Integer.toString(page));

        if (searchQuery != null)
            jsoup.data("q", searchQuery);

        // We do not want to follow redirects here, because SteamGifts redirects to the main (giveaways) page if we're not logged in.
        // For all other pages however, if we're not logged in, we're redirected once as well?
        if (type == DiscussionListFragment.Type.CREATED)
            jsoup.followRedirects(false);

        if (SteamGiftsUserData.getCurrent(fragment.getContext()).isLoggedIn())
            jsoup.cookie("PHPSESSID", SteamGiftsUserData.getCurrent(fragment.getContext()).getSessionId());
        Document document = jsoup.get();

        SteamGiftsUserData.extract(fragment.getContext(), document);

        // Parse all rows of discussions
        Elements discussions = document.select(".table__row-inner-wrap");
        Log.d(TAG, "Found inner " + discussions.size() + " elements");

        List<Discussion> discussionList = new ArrayList<>();
        for (Element element : discussions) {
            Element link = element.select("h3 a").first();

            // Basic information
            Uri uri = Uri.parse(link.attr("href"));
            String discussionId = uri.getPathSegments().get(1);
            String discussionName = uri.getPathSegments().get(2);

            Discussion discussion = new Discussion(discussionId);
            discussion.setTitle(link.text());
            discussion.setName(discussionName);

            Element p = element.select(".table__column--width-fill p").first();
            discussion.setCreatedTime(Integer.valueOf(p.select("span").first().attr("data-timestamp")));
            discussion.setCreator(p.select("a").last().text());

            // The creator's avatar
            Element avatarNode = element.select(".table_image_avatar").first();
            if (avatarNode != null)
                discussion.setCreatorAvatar(Utils.extractAvatar(avatarNode.attr("style")));

            discussion.setLocked(element.hasClass("is-faded"));
            discussion.setPoll(!element.select("h3 i.fa-align-left").isEmpty());
            discussionList.add(discussion);
        }
        return discussionList;
    } catch (Exception e) {
        Log.e(TAG, "Error fetching URL", e);
        return null;
    }
}
 
Example 15
Source File: EnterLeaveGiveawayTask.java    From SteamGifts with MIT License 4 votes vote down vote up
@Override
public void addExtraParameters(Connection connection) {
    connection.data("code", giveawayId);
}
 
Example 16
Source File: JsoupUtil.java    From crawler-jsoup-maven with Apache License 2.0 4 votes vote down vote up
/**
 * 方法说明:根据绑定的数据type选择绑定数据种类模拟浏览器
 * 
 * @param url           request url
 * @param bindData      bind data
 * @param requestType   request type: "headers" "data" "cookies" etc.
 * @return              Document object.
 * @throws Exception    Exception
 */
public static Document getDocument(String url, Map<String, String> bindData, String requestType) throws Exception {

    Document doc = null;
    Connection conn = null;
    StringWriter strWriter = new StringWriter();
    PrintWriter prtWriter = new PrintWriter(strWriter);

    // En:get max retry count from properties file(com-constants.properties)
    // Jp:プロパティファイルでロックタイムアウトのリトライ回数を取得する Zh:通过properties获取最大retry次数
    int maxRetry = Integer.parseInt(PropertyReader.getProperties(SystemConstants.COM_CONSTANTS)
            .getProperty(UtilsConstants.MAX_RETRY_COUNT));
    // En: get sleep time from properties file Jp:プロパティファイルでロックタイムアウトのスリープ時間を取得する
    int sleepTime = Integer.parseInt(PropertyReader.getProperties(SystemConstants.COM_CONSTANTS)
            .getProperty(UtilsConstants.SLEEP_TIME_COUNT));
    
    int temp = 0;

    // En: if exception is occurred then retry loop is continue to run;
    // Jp: 異常を起きる場合、ループを続き実行する。
    for (int j = 1; j <= maxRetry; j++) {

        try {
            if (j != 1) {
                Thread.sleep(sleepTime);
            }
            temp = Integer.parseInt(String.valueOf(Math.round(Math.random() * (UserAgent.length - 1))));
            conn = Jsoup.connect(url).timeout(10000)
                   // .userAgent(
                            // add userAgent. TODO There is a plan to configure userAgent to load that userAgent from a property file.
                     //       "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.122 Safari/534.30");
                    .userAgent(UserAgent[temp]);
            if (bindData != null && bindData.size() != 0 && !StringUtil.isEmpty(requestType)) {
                switch (requestType) {
                case UtilsConstants.REQUEST_HEADERS:
                    // adds each of the supplied headers to the request. // bluetata 2017/03/22 add
                    conn.headers(bindData);
                    break;
                case UtilsConstants.REQUEST_DATA:
                    // adds all of the supplied data to the request data parameters. // 20170320 bluetata add
                    conn.data(bindData);
                    break;
                case UtilsConstants.REQUEST_COOKIES:
                    // adds each of the supplied cookies to the request. // bluetata 2017/03/22 add
                    conn.cookies(bindData);
                    break;
                default:
                    // TODO stream etc. logic is adding. bluetata 2017/03/22 add
                    break;
                }
            }
            doc = conn.get();

            // En: normal finish situation,loop is broken.
            // Jp: サービスが正常に終了した場合、ループを中止します。
            // Zh: 正常终了的情况、终止循环。
            break;

        } catch (Exception ex) {
            // if throw new Exception(ex); dead code is occurred, retry is invalid.

            // StackTraceを文字列で取得
            ex.printStackTrace(prtWriter);
            String stackTrace = strWriter.toString();

            if (strWriter != null) {
                try {
                    strWriter.close();
                } catch (IOException ioe) {
                    ioe.printStackTrace();
                }
            }
            if (prtWriter != null) {
                prtWriter.close();
            }

            // En:info log is output. Jp: Infoログとして、エラー内容を出力。 Zh:输出到info log。
            Log4jUtil.error("第" + j + "次请求异常。");
            Log4jUtil.error(stackTrace);
        }
    }
    return doc;
}
 
Example 17
Source File: JsoupUtil.java    From xxl-crawler with GNU General Public License v3.0 4 votes vote down vote up
public static String loadPageSource(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageRequest.getUrl());
        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            conn.data(pageRequest.getParamMap());
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            conn.cookies(pageRequest.getCookieMap());
        }
        if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
            conn.headers(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent()!=null) {
            conn.userAgent(pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            conn.referrer(pageRequest.getReferrer());
        }
        conn.timeout(pageRequest.getTimeoutMillis());
        conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
        conn.maxBodySize(0);    // 取消默认1M限制

        // 代理
        if (pageRequest.getProxy() != null) {
            conn.proxy(pageRequest.getProxy());
        }

        conn.ignoreContentType(true);
        conn.method(pageRequest.isIfPost()?Connection.Method.POST:Connection.Method.GET);

        // 发出请求
        Connection.Response resp = conn.execute();
        String pageSource = resp.body();
        return pageSource;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 
Example 18
Source File: JsoupUtil.java    From xxl-crawler with GNU General Public License v3.0 4 votes vote down vote up
/**
 * 加载页面
 *
 * @param pageRequest
 *
 * @return Document
 */
public static Document load(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageRequest.getUrl());
        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            conn.data(pageRequest.getParamMap());
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            conn.cookies(pageRequest.getCookieMap());
        }
        if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
            conn.headers(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent()!=null) {
            conn.userAgent(pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            conn.referrer(pageRequest.getReferrer());
        }
        conn.timeout(pageRequest.getTimeoutMillis());
        conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
        conn.maxBodySize(0);    // 取消默认1M限制

        // 代理
        if (pageRequest.getProxy() != null) {
            conn.proxy(pageRequest.getProxy());
        }

        // 发出请求
        Document html = null;
        if (pageRequest.isIfPost()) {
            html = conn.post();
        } else {
            html = conn.get();
        }
        return html;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 
Example 19
Source File: HttpConnectionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test(expected=IllegalArgumentException.class) public void throwsOnOddData() {
    Connection con = HttpConnection.connect("http://example.com/");
    con.data("Name", "val", "what");
}
 
Example 20
Source File: HttpConnectionTest.java    From astor with GNU General Public License v2.0 4 votes vote down vote up
@Test(expected=IllegalArgumentException.class) public void throwsOnOdddData() {
    Connection con = HttpConnection.connect("http://example.com/");
    con.data("Name", "val", "what");
}