Java Code Examples for org.jsoup.Jsoup.connect()

The following are Jave code examples for showing how to use connect() of the org.jsoup.Jsoup class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
+ Save this method
Example 1
Project: wulkanowy   File: Api.java   View Source Code Vote up 6 votes
public Document postPageByUrl(String url, String[][] params) throws IOException {
    Connection connection = Jsoup.connect(url);

    for (String[] data : params) {
        connection.data(data[0], data[1]);
    }

    Connection.Response response = connection.cookies(getCookies())
            .followRedirects(true)
            .method(Connection.Method.POST)
            .execute();

    this.cookies.addItems(response.cookies());

    return response.parse();
}
 
Example 2
Project: EducationalAdministrationSystem   File: HttpUtils.java   View Source Code Vote up 6 votes
/**
 * 获取学校通知接口
 *
 * @param cookiesMap
 * @param index
 * @return
 */
public static Response getSchoolNotice(Map<String, String> cookiesMap, int index) {

    try {
        Connection con = Jsoup.connect(Constant.SchoolNotice.URL);
        con.ignoreContentType(true);
        Iterator<Map.Entry<String, String>> it = cookiesMap.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, String> en = it.next();
            con = con.cookie(en.getKey(), en.getValue());
        }

        return con.method(Connection.Method.POST)
                .data(Constant.SchoolNotice.PARAM_PAGE_SIZE, "10")
                .data(Constant.SchoolNotice.PARAM_PAGE_INDEX, String.valueOf(index))
                .data(Constant.SchoolNotice.PARAM_A_TITLE, "")
                .data(Constant.SchoolNotice.PARAM_ORDER_BY_TYPE, "asc")
                .timeout(10000)
                .execute();

    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 
Example 3
Project: EducationalAdministrationSystem   File: HttpUtils.java   View Source Code Vote up 6 votes
/**
 * 获取课程id
 *
 * @param cookiesMap
 * @return
 */
public static Response getCourseId(Map<String, String> cookiesMap) {

    try {
        Connection con = Jsoup.connect(Constant.CoursePraise.COURSE_URL);
        con.ignoreContentType(true);
        Iterator<Map.Entry<String, String>> it = cookiesMap.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, String> en = it.next();
            con = con.cookie(en.getKey(), en.getValue());
        }

        return con.method(Connection.Method.GET)
                .data(Constant.CoursePraise.COURSE_URL_OTHER_PARAM, "zTreeAsyncTest")
                .data(Constant.CoursePraise.COURSE_URL_, "1507812989512")
                .timeout(10000)
                .execute();

    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 
Example 4
Project: EducationalAdministrationSystem   File: HttpUtils.java   View Source Code Vote up 6 votes
public static Response getCourseInfo(Map<String, String> cookiesMap, String id, String name) {

        try {
            Connection con = Jsoup.connect(Constant.CoursePraise.COURSE_URL);
            con.ignoreContentType(true);
            Iterator<Map.Entry<String, String>> it = cookiesMap.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<String, String> en = it.next();
                con = con.cookie(en.getKey(), en.getValue());
            }

            return con.method(Connection.Method.GET)
                    .data("id", id)
                    .data("name", name)
                    .data("pId", "")
                    .data("level", "0")
                    .data(Constant.CoursePraise.COURSE_URL_OTHER_PARAM, "zTreeAsyncTest")
                    .data(Constant.CoursePraise.COURSE_URL_, "1507812989512")
                    .timeout(10000)
                    .execute();

        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
 
Example 5
Project: HouseSearch   File: HouseController.java   View Source Code Vote up 6 votes
/**
 * 获取总页数,返回给前台
 * 参数
 *
 * @param cityCode 城市
 * @param minPrice 最低价格
 * @param maxPrice 最高价格
 * @return
 */
@ResponseBody
@RequestMapping(value = "/GetTotalPages", method = RequestMethod.POST)
public int GetTotalPages(String cityCode, int minPrice, int maxPrice, String area, String subway) {
    //构建URL
    String oldUrl = "http://" + cityCode + ".58.com";
    Connection conn = Jsoup.connect(oldUrl);
    int pages = 0;
    try {
        Response response = conn.method(Method.GET).execute();
        newUrl = response.url().toString() + "/pinpaigongyu/pn/";
        String nowUrl = newUrl + "1/?minprice=" + minPrice + "_" + maxPrice + area + subway;
        Document doc = Jsoup.connect(nowUrl).get();
        int listsum = Integer.valueOf(doc.getElementsByClass("listsum").select("em").text());
        pages = listsum % 20 == 0 ? listsum / 20 : listsum / 20 + 1;  //计算页数
    } catch (IOException ex) {

    }
    return pages;
}
 
Example 6
Project: ripme   File: Http.java   View Source Code Vote up 5 votes
private void defaultSettings() {
    this.retries = Utils.getConfigInteger("download.retries", 1);
    connection = Jsoup.connect(this.url);
    connection.userAgent(AbstractRipper.USER_AGENT);
    connection.method(Method.GET);
    connection.timeout(TIMEOUT);
    connection.maxBodySize(0);
}
 
Example 7
Project: CrawlerSYS   File: WebCrawler.java   View Source Code Vote up 5 votes
public static Connection getConnect(String url, Map<String,String> header, Map<String,String> cookie){
		Connection con;
//		Document doc = null;
		url = url.trim();
		con = Jsoup.connect(url);
		con.ignoreContentType(true);
//		if(!url.startsWith("http:/")){
//			return null;
//		}
		if(url.split(":", 2)[0].equals("https")){
			con.validateTLSCertificates(false);
		}
		if(!(header==null||header.isEmpty())){
			for(Object key : header.keySet()){
				con.header((String) key, header.get(key));
			}
		}else{
			con.header("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
		}
		if(!(cookie==null||cookie.isEmpty()))
			con.cookies(cookie);
//		try {
//			doc = con.get();
//		} catch (IOException e) {
//			// TODO Auto-generated catch block
//		e.printStackTrace();logger.error("Exception",e);
//		}
		con.ignoreHttpErrors(true).ignoreContentType(true);
		return con;
	}
 
Example 8
Project: guanggoo-android   File: BaseTask.java   View Source Code Vote up 5 votes
protected Connection getConnection(String url) {
    Connection connection = Jsoup.connect(url);

    Map<String, String> cookies = getCookies();
    if (cookies.size() > 0) {
        connection.cookies(cookies);
    }

    return connection;
}
 
Example 9
Project: xxl-crawler   File: JsoupUtil.java   View Source Code Vote up 5 votes
/**
 * 加载页面
 *
 * @param pageLoadInfo
 *
 * @return
 */
public static Document load(PageLoadInfo pageLoadInfo) {
    if (!UrlUtil.isUrl(pageLoadInfo.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageLoadInfo.getUrl());
        if (pageLoadInfo.getParamMap() != null && !pageLoadInfo.getParamMap().isEmpty()) {
            conn.data(pageLoadInfo.getParamMap());
        }
        if (pageLoadInfo.getCookieMap() != null && !pageLoadInfo.getCookieMap().isEmpty()) {
            conn.cookies(pageLoadInfo.getCookieMap());
        }
        if (pageLoadInfo.getHeaderMap()!=null && !pageLoadInfo.getHeaderMap().isEmpty()) {
            conn.headers(pageLoadInfo.getHeaderMap());
        }
        if (pageLoadInfo.getUserAgent()!=null) {
            conn.userAgent(pageLoadInfo.getUserAgent());
        }
        if (pageLoadInfo.getReferrer() != null) {
            conn.referrer(pageLoadInfo.getReferrer());
        }
        conn.timeout(pageLoadInfo.getTimeoutMillis());

        // 代理
        if (pageLoadInfo.getProxy() != null) {
            conn.proxy(pageLoadInfo.getProxy());
        }

        // 发出请求
        Document html = null;
        if (pageLoadInfo.getIfPost()) {
            html = conn.post();
        } else {
            html = conn.get();
        }
        return html;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 
Example 10
Project: CoolApk-Console   File: JsoupUtil.java   View Source Code Vote up 5 votes
public static Document getDocument(String url, boolean loginCoolApk) throws IOException {
    if (!url.startsWith("https://") || !url.startsWith("http://"))
        url = "http://" + url;
    Connection connection = Jsoup.connect(url);
    if (loginCoolApk) {
        connection.cookies(new UserSave().buildWebRequestCookie());
    }
    return connection.get();
}
 
Example 11
Project: DWSurvey   File: JspToHtml.java   View Source Code Vote up 5 votes
public void postJspToHtml(String postUrl, String filePath,String fileName) throws Exception{
	HttpServletRequest request=Struts2Utils.getRequest();
	//${pageContext.request.scheme}://${pageContext.request.serverName }:${pageContext.request.serverPort} pageContext.request.contextPath
	String reqTarget = request.getScheme()+"://"+request.getServerName()+(request.getServerPort()==80?"":":"+request.getServerPort())+request.getContextPath();
	reqTarget =reqTarget+"/toHtml";
	//?url="+postUrl+"&filePath="+filePath+"&fileName="+fileName;
	Map<String, String> map=new HashMap<String, String>();
	map.put("url", postUrl);
	map.put("filePath", filePath);
	map.put("fileName", fileName);
	Connection connection = Jsoup.connect(reqTarget);
	connection.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31");
	connection.data(map);
	Document doc=connection.timeout(8000).get();
}
 
Example 12
Project: 9AnimeAndroid   File: NineAnimeApi.java   View Source Code Vote up 4 votes
private NineAnimeApi() throws IOException {
    frontpageFilter = ContentFilter.RecentlyUpdated;
    Connection con = Jsoup.connect(HOMEPAGE_URL);
    homepage = con.execute().parse();
    Log.d(TAG, "9Anime homepage parsed");
}
 
Example 13
Project: crawler-jsoup-maven   File: GITHUBLoginApater.java   View Source Code Vote up 4 votes
/**
 * @param userName 用户名
 * @param pwd 密码
 * @throws Exception
 */
public static void simulateLogin(String userName, String pwd) throws Exception {

    /* 
     * 第一次请求 
     * grab login form page first
     * 获取登陆提交的表单信息,及修改其提交data数据(login,password)
     */
    // get the response, which we will post to the action URL(rs.cookies())
    Connection con = Jsoup.connect(LOGIN_URL);  // 获取connection
    con.header(USER_AGENT, USER_AGENT_VALUE);   // 配置模拟浏览器
    Response rs = con.execute();                // 获取响应
    Document d1 = Jsoup.parse(rs.body());       // 转换为Dom树
    List<Element> eleList = d1.select("form");  // 获取提交form表单,可以通过查看页面源码代码得知

    // 获取cooking和表单属性
    // lets make data map containing all the parameters and its values found in the form
    Map<String, String> datas = new HashMap<>();
    for (Element e : eleList.get(0).getAllElements()) {
        // 设置用户名
        if (e.attr("name").equals("login")) {
            e.attr("value", userName);
        }
        // 设置用户密码
        if (e.attr("name").equals("password")) {
            e.attr("value", pwd);
        }
        // 排除空值表单属性
        if (e.attr("name").length() > 0) {
            datas.put(e.attr("name"), e.attr("value"));
        }
    }

    /*
     * 第二次请求,以post方式提交表单数据以及cookie信息
     */
    Connection con2 = Jsoup.connect("https://github.com/session");
    con2.header(USER_AGENT, USER_AGENT_VALUE);
    // 设置cookie和post上面的map数据
    Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
    // 打印,登陆成功后的信息
    System.out.println(login.body());

    // 登陆成功后的cookie信息,可以保存到本地,以后登陆时,只需一次登陆即可
    Map<String, String> map = login.cookies();
    for (String s : map.keySet()) {
        System.out.println(s + " : " + map.get(s));
    }
}
 
Example 14
Project: crawler-jsoup-maven   File: Jsoup403ForbiddenExample.java   View Source Code Vote up 4 votes
public static void main(String[] args) {
    
    try{
        
        // connect to the website         '1
        Connection connection = Jsoup.connect("http://www.bluetata.com");
        
        // get the HTML document          '2
        Document doc = connection.get();
        
        // parse text from HTML           '3
        String strHTML = doc.text();
        
        // out put dom                    '4
        System.out.println(strHTML);
        
    }catch(IOException ioex){
        ioex.printStackTrace();
    }
 
}
 
Example 15
Project: WebtoonDownloadManager   File: CommonService.java   View Source Code Vote up 2 votes
/**
 * 네이버 웹툰을 다운로드 받기 위해 연결함.
 * 
 * @param webCode
 *            - 웹툰코드
 * @param i
 *            - 편수
 * @return
 */
public Connection getConnection(String webCode, int i) {
	return Jsoup.connect("http://comic.naver.com/webtoon/detail.nhn?titleId=" + webCode + "&no=" + i);
}
 
Example 16
Project: WebtoonDownloadManager   File: CommonService.java   View Source Code Vote up 2 votes
/**
 * 네이버 웹툰 연결
 * 
 * @param webCode
 * @return
 */
public Connection getConnection(String webCode) {
	return Jsoup.connect("http://comic.naver.com/webtoon/detail.nhn?titleId=" + webCode);
}