Java Code Examples for org.jsoup.Jsoup.connect()

public Document postPageByUrl(String url, String[][] params) throws IOException {
    Connection connection = Jsoup.connect(url);

    for (String[] data : params) {
        connection.data(data[0], data[1]);
    }

    Connection.Response response = connection.cookies(getCookies())
            .followRedirects(true)
            .method(Connection.Method.POST)
            .execute();

    this.cookies.addItems(response.cookies());

    return response.parse();
}
 

/**
 * 获取学校通知接口
 *
 * @param cookiesMap
 * @param index
 * @return
 */
public static Response getSchoolNotice(Map<String, String> cookiesMap, int index) {

    try {
        Connection con = Jsoup.connect(Constant.SchoolNotice.URL);
        con.ignoreContentType(true);
        Iterator<Map.Entry<String, String>> it = cookiesMap.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, String> en = it.next();
            con = con.cookie(en.getKey(), en.getValue());
        }

        return con.method(Connection.Method.POST)
                .data(Constant.SchoolNotice.PARAM_PAGE_SIZE, "10")
                .data(Constant.SchoolNotice.PARAM_PAGE_INDEX, String.valueOf(index))
                .data(Constant.SchoolNotice.PARAM_A_TITLE, "")
                .data(Constant.SchoolNotice.PARAM_ORDER_BY_TYPE, "asc")
                .timeout(10000)
                .execute();

    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

/**
 * 获取课程id
 *
 * @param cookiesMap
 * @return
 */
public static Response getCourseId(Map<String, String> cookiesMap) {

    try {
        Connection con = Jsoup.connect(Constant.CoursePraise.COURSE_URL);
        con.ignoreContentType(true);
        Iterator<Map.Entry<String, String>> it = cookiesMap.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<String, String> en = it.next();
            con = con.cookie(en.getKey(), en.getValue());
        }

        return con.method(Connection.Method.GET)
                .data(Constant.CoursePraise.COURSE_URL_OTHER_PARAM, "zTreeAsyncTest")
                .data(Constant.CoursePraise.COURSE_URL_, "1507812989512")
                .timeout(10000)
                .execute();

    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

public static Response getCourseInfo(Map<String, String> cookiesMap, String id, String name) {

        try {
            Connection con = Jsoup.connect(Constant.CoursePraise.COURSE_URL);
            con.ignoreContentType(true);
            Iterator<Map.Entry<String, String>> it = cookiesMap.entrySet().iterator();
            while (it.hasNext()) {
                Map.Entry<String, String> en = it.next();
                con = con.cookie(en.getKey(), en.getValue());
            }

            return con.method(Connection.Method.GET)
                    .data("id", id)
                    .data("name", name)
                    .data("pId", "")
                    .data("level", "0")
                    .data(Constant.CoursePraise.COURSE_URL_OTHER_PARAM, "zTreeAsyncTest")
                    .data(Constant.CoursePraise.COURSE_URL_, "1507812989512")
                    .timeout(10000)
                    .execute();

        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }
 

/**
 * 获取总页数，返回给前台
 * 参数
 *
 * @param cityCode 城市
 * @param minPrice 最低价格
 * @param maxPrice 最高价格
 * @return
 */
@ResponseBody
@RequestMapping(value = "/GetTotalPages", method = RequestMethod.POST)
public int GetTotalPages(String cityCode, int minPrice, int maxPrice, String area, String subway) {
    //构建URL
    String oldUrl = "http://" + cityCode + ".58.com";
    Connection conn = Jsoup.connect(oldUrl);
    int pages = 0;
    try {
        Response response = conn.method(Method.GET).execute();
        newUrl = response.url().toString() + "/pinpaigongyu/pn/";
        String nowUrl = newUrl + "1/?minprice=" + minPrice + "_" + maxPrice + area + subway;
        Document doc = Jsoup.connect(nowUrl).get();
        int listsum = Integer.valueOf(doc.getElementsByClass("listsum").select("em").text());
        pages = listsum % 20 == 0 ? listsum / 20 : listsum / 20 + 1;  //计算页数
    } catch (IOException ex) {

    }
    return pages;
}
 

private void defaultSettings() {
    this.retries = Utils.getConfigInteger("download.retries", 1);
    connection = Jsoup.connect(this.url);
    connection.userAgent(AbstractRipper.USER_AGENT);
    connection.method(Method.GET);
    connection.timeout(TIMEOUT);
    connection.maxBodySize(0);
}
 

public static Connection getConnect(String url, Map<String,String> header, Map<String,String> cookie){
		Connection con;
//		Document doc = null;
		url = url.trim();
		con = Jsoup.connect(url);
		con.ignoreContentType(true);
//		if(!url.startsWith("http:/")){
//			return null;
//		}
		if(url.split(":", 2)[0].equals("https")){
			con.validateTLSCertificates(false);
		}
		if(!(header==null||header.isEmpty())){
			for(Object key : header.keySet()){
				con.header((String) key, header.get(key));
			}
		}else{
			con.header("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
		}
		if(!(cookie==null||cookie.isEmpty()))
			con.cookies(cookie);
//		try {
//			doc = con.get();
//		} catch (IOException e) {
//			// TODO Auto-generated catch block
//		e.printStackTrace();logger.error("Exception",e);
//		}
		con.ignoreHttpErrors(true).ignoreContentType(true);
		return con;
	}
 

protected Connection getConnection(String url) {
    Connection connection = Jsoup.connect(url);

    Map<String, String> cookies = getCookies();
    if (cookies.size() > 0) {
        connection.cookies(cookies);
    }

    return connection;
}
 

/**
 * 加载页面
 *
 * @param pageLoadInfo
 *
 * @return
 */
public static Document load(PageLoadInfo pageLoadInfo) {
    if (!UrlUtil.isUrl(pageLoadInfo.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageLoadInfo.getUrl());
        if (pageLoadInfo.getParamMap() != null && !pageLoadInfo.getParamMap().isEmpty()) {
            conn.data(pageLoadInfo.getParamMap());
        }
        if (pageLoadInfo.getCookieMap() != null && !pageLoadInfo.getCookieMap().isEmpty()) {
            conn.cookies(pageLoadInfo.getCookieMap());
        }
        if (pageLoadInfo.getHeaderMap()!=null && !pageLoadInfo.getHeaderMap().isEmpty()) {
            conn.headers(pageLoadInfo.getHeaderMap());
        }
        if (pageLoadInfo.getUserAgent()!=null) {
            conn.userAgent(pageLoadInfo.getUserAgent());
        }
        if (pageLoadInfo.getReferrer() != null) {
            conn.referrer(pageLoadInfo.getReferrer());
        }
        conn.timeout(pageLoadInfo.getTimeoutMillis());

        // 代理
        if (pageLoadInfo.getProxy() != null) {
            conn.proxy(pageLoadInfo.getProxy());
        }

        // 发出请求
        Document html = null;
        if (pageLoadInfo.getIfPost()) {
            html = conn.post();
        } else {
            html = conn.get();
        }
        return html;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 

public static Document getDocument(String url, boolean loginCoolApk) throws IOException {
    if (!url.startsWith("https://") || !url.startsWith("http://"))
        url = "http://" + url;
    Connection connection = Jsoup.connect(url);
    if (loginCoolApk) {
        connection.cookies(new UserSave().buildWebRequestCookie());
    }
    return connection.get();
}
 

public void postJspToHtml(String postUrl, String filePath,String fileName) throws Exception{
	HttpServletRequest request=Struts2Utils.getRequest();
	//${pageContext.request.scheme}://${pageContext.request.serverName }:${pageContext.request.serverPort} pageContext.request.contextPath
	String reqTarget = request.getScheme()+"://"+request.getServerName()+(request.getServerPort()==80?"":":"+request.getServerPort())+request.getContextPath();
	reqTarget =reqTarget+"/toHtml";
	//?url="+postUrl+"&filePath="+filePath+"&fileName="+fileName;
	Map<String, String> map=new HashMap<String, String>();
	map.put("url", postUrl);
	map.put("filePath", filePath);
	map.put("fileName", fileName);
	Connection connection = Jsoup.connect(reqTarget);
	connection.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31");
	connection.data(map);
	Document doc=connection.timeout(8000).get();
}
 

private NineAnimeApi() throws IOException {
    frontpageFilter = ContentFilter.RecentlyUpdated;
    Connection con = Jsoup.connect(HOMEPAGE_URL);
    homepage = con.execute().parse();
    Log.d(TAG, "9Anime homepage parsed");
}
 

/**
 * @param userName 用户名
 * @param pwd 密码
 * @throws Exception
 */
public static void simulateLogin(String userName, String pwd) throws Exception {

    /* 
     * 第一次请求 
     * grab login form page first
     * 获取登陆提交的表单信息，及修改其提交data数据（login，password）
     */
    // get the response, which we will post to the action URL(rs.cookies())
    Connection con = Jsoup.connect(LOGIN_URL);  // 获取connection
    con.header(USER_AGENT, USER_AGENT_VALUE);   // 配置模拟浏览器
    Response rs = con.execute();                // 获取响应
    Document d1 = Jsoup.parse(rs.body());       // 转换为Dom树
    List<Element> eleList = d1.select("form");  // 获取提交form表单，可以通过查看页面源码代码得知

    // 获取cooking和表单属性
    // lets make data map containing all the parameters and its values found in the form
    Map<String, String> datas = new HashMap<>();
    for (Element e : eleList.get(0).getAllElements()) {
        // 设置用户名
        if (e.attr("name").equals("login")) {
            e.attr("value", userName);
        }
        // 设置用户密码
        if (e.attr("name").equals("password")) {
            e.attr("value", pwd);
        }
        // 排除空值表单属性
        if (e.attr("name").length() > 0) {
            datas.put(e.attr("name"), e.attr("value"));
        }
    }

    /*
     * 第二次请求，以post方式提交表单数据以及cookie信息
     */
    Connection con2 = Jsoup.connect("https://github.com/session");
    con2.header(USER_AGENT, USER_AGENT_VALUE);
    // 设置cookie和post上面的map数据
    Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
    // 打印，登陆成功后的信息
    System.out.println(login.body());

    // 登陆成功后的cookie信息，可以保存到本地，以后登陆时，只需一次登陆即可
    Map<String, String> map = login.cookies();
    for (String s : map.keySet()) {
        System.out.println(s + " : " + map.get(s));
    }
}
 

public static void main(String[] args) {
    
    try{
        
        // connect to the website         '1
        Connection connection = Jsoup.connect("http://www.bluetata.com");
        
        // get the HTML document          '2
        Document doc = connection.get();
        
        // parse text from HTML           '3
        String strHTML = doc.text();
        
        // out put dom                    '4
        System.out.println(strHTML);
        
    }catch(IOException ioex){
        ioex.printStackTrace();
    }
 
}
 

/**
 * 네이버 웹툰을 다운로드 받기 위해 연결함.
 * 
 * @param webCode
 *            - 웹툰코드
 * @param i
 *            - 편수
 * @return
 */
public Connection getConnection(String webCode, int i) {
	return Jsoup.connect("http://comic.naver.com/webtoon/detail.nhn?titleId=" + webCode + "&no=" + i);
}
 

/**
 * 네이버 웹툰 연결
 * 
 * @param webCode
 * @return
 */
public Connection getConnection(String webCode) {
	return Jsoup.connect("http://comic.naver.com/webtoon/detail.nhn?titleId=" + webCode);
}