Java Code Examples for org.jsoup.Jsoup#connect()

public static String get(String url, Map<String, String> headers,
		Map<String, String> params) {
	try {
		Connection connect = Jsoup.connect(url);
		connect.ignoreContentType(true);
		connect.ignoreHttpErrors(true);
		if (params != null) {
			connect.data(params);
		}
		if (headers != null) {
			connect.headers(headers);
		}
		return connect.execute().body();
	} catch (Throwable e) {
	}
	return null;
}
 

/**
 * @param request the request
 */
private Response uploadFile(HttpRequest request) throws IOException {
    Connection conn = Jsoup.connect(request.getUrl());
    conn.method(Method.POST)
            .timeout(request.getTimeout())
            .ignoreHttpErrors(true)
            // unlimited size
            .maxBodySize(0)
            .ignoreContentType(true);
    addHeadersAndCookies(request, conn);
    Object body = request.getBody();
    // handle MultiPart
    if (body instanceof MultiPart) {
        return handleMultiPart(conn, (MultiPart) body);
    } else {
        return handleMultiPart(conn, convertInputStreamAndFile(request));
    }
}
 

@Override
protected Object doInBackground(Object[] params) {
    Connection connection  = Jsoup.connect(URL_WORDPRESS + "/wp-comments-post.php");
    connection.method(Connection.Method.POST);
    connection.data("author", name);
    connection.data("email", email);
    connection.data("url", url);
    connection.data("comment", comment);
    connection.data("comment_post_ID", id);
    connection.data("comment_parent", commentParent);
    try {
        response = connection.execute().statusCode();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

@Test
public void followsNewTempRedirect() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/307.pl"); // http://jsoup.org
    Document doc = con.get();
    assertTrue(doc.title().contains("jsoup"));
    assertEquals("https://jsoup.org/", con.response().url().toString());
}
 

protected Document getDocument(HttpRequest httpRequest){
    Document doc = null;
    Connection connection = Jsoup.connect(httpRequest.getUrl());
    if (httpRequest.getHeader() != null) {
        connection.headers(getBasicHeaders());
        connection.headers(httpRequest.getHeader());
    }
    try {
        doc = connection.timeout(10 * 1000).get();
    } catch (IOException e) {
        log.error("Fail to connect!", e);
    }
    return doc;
}
 

public Document getWebpage(String url) throws Exception {
    Connection conn;
    try {
        conn = Jsoup.connect(url);
    } catch (Exception e1) {
        log("Couldn't connect with the website.");
        return null;
    }
    return conn
            //.header("HTTP-User-Agent", "")
            .get();
}
 

@Test
public void followsRelativeDotRedirect() throws IOException {
    // redirects to "./ok.html", should resolve to http://direct.infohound.net/tools/ok.html
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-rel-dot.pl"); // to ./ok.html
    Document doc = con.post();
    assertTrue(doc.title().contains("OK"));
    assertEquals(doc.location(), "http://direct.infohound.net/tools/ok.html");
}
 

private void defaultSettings() {
    this.retries = Utils.getConfigInteger("download.retries", 1);
    connection = Jsoup.connect(this.url);
    connection.userAgent(AbstractRipper.USER_AGENT);
    connection.method(Method.GET);
    connection.timeout(TIMEOUT);
    connection.maxBodySize(0);

    // Extract cookies from config entry:
    // Example config entry:
    // cookies.reddit.com = reddit_session=<value>; other_cookie=<value>
    connection.cookies(cookiesForURL(this.url));
}
 

@Test
public void multiCookieSet() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-cookie.pl");
    Connection.Response res = con.execute();

    // test cookies set by redirect:
    Map<String, String> cookies = res.cookies();
    assertEquals("asdfg123", cookies.get("token"));
    assertEquals("jhy", cookies.get("uid"));

    // send those cookies into the echo URL by map:
    Document doc = Jsoup.connect(echoURL).cookies(cookies).get();
    assertEquals("token=asdfg123; uid=jhy", ihVal("HTTP_COOKIE", doc));
}
 

@Test
public void followsRedirectToHttps() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-secure.pl"); // https://www.google.com
    con.data("id", "5");
    Document doc = con.get();
    assertTrue(doc.title().contains("Google"));
}
 

@Override
protected Document loadPage(URL url) throws IOException {
	if (connection == null){
		connection = Jsoup.connect(url.toString());
	} else {
		connection.url(url);
	}

	return connection.get();
}
 

@Test
public void combinesSameHeadersWithComma() throws IOException {
    // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2
    String url = "http://direct.infohound.net/tools/q.pl";
    Connection con = Jsoup.connect(url);
    con.get();

    assertEquals("text/html", con.response().header("Content-Type"));
    assertEquals("no-cache, no-store", con.response().header("Cache-Control"));
}
 

@Test
public void followsRedirectToHttps() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-secure.pl"); // https://www.google.com
    con.data("id", "5");
    Document doc = con.get();
    assertTrue(doc.title().contains("Google"));
}
 

@Test
public void followsRedirectsWithWithespaces() throws IOException {
    Connection con = Jsoup.connect("http://tinyurl.com/kgofxl8"); // to http://www.google.com/?q=white spaces
    Document doc = con.get();
    assertTrue(doc.title().contains("Google"));
}
 

@Test
public void fetchURIWithWihtespace() throws IOException {
    Connection con = Jsoup.connect("http://try.jsoup.org/#with whitespaces");
    Document doc = con.get();
    assertTrue(doc.title().contains("jsoup"));
}
 

public static void main(String[] args) {
    
    try{
        
        // connect to the website         '1
        Connection connection = Jsoup.connect("http://www.bluetata.com");
        
        // get the HTML document          '2
        Document doc = connection.get();
        
        // parse text from HTML           '3
        String strHTML = doc.text();
        
        // out put dom                    '4
        System.out.println(strHTML);
        
    }catch(IOException ioex){
        ioex.printStackTrace();
    }
 
}
 

@Test
public void followsRelativeRedirect() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-rel.pl"); // to /tidy/
    Document doc = con.post();
    assertTrue(doc.title().contains("HTML Tidy Online"));
}
 

/**
 * 加载页面
 *
 * @param pageRequest
 *
 * @return Document
 */
public static Document load(PageRequest pageRequest) {
    if (!UrlUtil.isUrl(pageRequest.getUrl())) {
        return null;
    }
    try {
        // 请求设置
        Connection conn = Jsoup.connect(pageRequest.getUrl());
        if (pageRequest.getParamMap() != null && !pageRequest.getParamMap().isEmpty()) {
            conn.data(pageRequest.getParamMap());
        }
        if (pageRequest.getCookieMap() != null && !pageRequest.getCookieMap().isEmpty()) {
            conn.cookies(pageRequest.getCookieMap());
        }
        if (pageRequest.getHeaderMap()!=null && !pageRequest.getHeaderMap().isEmpty()) {
            conn.headers(pageRequest.getHeaderMap());
        }
        if (pageRequest.getUserAgent()!=null) {
            conn.userAgent(pageRequest.getUserAgent());
        }
        if (pageRequest.getReferrer() != null) {
            conn.referrer(pageRequest.getReferrer());
        }
        conn.timeout(pageRequest.getTimeoutMillis());
        conn.validateTLSCertificates(pageRequest.isValidateTLSCertificates());
        conn.maxBodySize(0);    // 取消默认1M限制

        // 代理
        if (pageRequest.getProxy() != null) {
            conn.proxy(pageRequest.getProxy());
        }

        // 发出请求
        Document html = null;
        if (pageRequest.isIfPost()) {
            html = conn.post();
        } else {
            html = conn.get();
        }
        return html;
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
        return null;
    }
}
 

@Test
public void gracefullyHandleBrokenLocationRedirect() throws IOException {
    Connection con = Jsoup.connect("http://aag-ye.com"); // has Location: http:/temp/AAG_New/en/index.php
    con.get(); // would throw exception on error
    assertTrue(true);
}
 

@Test
public void followsRelativeRedirect() throws IOException {
    Connection con = Jsoup.connect("http://direct.infohound.net/tools/302-rel.pl"); // to /tidy/
    Document doc = con.post();
    assertTrue(doc.title().contains("HTML Tidy Online"));
}