Java Code Examples for org.jsoup.Connection.Response#body()

The following examples show how to use org.jsoup.Connection.Response#body() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ArtStationRipper.java    From ripme with MIT License 6 votes vote down vote up
private JSONObject getJson(URL url) throws IOException {
        Connection con = Http.url(url).method(Method.GET).connection();
        con.ignoreHttpErrors(true);
        con.ignoreContentType(true);
        con.userAgent(
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
        con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        con.header("Accept-Language", "en-US,en;q=0.5");
//        con.header("Accept-Encoding", "gzip, deflate, br");
        con.header("Upgrade-Insecure-Requests", "1");
        Response res = con.execute();
        int status = res.statusCode();
        if (status / 100 == 2) {
            String jsonString = res.body();
            return new JSONObject(jsonString);
        }
        throw new IOException("Error fetching json. Status code:" + status);
    }
 
Example 2
Source File: VkRipper.java    From ripme with MIT License 5 votes vote down vote up
private Map<String,String> getPhotoIDsToURLs(String photoID) throws IOException {
    Map<String,String> photoIDsToURLs = new HashMap<>();
    Map<String,String> postData = new HashMap<>();
    // act=show&al=1&list=album45506334_172415053&module=photos&photo=45506334_304658196
    postData.put("list", getGID(this.url));
    postData.put("act", "show");
    postData.put("al", "1");
    postData.put("module", "photos");
    postData.put("photo", photoID);
    Response res = Jsoup.connect("https://vk.com/al_photos.php")
            .header("Referer", this.url.toExternalForm())
            .header("Accept", "*/*")
            .header("Accept-Language", "en-US,en;q=0.5")
            .header("Content-Type", "application/x-www-form-urlencoded")
            .header("X-Requested-With", "XMLHttpRequest")
            .ignoreContentType(true)
            .userAgent(USER_AGENT)
            .timeout(5000)
            .data(postData)
            .method(Method.POST)
            .execute();
    String jsonString = res.body();
    JSONObject json = new JSONObject(jsonString);
    JSONObject photoObject = findJSONObjectContainingPhotoId(photoID, json);
    String bestSourceUrl = getBestSourceUrl(photoObject);

    if (bestSourceUrl != null) {
        photoIDsToURLs.put(photoID, bestSourceUrl);
    } else {
        LOGGER.error("Could not find image source for " + photoID);
    }

    return photoIDsToURLs;
}
 
Example 3
Source File: FuskatorRipper.java    From ripme with MIT License 5 votes vote down vote up
private void getXAuthToken() throws IOException {
    if (cookies == null || cookies.isEmpty()) {
        throw new IOException("Null cookies or no cookies found.");
    }
    Response res = Http.url(xAuthUrl).cookies(cookies).method(Method.POST).response();
    xAuthToken = res.body();
}
 
Example 4
Source File: ThechiveRipper.java    From ripme with MIT License 5 votes vote down vote up
@Override
public Document getNextPage(Document doc) throws IOException {
    Matcher matcher = p1.matcher(url.toExternalForm());

    if (matcher.matches()) {
        // url type thechive.com/YEAR/MONTH/DAY/POSTTITLE/ has a single page.
        return null;
    } else {
        if (nextSeed == null) {
            throw new IOException("No more pages.");
        }
    }

    // Following try block checks if the next JSON object has images or not.
    // This is done to avoid IOException in rip() method, caused when
    // getURLsFromPage() returns empty list.
    JSONArray imgList;
    try {
        Response response = Http.url(jsonUrl).data("seed", nextSeed).data("queryType", "by-username")
                .data("username", username).ignoreContentType().cookies(cookies).response();
        cookies = response.cookies();
        JSONObject json = new JSONObject(response.body());
        imgList = json.getJSONArray("uploads");
    } catch (Exception e) {
        throw new IOException("Error fetching next page.", e);
    }

    if (imgList != null && imgList.length() > 0) {
        // Pass empty document as it is of no use for thechive.com/userName url type.
        return new Document(url.toString());
    } else {
        // Return null as this is last page.
        return null;
    }
}
 
Example 5
Source File: CommentTest.java    From emotional_analysis with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {
//		for(int i = 138789;i < 139989;i++){
//			try{
//				System.setProperty("http.maxRedirects", "5000");
//				System.getProperties().setProperty("proxySet", "true");
//				// 如果不设置,只要代理IP和代理端口正确,此项不设置也可以
//				System.getProperties().setProperty("http.proxyHost", "124.79.56.115");
//				System.getProperties().setProperty("http.proxyPort", "8118");
//				String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);//limit
//				String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"offset\":0,\"limit\":10};","0CoJUm6Qyw8W8jud"), secKey);
//		        String encSecKey = EncryptUtils.rsaEncrypt(secKey);
//			    Response execute = Jsoup
//			    		.connect("https://music.163.com/weapi/v1/resource/comments/R_SO_4_"+i)
//			    		.header("User-Agent",
//								"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.32 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
//						.header("Cache-Control", "max-age=60").header("Accept", "*/*")
//						.header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
//						.header("Referer", "http://music.163.com/song?id=2332323")
//						.header("Origin", "http://music.163.com").header("Host", "music.163.com")
//						.header("Content-Type", "application/x-www-form-urlencoded")
//								.data("params",encText)
//						.data("encSecKey",encSecKey)
//						.method(Method.POST).ignoreContentType(true).timeout(2000000000)
//						.execute();
//				String string = execute.body().toString();
//				System.out.println(string);
////				IpProxy.timerGetIps();
////				String url = "http://music.163.com/weapi/v1/resource/comments/R_SO_4_" + i;
////				String data = CenterUrl.getDataByUrl(url, "{\"offset\":0,\"limit\":10};");
////				System.out.println(data);
//			}catch(Exception e){
//				e.printStackTrace();
//			}
//		}
		
		try {
			Response execute = Jsoup.connect("https://api.weibo.cn/2/cardlist?networktype=wifi&uicode=10000198"
					+ "&moduleID=708&featurecode=10000085"
					+ "&wb_version=3523"
					+ "&lcardid=1003030212_seqid%3A1671487208%7Ctype%3A3%7Ct%3A0%7Cpos%3A1-0-0%7Cq%3A%E6%9D%A8%E5%B9%82%7Cext%3A%26uid%3D1195242865%26&c=android&i=a032269&s=7f0ba7d8&ft=0&ua=Honor-CHM-TL00H__weibo__7.12.1__android__android4.4.4&wm=5091_0008&aid=01AsfiMEjVzg0YSShsV_ISlRgLOKx7UXK8bxQfU3UzNxYoTHo.&fid=1076031195242865_-_WEIBO_SECOND_PROFILE_WEIBO&uid=6432582011&v_f=2&v_p=55&from=107C195010&gsid=_2A253Pc5MDeRxGeBK6FAU-CzMyj2IHXVSa0aErDV6PUJbkdAKLUXikWpNHetkT4FG1cwd-aJIY8mFs-sLNfa9Yoqb&imsi=310260361179574&lang=zh_CN&lfid=100303type%3D3%26q%3D%E6%9D%A8%E5%B9%82%26t%3D0&page=1&skin=default&count=20&oldwm=5091_0008&sflag=1&containerid=1076031195242865_-_WEIBO_SECOND_PROFILE_WEIBO&ignore_inturrpted_error=true&luicode=10000003&need_head_cards=0")
		    		.header("User-Agent",
							"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.32 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
					.header("Cache-Control", "max-age=60").header("Accept", "*/*")
					.header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
					.header("Referer", "http://music.163.com/song?id=2332323")
					.header("Origin", "http://music.163.com").header("Host", "music.163.com")
					.header("Content-Type", "application/x-www-form-urlencoded")
					.method(Method.POST).ignoreContentType(true).timeout(2000000000)
					.execute();
			String body = execute.body();
			System.out.println(body);
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		
	}
 
Example 6
Source File: PostSend2.java    From crawler-jsoup-maven with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
        
        String url = "http://jzsc.mohurd.gov.cn/dataservice/query/comp/list";
        Response res = Jsoup.connect(url).execute();
        Map<String, String> cookies = res.cookies();
        System.out.println(cookies);
        
        Map<String, String> datas = new HashMap<>();
        datas.put("$total", "297267");
        datas.put("$reload", "1");
        datas.put("$pg", "31&&1=1");
        datas.put("$pgsz", "15");
        
        Response res1 = Jsoup.connect(url).cookies(cookies).data(datas).method(Method.POST).execute();
        
        String html = res1.body();
        System.out.println(html);
        
//        String jsonBody = "{pg:30,ps:15,tt:297267,pn:5,pc:19818,id:'',st:true}"; 
//        
//        Connection connection = Jsoup.connect("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .userAgent("Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36") // User-Agent of Chrome 55  
//        .referrer("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .header("Content-Type", "application/x-www-form-urlencoded")  
//        .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")  
//        .header("Accept-Encoding", "gzip, deflate")  
//        .header("Accept-Language", "zh-CN,en-US;q=0.8,zh;q=0.6,ja;q=0.4,en;q=0.2")  
//        .header("Cookie", "filter_comp=; JSESSIONID=F53A68FBA45CCF750F9F32ACF090DC9B")  
//        .header("Host", "jzsc.mohurd.gov.cn")  
//        .header("Connection", "keep-alive")  
//        .header("X-Requested-With", "XMLHttpRequest")  
//        .header("Upgrade-Insecure-Requests", "1")  
//        .requestBody(jsonBody)  
//        .maxBodySize(100)  
//        .timeout(1000 * 10)  
//        .method(Connection.Method.POST);  
//  
//        Response response = connection.execute(); 
//        
//        String body = response.body();
//        System.out.println(body);

    }