org.jsoup.Connection.Response#body

Source File: ArtStationRipper.java From ripme with MIT License

6 votes

private JSONObject getJson(URL url) throws IOException {
        Connection con = Http.url(url).method(Method.GET).connection();
        con.ignoreHttpErrors(true);
        con.ignoreContentType(true);
        con.userAgent(
                "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
        con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        con.header("Accept-Language", "en-US,en;q=0.5");
//        con.header("Accept-Encoding", "gzip, deflate, br");
        con.header("Upgrade-Insecure-Requests", "1");
        Response res = con.execute();
        int status = res.statusCode();
        if (status / 100 == 2) {
            String jsonString = res.body();
            return new JSONObject(jsonString);
        }
        throw new IOException("Error fetching json. Status code:" + status);
    }

Source File: VkRipper.java From ripme with MIT License

5 votes

private Map<String,String> getPhotoIDsToURLs(String photoID) throws IOException {
    Map<String,String> photoIDsToURLs = new HashMap<>();
    Map<String,String> postData = new HashMap<>();
    // act=show&al=1&list=album45506334_172415053&module=photos&photo=45506334_304658196
    postData.put("list", getGID(this.url));
    postData.put("act", "show");
    postData.put("al", "1");
    postData.put("module", "photos");
    postData.put("photo", photoID);
    Response res = Jsoup.connect("https://vk.com/al_photos.php")
            .header("Referer", this.url.toExternalForm())
            .header("Accept", "*/*")
            .header("Accept-Language", "en-US,en;q=0.5")
            .header("Content-Type", "application/x-www-form-urlencoded")
            .header("X-Requested-With", "XMLHttpRequest")
            .ignoreContentType(true)
            .userAgent(USER_AGENT)
            .timeout(5000)
            .data(postData)
            .method(Method.POST)
            .execute();
    String jsonString = res.body();
    JSONObject json = new JSONObject(jsonString);
    JSONObject photoObject = findJSONObjectContainingPhotoId(photoID, json);
    String bestSourceUrl = getBestSourceUrl(photoObject);

    if (bestSourceUrl != null) {
        photoIDsToURLs.put(photoID, bestSourceUrl);
    } else {
        LOGGER.error("Could not find image source for " + photoID);
    }

    return photoIDsToURLs;
}

Source File: FuskatorRipper.java From ripme with MIT License

5 votes

private void getXAuthToken() throws IOException {
    if (cookies == null || cookies.isEmpty()) {
        throw new IOException("Null cookies or no cookies found.");
    }
    Response res = Http.url(xAuthUrl).cookies(cookies).method(Method.POST).response();
    xAuthToken = res.body();
}

Source File: ThechiveRipper.java From ripme with MIT License

5 votes

@Override
public Document getNextPage(Document doc) throws IOException {
    Matcher matcher = p1.matcher(url.toExternalForm());

    if (matcher.matches()) {
        // url type thechive.com/YEAR/MONTH/DAY/POSTTITLE/ has a single page.
        return null;
    } else {
        if (nextSeed == null) {
            throw new IOException("No more pages.");
        }
    }

    // Following try block checks if the next JSON object has images or not.
    // This is done to avoid IOException in rip() method, caused when
    // getURLsFromPage() returns empty list.
    JSONArray imgList;
    try {
        Response response = Http.url(jsonUrl).data("seed", nextSeed).data("queryType", "by-username")
                .data("username", username).ignoreContentType().cookies(cookies).response();
        cookies = response.cookies();
        JSONObject json = new JSONObject(response.body());
        imgList = json.getJSONArray("uploads");
    } catch (Exception e) {
        throw new IOException("Error fetching next page.", e);
    }

    if (imgList != null && imgList.length() > 0) {
        // Pass empty document as it is of no use for thechive.com/userName url type.
        return new Document(url.toString());
    } else {
        // Return null as this is last page.
        return null;
    }
}

Source File: CommentTest.java From emotional_analysis with Apache License 2.0

4 votes

public static void main(String[] args) {
//		for(int i = 138789;i < 139989;i++){
//			try{
//				System.setProperty("http.maxRedirects", "5000");
//				System.getProperties().setProperty("proxySet", "true");
//				// 如果不设置，只要代理IP和代理端口正确,此项不设置也可以
//				System.getProperties().setProperty("http.proxyHost", "124.79.56.115");
//				System.getProperties().setProperty("http.proxyPort", "8118");
//				String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);//limit
//				String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"offset\":0,\"limit\":10};","0CoJUm6Qyw8W8jud"), secKey);
//		        String encSecKey = EncryptUtils.rsaEncrypt(secKey);
//			    Response execute = Jsoup
//			    		.connect("https://music.163.com/weapi/v1/resource/comments/R_SO_4_"+i)
//			    		.header("User-Agent",
//								"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.32 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
//						.header("Cache-Control", "max-age=60").header("Accept", "*/*")
//						.header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
//						.header("Referer", "http://music.163.com/song?id=2332323")
//						.header("Origin", "http://music.163.com").header("Host", "music.163.com")
//						.header("Content-Type", "application/x-www-form-urlencoded")
//								.data("params",encText)
//						.data("encSecKey",encSecKey)
//						.method(Method.POST).ignoreContentType(true).timeout(2000000000)
//						.execute();
//				String string = execute.body().toString();
//				System.out.println(string);
////				IpProxy.timerGetIps();
////				String url = "http://music.163.com/weapi/v1/resource/comments/R_SO_4_" + i;
////				String data = CenterUrl.getDataByUrl(url, "{\"offset\":0,\"limit\":10};");
////				System.out.println(data);
//			}catch(Exception e){
//				e.printStackTrace();
//			}
//		}
		
		try {
			Response execute = Jsoup.connect("https://api.weibo.cn/2/cardlist?networktype=wifi&uicode=10000198"
					+ "&moduleID=708&featurecode=10000085"
					+ "&wb_version=3523"
					+ "&lcardid=1003030212_seqid%3A1671487208%7Ctype%3A3%7Ct%3A0%7Cpos%3A1-0-0%7Cq%3A%E6%9D%A8%E5%B9%82%7Cext%3A%26uid%3D1195242865%26&c=android&i=a032269&s=7f0ba7d8&ft=0&ua=Honor-CHM-TL00H__weibo__7.12.1__android__android4.4.4&wm=5091_0008&aid=01AsfiMEjVzg0YSShsV_ISlRgLOKx7UXK8bxQfU3UzNxYoTHo.&fid=1076031195242865_-_WEIBO_SECOND_PROFILE_WEIBO&uid=6432582011&v_f=2&v_p=55&from=107C195010&gsid=_2A253Pc5MDeRxGeBK6FAU-CzMyj2IHXVSa0aErDV6PUJbkdAKLUXikWpNHetkT4FG1cwd-aJIY8mFs-sLNfa9Yoqb&imsi=310260361179574&lang=zh_CN&lfid=100303type%3D3%26q%3D%E6%9D%A8%E5%B9%82%26t%3D0&page=1&skin=default&count=20&oldwm=5091_0008&sflag=1&containerid=1076031195242865_-_WEIBO_SECOND_PROFILE_WEIBO&ignore_inturrpted_error=true&luicode=10000003&need_head_cards=0")
		    		.header("User-Agent",
							"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.32 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
					.header("Cache-Control", "max-age=60").header("Accept", "*/*")
					.header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
					.header("Referer", "http://music.163.com/song?id=2332323")
					.header("Origin", "http://music.163.com").header("Host", "music.163.com")
					.header("Content-Type", "application/x-www-form-urlencoded")
					.method(Method.POST).ignoreContentType(true).timeout(2000000000)
					.execute();
			String body = execute.body();
			System.out.println(body);
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		
	}

Source File: PostSend2.java From crawler-jsoup-maven with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
        
        String url = "http://jzsc.mohurd.gov.cn/dataservice/query/comp/list";
        Response res = Jsoup.connect(url).execute();
        Map<String, String> cookies = res.cookies();
        System.out.println(cookies);
        
        Map<String, String> datas = new HashMap<>();
        datas.put("$total", "297267");
        datas.put("$reload", "1");
        datas.put("$pg", "31&&1=1");
        datas.put("$pgsz", "15");
        
        Response res1 = Jsoup.connect(url).cookies(cookies).data(datas).method(Method.POST).execute();
        
        String html = res1.body();
        System.out.println(html);
        
//        String jsonBody = "{pg:30,ps:15,tt:297267,pn:5,pc:19818,id:'',st:true}"; 
//        
//        Connection connection = Jsoup.connect("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .userAgent("Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36") // User-Agent of Chrome 55  
//        .referrer("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .header("Content-Type", "application/x-www-form-urlencoded")  
//        .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")  
//        .header("Accept-Encoding", "gzip, deflate")  
//        .header("Accept-Language", "zh-CN,en-US;q=0.8,zh;q=0.6,ja;q=0.4,en;q=0.2")  
//        .header("Cookie", "filter_comp=; JSESSIONID=F53A68FBA45CCF750F9F32ACF090DC9B")  
//        .header("Host", "jzsc.mohurd.gov.cn")  
//        .header("Connection", "keep-alive")  
//        .header("X-Requested-With", "XMLHttpRequest")  
//        .header("Upgrade-Insecure-Requests", "1")  
//        .requestBody(jsonBody)  
//        .maxBodySize(100)  
//        .timeout(1000 * 10)  
//        .method(Connection.Method.POST);  
//  
//        Response response = connection.execute(); 
//        
//        String body = response.body();
//        System.out.println(body);

    }

Java Code Examples for org.jsoup.Connection.Response#body()