Java Code Examples for org.jsoup.Connection.Response#cookies()

The following examples show how to use org.jsoup.Connection.Response#cookies() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TwodgalleriesRipper.java    From ripme with MIT License 6 votes vote down vote up
private void login() throws IOException {
    Response resp = Http.url(this.url).response();
    cookies = resp.cookies();
    String ctoken = resp.parse().select("form > input[name=ctoken]").first().attr("value");

    Map<String,String> postdata = new HashMap<>();
    postdata.put("user[login]", new String(Base64.decode("cmlwbWU=")));
    postdata.put("user[password]", new String(Base64.decode("cmlwcGVy")));
    postdata.put("rememberme", "1");
    postdata.put("ctoken", ctoken);

    resp = Http.url("http://en.2dgalleries.com/account/login")
               .referrer("http://en.2dgalleries.com/")
               .cookies(cookies)
               .data(postdata)
               .method(Method.POST)
               .response();
    cookies = resp.cookies();
}
 
Example 2
Source File: FuskatorRipper.java    From ripme with MIT License 5 votes vote down vote up
@Override
public Document getFirstPage() throws IOException {
    // return Http.url(url).get();
    Response res = Http.url(url).response();
    cookies = res.cookies();
    return res.parse();
}
 
Example 3
Source File: ThechiveRipper.java    From ripme with MIT License 5 votes vote down vote up
@Override
public Document getNextPage(Document doc) throws IOException {
    Matcher matcher = p1.matcher(url.toExternalForm());

    if (matcher.matches()) {
        // url type thechive.com/YEAR/MONTH/DAY/POSTTITLE/ has a single page.
        return null;
    } else {
        if (nextSeed == null) {
            throw new IOException("No more pages.");
        }
    }

    // Following try block checks if the next JSON object has images or not.
    // This is done to avoid IOException in rip() method, caused when
    // getURLsFromPage() returns empty list.
    JSONArray imgList;
    try {
        Response response = Http.url(jsonUrl).data("seed", nextSeed).data("queryType", "by-username")
                .data("username", username).ignoreContentType().cookies(cookies).response();
        cookies = response.cookies();
        JSONObject json = new JSONObject(response.body());
        imgList = json.getJSONArray("uploads");
    } catch (Exception e) {
        throw new IOException("Error fetching next page.", e);
    }

    if (imgList != null && imgList.length() > 0) {
        // Pass empty document as it is of no use for thechive.com/userName url type.
        return new Document(url.toString());
    } else {
        // Return null as this is last page.
        return null;
    }
}
 
Example 4
Source File: AmazonLoginApater.java    From crawler-jsoup-maven with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    // grab login form page first
    try {
        
        //lets make data map containing all the parameters and its values found in the form
        Map<String, String> mapParamsData = new HashMap<String, String>();
        mapParamsData.put("email", "[email protected]");
        mapParamsData.put("password", "bluetata");
        
        Response loginResponse = Jsoup.connect("https://passport.jd.com/new/login.aspx")
                .userAgent(USER_AGENT)
                .timeout(TIMEOUT_UNIT * TIMEOUT_TIMES)
                .data(mapParamsData)
                .method(Method.POST)
                .followRedirects(true)
                .execute();
        
        System.out.println("Fetched login page");
        // System.out.println(loginResponse.toString());
        
      //get the cookies from the response, which we will post to the action URL
        Map<String, String> mapLoginPageCookies = loginResponse.cookies();
        
        System.out.println(mapLoginPageCookies);
        
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 5
Source File: WebtoonsRipper.java    From ripme with MIT License 4 votes vote down vote up
@Override
public Document getFirstPage() throws IOException {
    Response resp = Http.url(url).response();
    cookies = resp.cookies();
    return Http.url(url).get();
}
 
Example 6
Source File: GITHUBLoginApater.java    From crawler-jsoup-maven with Apache License 2.0 4 votes vote down vote up
/**
     * @param userName 用户名
     * @param pwd 密码
     * @throws Exception
     */
    public static void simulateLogin(String userName, String pwd) throws Exception {

        /* 
         * 第一次请求 
         * grab login form page first
         * 获取登陆提交的表单信息,及修改其提交data数据(login,password)
         */
        // get the response, which we will post to the action URL(rs.cookies())
        Connection con = Jsoup.connect(LOGIN_URL);  // 获取connection
        con.header(USER_AGENT, USER_AGENT_VALUE);   // 配置模拟浏览器
        Response rs = con.execute();                // 获取响应
        Document d1 = Jsoup.parse(rs.body());       // 转换为Dom树
        
        System.out.println(d1);
        
        List<Element> eleList = d1.select("form");  // 获取提交form表单,可以通过查看页面源码代码得知

        // 获取cooking和表单属性
        // lets make data map containing all the parameters and its values found in the form
        Map<String, String> datas = new HashMap<>();
        
        // 01/24/2019 17:45 bluetata 更新 -------------------------------------------------------------- Start ----------
        // GitHub多次改版更新,最新的提交request data为:
        
        // authenticity_token   ll0RJnG1f9XDAaN1DxnyTDzCs+YXweEZWel9kGkq8TvXH83HjCwPG048sJ/VVjDA94YmbF0qvUgcJx8/IKlP8Q==
        // commit  Sign+in
        // login   bluetata
        // password    password123
        // utf8    ✓
        
        for(int i = 0; i < eleList.size(); i++) {
        
            for (Element e : eleList.get(i).getAllElements()) {
                // 设置用户名
                if (e.attr("name").equals("login")) {
                    e.attr("value", userName);
                }
                // 设置用户密码
                if (e.attr("name").equals("password")) {
                    e.attr("value", pwd);
                }
                // 排除空值表单属性
                if (e.attr("name").length() > 0) {
                    datas.put(e.attr("name"), e.attr("value"));
                }
            }
        }

        
//      旧逻辑  delete  01/24/2019 17:49 bluetata --------------------------------------------start
//        for (Element e : eleList.get(0).getAllElements()) {
//            // 设置用户名
//            if (e.attr("name").equals("login")) {
//                e.attr("value", userName);
//            }
//            // 设置用户密码
//            if (e.attr("name").equals("password")) {
//                e.attr("value", pwd);
//            }
//            // 排除空值表单属性
//            if (e.attr("name").length() > 0) {
//                datas.put(e.attr("name"), e.attr("value"));
//            }
//        }
//      旧逻辑  delete  01/24/2019 17:49 bluetata --------------------------------------------end
        
        
        // 01/24/2019 17:45 bluetata 更新 --------------------------------------------------------------- End -----------
        
        /*
         * 第二次请求,以post方式提交表单数据以及cookie信息
         */
        Connection con2 = Jsoup.connect("https://github.com/session");
        con2.header(USER_AGENT, USER_AGENT_VALUE);
        // 设置cookie和post上面的map数据
        Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
        // 打印,登陆成功后的信息
        System.out.println(login.body());

        // 登陆成功后的cookie信息,可以保存到本地,以后登陆时,只需一次登陆即可
        Map<String, String> map = login.cookies();
        for (String s : map.keySet()) {
            System.out.println(s + " : " + map.get(s));
        }
    }
 
Example 7
Source File: CentralnicLoinApater.java    From crawler-jsoup-maven with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
    
    String url = "https://registrar-console.centralnic.com/dashboard/login";
    String userAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36";
    
    Response res = Jsoup.connect(url).userAgent(userAgent).timeout(1000 * 3).execute();

    Document d1 = Jsoup.parse(res.body());
    
    List<Element> et = d1.select("#login_form");// 获取form表单
    String XSRF = null;
    // 获取,cooking和表单属性,下面map存放post时的数据
    Map<String, String> datas = new java.util.HashMap<>();
    for (Element e : et.get(0).getAllElements()) {
      if (e.attr("name").equals("regid")) {
        e.attr("value", "*****");// 设置id
      }
      if (e.attr("name").equals("user")) {
        e.attr("value", "*****");// 设置用户名
      }
      if (e.attr("name").equals("password")) {
        e.attr("value", "*****"); // 设置用户密码
      }
      if (e.attr("name").equals("XSRF")) {
        XSRF = e.attr("value");
      }
      if (e.attr("name").length() > 0) {// 排除空值表单属性
        datas.put(e.attr("name"), e.attr("value"));
      }
    }
    
    System.out.println(datas);
    System.out.println("======================");
    System.out.println(res.url());
    System.out.println(res.statusCode());
    
    Map<String, String> cookies = res.cookies();
    cookies.put("remember_me", "H2454482%3Atoddhan");
    cookies.put("lang", "en");
    
    System.out.println(res.cookies());
    
    /* 
     * 第二次请求,以post方式提交表单数据以及cookie信息 
     */  
    Connection con2 = Jsoup.connect("https://registrar-console.centralnic.com/dashboard/login_target");  
    con2.header("User-Agent", userAgent);  
    // 设置cookie和post上面的map数据  
    Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST)  
                            .data(datas).cookies(cookies).referrer("https://registrar-console.centralnic.com/dashboard/login")
                            .header("host", "registrar-console.centralnic.com")
                            .execute();  
    
    Map<String, String> cookies2 = login.cookies();
    cookies2.put("remember_me", "H2454482%3Atoddhan");
    cookies2.put("lang", "en");
    System.out.println(cookies2);
    
    System.out.println(login.statusCode());
    
    System.err.println(JsoupUtil.getDocumentWithCookies("https://registrar-console.centralnic.com/", cookies2));
    
    
    System.out.println(JsoupUtil.getDocumentWithCookies("https://registrar-console.centralnic.com/graphs/file_browser/droplist", cookies2));
    
    
}
 
Example 8
Source File: JDLoginApater.java    From crawler-jsoup-maven with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {
    // grab login form page first
    try {
        
        //lets make data map containing all the parameters and its values found in the form
        Map<String, String> mapParamsData = new HashMap<String, String>();
        mapParamsData.put("loginType", "f");
        mapParamsData.put("loginname", "18241141433");
        mapParamsData.put("nloginpwd", "password");
        mapParamsData.put("eid", "2QE5VJVZUBCRYD7LQZBJBTEFRNKPMQQA5OXKXNY7AAN4A3DKDTR7IN3GXHE5C6B4GTMW3Z53B4RGORB6YG5LUWF2UA");
        mapParamsData.put("fp", "ae5baf289624644fced3f921c6a3792c");
        mapParamsData.put("pubKey", "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDC7kw8r6tq43pwApYvkJ5laljaN9BZb21TAIfT/vexbobzH7Q8SUdP5uDPXEBKzOjx2L28y7Xs1d9v3tdPfKI2LR7PAzWBmDMn8riHrDDNpUpJnlAGUqJG9ooPn8j7YNpcxCa1iybOlc2kEhmJn5uwoanQq+CA6agNkqly2H4j6wIDAQAB");
        mapParamsData.put("sa_token", "B68C442BE645754F33277E701208059080DD726A94A73F76DEC3053A838549C06EB7D3797CE1C5BBE7C2B2EF9CA7D4676F3D489984B517943EA13575FA80C7E73160F85EDB2705D145C52621D18B86C98B49AAF0DA97A2A7A964A78DDDA048AE592CF17C2AD8BF442AD743460B9316993DCDF5924AD8536FD4392C95A998E1C4C4EEDF76BF8FF03AAC145E449EAB889368EE1E7DA72B18881D527D9F51BAD9E2678DDEAFB33A647DD6D48B2A3BE1BC51DDC55AB1EAAEE2DE9A3CEA3702F93AAD1EC8EF740B632F5A4EC102498CDB31AF91CEA15DB3B6DF6FAC6CA31473ACC5E2CD727F80D2746F504A85379E7F3971086C13BA743F21731CEBFEC558E54E8D5D486CC3A19266E238F539A59C2F8630964981217DCC3B31B324F7DBF41FAEA47CA363904F06816BA9730B31BDD9FFA5498C1D0C36D67F315BA4F9236AC77BAFD5");
        mapParamsData.put("seqSid", "5844668515208615000");
        mapParamsData.put("uuid", "5653262a-5ef1-47c6-8ac2-427f519fdcfa");
        
        Response loginResponse = Jsoup.connect(LOGIN_URI)
                .userAgent(USER_AGENT)
                .timeout(TIMEOUT_UNIT * TIMEOUT_TIMES)
                .data(mapParamsData)
                .method(Method.POST)
                .followRedirects(true)
                .execute();
        
        System.out.println("Fetched login page");
        // System.out.println(loginResponse.toString());
        
      //get the cookies from the response, which we will post to the action URL
        Map<String, String> mapLoginPageCookies = loginResponse.cookies();
        
        System.out.println(mapLoginPageCookies);
        
        Document doc = Jsoup.connect("http://order.jd.com/center/list.action")
                .cookies(mapLoginPageCookies)
                .timeout(30000)
                .get();
        System.out.println(doc.toString());
        
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 9
Source File: PostSend2.java    From crawler-jsoup-maven with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
        
        String url = "http://jzsc.mohurd.gov.cn/dataservice/query/comp/list";
        Response res = Jsoup.connect(url).execute();
        Map<String, String> cookies = res.cookies();
        System.out.println(cookies);
        
        Map<String, String> datas = new HashMap<>();
        datas.put("$total", "297267");
        datas.put("$reload", "1");
        datas.put("$pg", "31&&1=1");
        datas.put("$pgsz", "15");
        
        Response res1 = Jsoup.connect(url).cookies(cookies).data(datas).method(Method.POST).execute();
        
        String html = res1.body();
        System.out.println(html);
        
//        String jsonBody = "{pg:30,ps:15,tt:297267,pn:5,pc:19818,id:'',st:true}"; 
//        
//        Connection connection = Jsoup.connect("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .userAgent("Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36") // User-Agent of Chrome 55  
//        .referrer("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")  
//        .header("Content-Type", "application/x-www-form-urlencoded")  
//        .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")  
//        .header("Accept-Encoding", "gzip, deflate")  
//        .header("Accept-Language", "zh-CN,en-US;q=0.8,zh;q=0.6,ja;q=0.4,en;q=0.2")  
//        .header("Cookie", "filter_comp=; JSESSIONID=F53A68FBA45CCF750F9F32ACF090DC9B")  
//        .header("Host", "jzsc.mohurd.gov.cn")  
//        .header("Connection", "keep-alive")  
//        .header("X-Requested-With", "XMLHttpRequest")  
//        .header("Upgrade-Insecure-Requests", "1")  
//        .requestBody(jsonBody)  
//        .maxBodySize(100)  
//        .timeout(1000 * 10)  
//        .method(Connection.Method.POST);  
//  
//        Response response = connection.execute(); 
//        
//        String body = response.body();
//        System.out.println(body);

    }