org.jsoup.Connection.Method Java Examples
The following examples show how to use
org.jsoup.Connection.Method.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: HttpUtil.java From JsDroidCmd with Mozilla Public License 2.0 | 6 votes |
|
public static String post(String url, Map<String, String> headers,
Map<String, String> params) {
try {
Connection connect = Jsoup.connect(url);
connect.ignoreContentType(true);
connect.ignoreHttpErrors(true);
if (params != null) {
connect.data(params);
}
if (headers != null) {
connect.headers(headers);
}
connect.method(Method.POST);
return connect.execute().body();
} catch (Throwable e) {
}
return null;
}
Example #2
| Source File: ArtStationRipper.java From ripme with MIT License | 6 votes |
|
private JSONObject getJson(URL url) throws IOException {
Connection con = Http.url(url).method(Method.GET).connection();
con.ignoreHttpErrors(true);
con.ignoreContentType(true);
con.userAgent(
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
con.header("Accept-Language", "en-US,en;q=0.5");
// con.header("Accept-Encoding", "gzip, deflate, br");
con.header("Upgrade-Insecure-Requests", "1");
Response res = con.execute();
int status = res.statusCode();
if (status / 100 == 2) {
String jsonString = res.body();
return new JSONObject(jsonString);
}
throw new IOException("Error fetching json. Status code:" + status);
}
Example #3
| Source File: TwodgalleriesRipper.java From ripme with MIT License | 6 votes |
|
private void login() throws IOException {
Response resp = Http.url(this.url).response();
cookies = resp.cookies();
String ctoken = resp.parse().select("form > input[name=ctoken]").first().attr("value");
Map<String,String> postdata = new HashMap<>();
postdata.put("user[login]", new String(Base64.decode("cmlwbWU=")));
postdata.put("user[password]", new String(Base64.decode("cmlwcGVy")));
postdata.put("rememberme", "1");
postdata.put("ctoken", ctoken);
resp = Http.url("http://en.2dgalleries.com/account/login")
.referrer("http://en.2dgalleries.com/")
.cookies(cookies)
.data(postdata)
.method(Method.POST)
.response();
cookies = resp.cookies();
}
Example #4
| Source File: PersonalTest.java From emotional_analysis with Apache License 2.0 | 6 votes |
|
/**
* 抓取个人页面
* <p>Title: test2</p>
* <p>Description: </p>
* @throws Exception
*/
@Test
public void test2() throws Exception{
System.setProperty("http.maxRedirects", "5000");
System.getProperties().setProperty("proxySet", "true");
// 如果不设置，只要代理IP和代理端口正确,此项不设置也可以
System.getProperties().setProperty("http.proxyHost", "139.224.80.139");
System.getProperties().setProperty("http.proxyPort", "3128");
String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);
String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"uid\":2763211,\"offset\":0,\"limit\":50};","0CoJUm6Qyw8W8jud"), secKey);
String encSecKey = EncryptUtils.rsaEncrypt(secKey);
Response execute = Jsoup.connect("http://music.163.com/weapi/user/playlist")
.data("params",encText)
.data("encSecKey",encSecKey)
.method(Method.POST).ignoreContentType(true).execute();
String string = execute.body().toString();
System.out.println(string);
}
Example #5
| Source File: Http.java From ripme with MIT License | 5 votes |
|
private void defaultSettings() {
this.retries = Utils.getConfigInteger("download.retries", 1);
connection = Jsoup.connect(this.url);
connection.userAgent(AbstractRipper.USER_AGENT);
connection.method(Method.GET);
connection.timeout(TIMEOUT);
connection.maxBodySize(0);
// Extract cookies from config entry:
// Example config entry:
// cookies.reddit.com = reddit_session=<value>; other_cookie=<value>
connection.cookies(cookiesForURL(this.url));
}
Example #6
| Source File: VkRipper.java From ripme with MIT License | 5 votes |
|
private Map<String,String> getPhotoIDsToURLs(String photoID) throws IOException {
Map<String,String> photoIDsToURLs = new HashMap<>();
Map<String,String> postData = new HashMap<>();
// act=show&al=1&list=album45506334_172415053&module=photos&photo=45506334_304658196
postData.put("list", getGID(this.url));
postData.put("act", "show");
postData.put("al", "1");
postData.put("module", "photos");
postData.put("photo", photoID);
Response res = Jsoup.connect("https://vk.com/al_photos.php")
.header("Referer", this.url.toExternalForm())
.header("Accept", "*/*")
.header("Accept-Language", "en-US,en;q=0.5")
.header("Content-Type", "application/x-www-form-urlencoded")
.header("X-Requested-With", "XMLHttpRequest")
.ignoreContentType(true)
.userAgent(USER_AGENT)
.timeout(5000)
.data(postData)
.method(Method.POST)
.execute();
String jsonString = res.body();
JSONObject json = new JSONObject(jsonString);
JSONObject photoObject = findJSONObjectContainingPhotoId(photoID, json);
String bestSourceUrl = getBestSourceUrl(photoObject);
if (bestSourceUrl != null) {
photoIDsToURLs.put(photoID, bestSourceUrl);
} else {
LOGGER.error("Could not find image source for " + photoID);
}
return photoIDsToURLs;
}
Example #7
| Source File: FuskatorRipper.java From ripme with MIT License | 5 votes |
|
private void getXAuthToken() throws IOException {
if (cookies == null || cookies.isEmpty()) {
throw new IOException("Null cookies or no cookies found.");
}
Response res = Http.url(xAuthUrl).cookies(cookies).method(Method.POST).response();
xAuthToken = res.body();
}
Example #8
| Source File: IpProxy.java From emotional_analysis with Apache License 2.0 | 5 votes |
|
public static List<IpEntity> getProxyIp(String url) throws Exception{
ArrayList<IpEntity> ipList = new ArrayList<>();
Response execute = Jsoup.connect(url)
.header("User-Agent",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
.header("Cache-Control", "max-age=60").header("Accept", "*/*")
.header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
.header("Referer", "http://music.163.com/song?id=186016")
.header("Origin", "http://music.163.com").header("Host", "music.163.com")
.header("Content-Type", "application/x-www-form-urlencoded")
.header("Cookie",
"UM_distinctid=15e9863cf14335-0a09f939cd2af9-6d1b137c-100200-15e9863cf157f1; vjuids=414b87eb3.15e9863cfc1.0.ec99d6f660d09; _ntes_nnid=4543481cc76ab2fd3110ecaafd5f1288,1505795231854; _ntes_nuid=4543481cc76ab2fd3110ecaafd5f1288; __s_=1; __gads=ID=6cbc4ab41878c6b9:T=1505795247:S=ALNI_MbCe-bAY4kZyMbVKlS4T2BSuY75kw; usertrack=c+xxC1nMphjBCzKpBPJjAg==; NTES_CMT_USER_INFO=100899097%7Cm187****4250%7C%7Cfalse%7CbTE4NzAzNDE0MjUwQDE2My5jb20%3D; [email protected]|1507178162|2|mail163|00&99|CA&1506163335&mail163#hun&430800#10#0#0|187250&1|163|[email protected]; vinfo_n_f_l_n3=8ba0369be425c0d2.1.7.1505795231863.1507950353704.1508150387844; vjlast=1505795232.1508150167.11; Province=0450; City=0454; _ga=GA1.2.1044198758.1506584097; _gid=GA1.2.763458995.1508907342; JSESSIONID-WYYY=Zm%2FnBG6%2B1vb%2BfJp%5CJP8nIyBZQfABmnAiIqMM8fgXABoqI0PdVq%2FpCsSPDROY1APPaZnFgh14pR2pV9E0Vdv2DaO%2BKkifMncYvxRVlOKMEGzq9dTcC%2F0PI07KWacWqGpwO88GviAmX%2BVuDkIVNBEquDrJ4QKhTZ2dzyGD%2Bd2T%2BbiztinJ%3A1508946396692; _iuqxldmzr_=32; playerid=20572717; MUSIC_U=39d0b2b5e15675f10fd5d9c05e8a5d593c61fcb81368d4431bab029c28eff977d4a57de2f409f533b482feaf99a1b61e80836282123441c67df96e4bf32a71bc38be3a5b629323e7bf122d59fa1ed6a2; __remember_me=true; __csrf=2032a8f34f1f92412a49ba3d6f68b2db; __utma=94650624.1044198758.1506584097.1508939111.1508942690.40; __utmb=94650624.20.10.1508942690; __utmc=94650624; __utmz=94650624.1508394258.18.4.utmcsr=xujin.org|utmccn=(referral)|utmcmd=referral|utmcct=/")
.method(Method.GET).ignoreContentType(true)
.timeout(2099999999).execute();
Document pageJson = execute.parse();
Element body = pageJson.body();
List<Node> childNodes = body.childNode(11).childNode(3).childNode(5).childNode(1).childNodes();
//把前10位的代理IP放到List中
for(int i = 2;i <= 30;i += 2){
IpEntity ipEntity = new IpEntity();
Node node = childNodes.get(i);
List<Node> nodes = node.childNodes();
String ip = nodes.get(3).childNode(0).toString();
int port = Integer.parseInt(nodes.get(5).childNode(0).toString());
ipEntity.setIp(ip);
ipEntity.setPort(port);
ipList.add(ipEntity);
}
return ipList;
}
Example #9
| Source File: FollowingTest.java From emotional_analysis with Apache License 2.0 | 5 votes |
|
@Test
public void test3() throws Exception {
String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);
String encSecKey = EncryptUtils.rsaEncrypt(secKey);
String encText = EncryptUtils.aesEncrypt(
EncryptUtils.aesEncrypt("{\"userId\":63362967,\"offset\":0,\"limit\":100};", "0CoJUm6Qyw8W8jud"),
secKey);
Response execute = Jsoup.connect("http://music.163.com/weapi/user/getfolloweds").data("params", encText)
.data("encSecKey", encSecKey).method(Method.POST).ignoreContentType(true).execute();
// String string = execute.body().toString();
// System.out.println(string);
// System.out.println(string.equals("{\"code\":200,\"more\":false,\"followeds\":[]}"));
// new FileSourceUtils().importData(string, "follow");
int i = 0;
while (true) {
encText = EncryptUtils.aesEncrypt(
EncryptUtils.aesEncrypt("{\"userId\":92271210,\"offset\":" + i + ",\"limit\":" + 100 + "};",
"0CoJUm6Qyw8W8jud"),
secKey);
execute = Jsoup.connect("http://music.163.com/weapi/user/getfolloweds").data("params", encText)
.data("encSecKey", encSecKey).method(Method.POST).ignoreContentType(true).execute();
String string1 = execute.body().toString();
if(string1.equals("") || i == 1000){
break;
}
System.out.println(string1);
new FileSourceUtils().importData(string1,"follow");
i += 100;
}
}
Example #10
| Source File: FollowerTest.java From emotional_analysis with Apache License 2.0 | 5 votes |
|
/**
* <p>Title: test3</p>
* <p>Description: </p>
* @throws Exception
*/
@Test
public void test3() throws Exception{
String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);
String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"offset\":0,\"offset\":0,\"limit\":50};","0CoJUm6Qyw8W8jud"), secKey);
String encSecKey = EncryptUtils.rsaEncrypt(secKey);
Response execute = Jsoup.connect("http://music.163.com/weapi/user/getfollows/380377129")
.data("params",encText)
.data("encSecKey",encSecKey)
.method(Method.POST).ignoreContentType(true).execute();
String string = execute.body().toString();
System.out.println(string);
}
Example #11
| Source File: PersonalTest.java From emotional_analysis with Apache License 2.0 | 5 votes |
|
/**
* 个人动态
* @throws Exception
*/
@Test
public void test3() throws Exception{
String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);
String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"uid\":2763211,\"offset\":0,\"limit\":50};","0CoJUm6Qyw8W8jud"), secKey);
String encSecKey = EncryptUtils.rsaEncrypt(secKey);
Response execute = Jsoup.connect("http://music.163.com/weapi/event/get/2763211")
.data("params",encText)
.data("encSecKey",encSecKey)
.method(Method.POST).ignoreContentType(true).execute();
String string = execute.body().toString();
System.out.println(string);
}
Example #12
| Source File: CommonThread.java From emotional_analysis with Apache License 2.0 | 5 votes |
|
@Override
public void run() {
for(int i = 0;i <= pageSize;i++){
try{
String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);//limit
String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"offset\":"+ i * 10 +",\"limit\":"+(i+1) * 10+"};","0CoJUm6Qyw8W8jud"), secKey);
String encSecKey = EncryptUtils.rsaEncrypt(secKey);
Response execute = Jsoup.connect("http://music.163.com/weapi/v1/resource/comments/R_SO_4_"+songId)
.data("params",encText)
.data("encSecKey",encSecKey)
.method(Method.POST).ignoreContentType(true).timeout(2000000000).execute();
String string = execute.body().toString();
System.out.println(string);
ObjectMapper objectMapper = new ObjectMapper();
CommentBean readValue = objectMapper.readValue(string.getBytes(), CommentBean.class);
long total = readValue.getTotal();
pageSize = total / 10;
List<Comments> comments = readValue.getComments();
for (Comments comments2 : comments) {
String content = comments2.getContent();
long time = comments2.getTime();
User user = comments2.getUser();
String avatarUrl = user.getAvatarUrl();
String nickname = user.getNickname();
long userId = user.getUserId();
//=========================================数据持久化==========================
System.out.println("昵称:"+nickname+"评论内容为："+content+"评论时间为:"+time+"头像地址"+avatarUrl+"用户的ID"+userId);
}
}catch(Exception e){
}
}
}
Example #13
| Source File: FacebookLoginApater.java From crawler-jsoup-maven with Apache License 2.0 | 5 votes |
|
static Map<String,String> connect() throws IOException{
Connection.Response res = Jsoup.connect("https://www.facebook.com/login.php")
.data("username", "[email protected]", "password", "password")
.timeout(30 * 1000)
.userAgent("Mozilla/5.0")
.method(Method.POST)
.execute();
Document doc = res.parse();
System.out.println(doc);
Map<String, String> loginCookies = res.cookies();
String sessionId = res.cookie("SESSIONID");
return loginCookies;
}
Example #14
| Source File: WeiboCNLoginApater.java From crawler-jsoup-maven with Apache License 2.0 | 5 votes |
|
public static void main(String[] args) throws IOException {
try {
String url = "https://www.oschina.net/home/login";
String userAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.130 Safari/537.36";
Connection.Response response = Jsoup.connect(url).userAgent(userAgent).method(Connection.Method.GET)
.execute();
response = Jsoup.connect(url).cookies(response.cookies()).userAgent(userAgent)
.referrer("https://www.oschina.net/home/login?goto_page=https%3A%2F%2Fmy.oschina.net%2Fbluetata")
.data("username", "[email protected]", "password", "lvmeng152300").data("save_login", "1")
.followRedirects(false)
.method(Connection.Method.POST).followRedirects(true).timeout(30 * 1000).execute();
System.err.println(response.statusCode());
Document doc = Jsoup.connect("https://my.oschina.net/bluetata").cookies(response.cookies())
.userAgent(userAgent).timeout(30 * 1000).get();
System.out.println(doc);
} catch (IOException e) {
e.printStackTrace();
}
}
Example #15
| Source File: WeiboCNLoginApater.java From crawler-jsoup-maven with Apache License 2.0 | 5 votes |
|
static Map<String, String> connect() throws IOException {
// Connection.Response loginForm =
// Jsoup.connect("https://passport.weibo.cn/signin/login")
// .method(Connection.Method.GET)
// .execute();
//
// Connection.Response res =
// Jsoup.connect("https://passport.weibo.cn/signin/login")
// .data("username", "18241141433", "password", "152300")
// .data("ec", "0", "entry", "mweibo")
// .data("mainpageflag", "1", "savestate", "1")
// .timeout(30 * 1000)
// .userAgent("Mozilla/5.0")
// .cookies(loginForm.cookies())
// .method(Method.POST)
// .execute();
// Document doc = res.parse();
// System.out.println(doc);
Connection.Response loginForm = Jsoup.connect("https://www.oschina.net/home/login")
.method(Connection.Method.GET).execute();
Connection.Response res = Jsoup.connect("https://www.oschina.net/home/login").header("Host", "www.oschina.net")
.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0")
.referrer("https://www.oschina.net/home/login")
.data("username", "[email protected]", "password", "lvmeng152300").data("save_login", "1")
.timeout(30 * 1000).cookies(loginForm.cookies()).method(Method.POST).execute();
Document doc = res.parse();
System.out.println(doc);
Map<String, String> loginCookies = res.cookies();
String sessionId = res.cookie("SESSIONID");
return loginCookies;
}
Example #16
| Source File: AmazonLoginApater.java From crawler-jsoup-maven with Apache License 2.0 | 5 votes |
|
public static void main(String[] args) {
// grab login form page first
try {
//lets make data map containing all the parameters and its values found in the form
Map<String, String> mapParamsData = new HashMap<String, String>();
mapParamsData.put("email", "[email protected]m");
mapParamsData.put("password", "bluetata");
Response loginResponse = Jsoup.connect("https://passport.jd.com/new/login.aspx")
.userAgent(USER_AGENT)
.timeout(TIMEOUT_UNIT * TIMEOUT_TIMES)
.data(mapParamsData)
.method(Method.POST)
.followRedirects(true)
.execute();
System.out.println("Fetched login page");
// System.out.println(loginResponse.toString());
//get the cookies from the response, which we will post to the action URL
Map<String, String> mapLoginPageCookies = loginResponse.cookies();
System.out.println(mapLoginPageCookies);
} catch (Exception e) {
e.printStackTrace();
}
}
Example #17
| Source File: WebConnector.java From JavaSkype with MIT License | 5 votes |
|
private void updateContacts() throws IOException {
if (updated) {
return;
}
updated = true;
String selfResponse = sendRequest(Method.GET, "/users/self/profile").body();
JSONObject selfJSON = new JSONObject(selfResponse);
updateUser(selfJSON, false);
String profilesResponse =
sendRequest(Method.GET, "https://contacts.skype.com/contacts/v2/users/" + getSelfLiveUsername() + "/contacts", true).body();
try {
JSONObject json = new JSONObject(profilesResponse);
if (json.optString("message", null) != null) {
throw new ParseException("Error while parsing contacts response: " + json.optString("message"));
}
JSONArray profilesJSON = json.getJSONArray("contacts");
for (int i = 0; i < profilesJSON.length(); i++) {
User user = updateUser(profilesJSON.getJSONObject(i), true);
if (user != null && !user.getUsername().equalsIgnoreCase("echo123")) {
skype.addContact(user.getUsername());
}
}
} catch (JSONException e) {
throw new ParseException(e);
}
}
Example #18
| Source File: WebConnector.java From JavaSkype with MIT License | 5 votes |
|
private Response sendRequest(Method method, String apiPath, boolean absoluteApiPath, String... keyval) throws IOException {
String url = absoluteApiPath ? apiPath : SERVER_HOSTNAME + apiPath;
Connection conn = Jsoup.connect(url).maxBodySize(100 * 1024 * 1024).timeout(10000).method(method).ignoreContentType(true).ignoreHttpErrors(true);
logger.finest("Sending " + method + " request at " + url);
if (skypeToken != null) {
conn.header("X-Skypetoken", skypeToken);
} else {
logger.fine("No token sent for the request at: " + url);
}
conn.data(keyval);
return conn.execute();
}
Example #19
| Source File: Http.java From ripme with MIT License | 4 votes |
|
public Document get() throws IOException {
connection.method(Method.GET);
return response().parse();
}
Example #20
| Source File: Http.java From ripme with MIT License | 4 votes |
|
public Document post() throws IOException {
connection.method(Method.POST);
return response().parse();
}
Example #21
| Source File: ArtStationRipper.java From ripme with MIT License | 4 votes |
|
/**
* Parses an ArtStation URL.
*
* @param url URL to an ArtStation user profile
* (https://www.artstation.com/username) or single project
* (https://www.artstation.com/artwork/projectid)
* @return ParsedURL object containing URL type, JSON location and ID (stores
* account name or project hash, depending of the URL type identified)
*
*/
private ParsedURL parseURL(URL url) {
String htmlSource;
ParsedURL parsedURL;
// Load HTML Source of the specified URL
try {
// htmlSource = Http.url(url).get().html();
Connection con = Http.url(url).method(Method.GET).connection();
con.ignoreHttpErrors(true);
con.userAgent("Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0");
con.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
con.header("Accept-Language", "en-US,en;q=0.5");
// con.header("Accept-Encoding", "gzip, deflate, br");
con.header("Upgrade-Insecure-Requests", "1");
Response res = con.execute();
int status = res.statusCode();
if (status / 100 == 2) {
htmlSource = res.parse().html();
} else if (status == 403 && url.toString().contains("artwork/")) {
// Catches cloudflare page. Error 403.
// Usually caused by artwork URLs( arstation.com/artwork/someProjectId)
String urlId = url.toString().substring(url.toString().lastIndexOf("/") + 1);
String jsonURL = "https://www.artstation.com/projects/" + urlId + ".json";
parsedURL = new ParsedURL(URL_TYPE.SINGLE_PROJECT, jsonURL, urlId);
return parsedURL;
} else {
LOGGER.error("Couldnt fetch URL: " + url);
throw new IOException("Error fetching URL: " + url + " Status Code: " + status);
}
} catch (IOException e) {
htmlSource = "";
}
// Check if HTML Source of the specified URL references a project
Pattern p = Pattern.compile("'/projects/(\\w+)\\.json'");
Matcher m = p.matcher(htmlSource);
if (m.find()) {
parsedURL = new ParsedURL(URL_TYPE.SINGLE_PROJECT,
"https://www.artstation.com/projects/" + m.group(1) + ".json", m.group(1));
return parsedURL;
}
// Check if HTML Source of the specified URL references a user profile
p = Pattern.compile("'/users/([\\w-]+)/quick\\.json'");
m = p.matcher(htmlSource);
if (m.find()) {
parsedURL = new ParsedURL(URL_TYPE.USER_PORTFOLIO,
"https://www.artstation.com/users/" + m.group(1) + "/projects.json", m.group(1));
return parsedURL;
}
// HTML Source of the specified URL doesn't reference a user profile or project
parsedURL = new ParsedURL(URL_TYPE.UNKNOWN, null, null);
return parsedURL;
}
Example #22
| Source File: DeviantartRipper.java From ripme with MIT License | 4 votes |
|
/**
* Stores logged in Cookies. Needed for art pieces only visible to logged in
* users.
*
*
* @throws IOException when failed to load webpage or failed to read/write
* cookies in file (used when running multiple instances of
* RipMe)
*/
private void login() throws IOException {
String customUsername = Utils.getConfigString("DeviantartCustomLoginUsername", this.username);
String customPassword = Utils.getConfigString("DeviantartCustomLoginPassword", this.password);
try {
String dACookies = Utils.getConfigString(utilsKey, null);
updateCookie(dACookies != null ? deserialize(dACookies) : null);
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
if (getDACookie() == null || !checkLogin()) {
LOGGER.info("Do Login now");
// Do login now
Map<String, String> tmpCookies = new HashMap<String, String>();
// Load login page
Response res = Http.url("https://www.deviantart.com/users/login").connection().method(Method.GET)
.referrer(referer).userAgent(userAgent).execute();
tmpCookies.putAll(res.cookies());
// Find tokens
Document doc = res.parse();
tmpCookies.putAll(res.cookies());
Element form = doc.getElementById("login");
String token = form.select("input[name=\"validate_token\"]").first().attr("value");
String key = form.select("input[name=\"validate_key\"]").first().attr("value");
LOGGER.info("Token: " + token + " & Key: " + key);
// Build Login Data
HashMap<String, String> loginData = new HashMap<String, String>();
loginData.put("challenge", "");
loginData.put("username", customUsername);
loginData.put("password", customPassword);
loginData.put("remember_me", "1");
loginData.put("validate_token", token);
loginData.put("validate_key", key);
// Log in using data. Handle redirect
res = Http.url("https://www.deviantart.com/users/login").connection().referrer(referer).userAgent(userAgent)
.method(Method.POST).data(loginData).cookies(tmpCookies).followRedirects(false).execute();
tmpCookies.putAll(res.cookies());
res = Http.url(res.header("location")).connection().referrer(referer).userAgent(userAgent)
.method(Method.GET).cookies(tmpCookies).followRedirects(false).execute();
// Store cookies
tmpCookies.putAll(res.cookies());
updateCookie(tmpCookies);
} else {
LOGGER.info("No new Login needed");
}
LOGGER.info("DA Cookies: " + getDACookie());
}
Example #23
| Source File: HentaifoundryRipper.java From ripme with MIT License | 4 votes |
|
@Override
public Document getFirstPage() throws IOException {
Response resp;
Document doc;
resp = Http.url("https://www.hentai-foundry.com/?enterAgree=1&size=1500")
.referrer("https://www.hentai-foundry.com/")
.cookies(cookies)
.response();
// The only cookie that seems to matter in getting around the age wall is the phpsession cookie
cookies.putAll(resp.cookies());
doc = resp.parse();
String csrf_token = doc.select("input[name=YII_CSRF_TOKEN]")
.first().attr("value");
if (csrf_token != null) {
Map<String,String> data = new HashMap<>();
data.put("YII_CSRF_TOKEN" , csrf_token);
data.put("rating_nudity" , "3");
data.put("rating_violence" , "3");
data.put("rating_profanity", "3");
data.put("rating_racism" , "3");
data.put("rating_sex" , "3");
data.put("rating_spoilers" , "3");
data.put("rating_yaoi" , "1");
data.put("rating_yuri" , "1");
data.put("rating_teen" , "1");
data.put("rating_guro" , "1");
data.put("rating_furry" , "1");
data.put("rating_beast" , "1");
data.put("rating_male" , "1");
data.put("rating_female" , "1");
data.put("rating_futa" , "1");
data.put("rating_other" , "1");
data.put("rating_scat" , "1");
data.put("rating_incest" , "1");
data.put("rating_rape" , "1");
data.put("filter_media" , "A");
data.put("filter_order" , Utils.getConfigString("hentai-foundry.filter_order","date_old"));
data.put("filter_type" , "0");
resp = Http.url("https://www.hentai-foundry.com/site/filters")
.referrer("https://www.hentai-foundry.com/")
.cookies(cookies)
.data(data)
.method(Method.POST)
.response();
cookies.putAll(resp.cookies());
}
else {
LOGGER.info("unable to find csrf_token and set filter");
}
resp = Http.url(url)
.referrer("https://www.hentai-foundry.com/")
.cookies(cookies)
.response();
cookies.putAll(resp.cookies());
return resp.parse();
}
Example #24
| Source File: CommentTest.java From emotional_analysis with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) {
// for(int i = 138789;i < 139989;i++){
// try{
// System.setProperty("http.maxRedirects", "5000");
// System.getProperties().setProperty("proxySet", "true");
// // 如果不设置，只要代理IP和代理端口正确,此项不设置也可以
// System.getProperties().setProperty("http.proxyHost", "124.79.56.115");
// System.getProperties().setProperty("http.proxyPort", "8118");
// String secKey = new BigInteger(100, new SecureRandom()).toString(32).substring(0, 16);//limit
// String encText = EncryptUtils.aesEncrypt(EncryptUtils.aesEncrypt("{\"offset\":0,\"limit\":10};","0CoJUm6Qyw8W8jud"), secKey);
// String encSecKey = EncryptUtils.rsaEncrypt(secKey);
// Response execute = Jsoup
// .connect("https://music.163.com/weapi/v1/resource/comments/R_SO_4_"+i)
// .header("User-Agent",
// "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.32 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
// .header("Cache-Control", "max-age=60").header("Accept", "*/*")
// .header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
// .header("Referer", "http://music.163.com/song?id=2332323")
// .header("Origin", "http://music.163.com").header("Host", "music.163.com")
// .header("Content-Type", "application/x-www-form-urlencoded")
// .data("params",encText)
// .data("encSecKey",encSecKey)
// .method(Method.POST).ignoreContentType(true).timeout(2000000000)
// .execute();
// String string = execute.body().toString();
// System.out.println(string);
//// IpProxy.timerGetIps();
//// String url = "http://music.163.com/weapi/v1/resource/comments/R_SO_4_" + i;
//// String data = CenterUrl.getDataByUrl(url, "{\"offset\":0,\"limit\":10};");
//// System.out.println(data);
// }catch(Exception e){
// e.printStackTrace();
// }
// }
try {
Response execute = Jsoup.connect("https://api.weibo.cn/2/cardlist?networktype=wifi&uicode=10000198"
+ "&moduleID=708&featurecode=10000085"
+ "&wb_version=3523"
+ "&lcardid=1003030212_seqid%3A1671487208%7Ctype%3A3%7Ct%3A0%7Cpos%3A1-0-0%7Cq%3A%E6%9D%A8%E5%B9%82%7Cext%3A%26uid%3D1195242865%26&c=android&i=a032269&s=7f0ba7d8&ft=0&ua=Honor-CHM-TL00H__weibo__7.12.1__android__android4.4.4&wm=5091_0008&aid=01AsfiMEjVzg0YSShsV_ISlRgLOKx7UXK8bxQfU3UzNxYoTHo.&fid=1076031195242865_-_WEIBO_SECOND_PROFILE_WEIBO&uid=6432582011&v_f=2&v_p=55&from=107C195010&gsid=_2A253Pc5MDeRxGeBK6FAU-CzMyj2IHXVSa0aErDV6PUJbkdAKLUXikWpNHetkT4FG1cwd-aJIY8mFs-sLNfa9Yoqb&imsi=310260361179574&lang=zh_CN&lfid=100303type%3D3%26q%3D%E6%9D%A8%E5%B9%82%26t%3D0&page=1&skin=default&count=20&oldwm=5091_0008&sflag=1&containerid=1076031195242865_-_WEIBO_SECOND_PROFILE_WEIBO&ignore_inturrpted_error=true&luicode=10000003&need_head_cards=0")
.header("User-Agent",
"Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.32 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36")
.header("Cache-Control", "max-age=60").header("Accept", "*/*")
.header("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6").header("Connection", "keep-alive")
.header("Referer", "http://music.163.com/song?id=2332323")
.header("Origin", "http://music.163.com").header("Host", "music.163.com")
.header("Content-Type", "application/x-www-form-urlencoded")
.method(Method.POST).ignoreContentType(true).timeout(2000000000)
.execute();
String body = execute.body();
System.out.println(body);
} catch (IOException e) {
e.printStackTrace();
}
}
Example #25
| Source File: RequestPayload.java From crawler-jsoup-maven with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws IOException {
HashMap<String, String> headers = new HashMap<>();
String USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/71.0.3578.80 Chrome/71.0.3578.80 Safari/537.36";
JSONObject payload = new JSONObject();
// {"pageIndex":2,"pageSize":20,"typeTab":1}
payload.put("pageIndex", 2);
payload.put("pageSize", 20);
payload.put("typeTab", 1);
// String jsonBody = "{\"name\":\"ACTIVATE\",\"value\":\"E0010\"}";
Connection connection = Jsoup.connect("https://www.tablenow.vn/ho-chi-minh/bo-suu-tap")
.userAgent(USER_AGENT) // User-Agent of Chrome 55
.referrer("https://www.tablenow.vn/ho-chi-minh/bo-suu-tap")
.header("Content-Type", "application/json;charset=UTF-8")
.header("Accept", "application/json, text/plain, */*")
.header("Accept-Encoding", "gzip, deflate, br")
.header("Accept-Language", "zh-CN,zh;q=0.9,ja;q=0.8,en;q=0.7")
.header("Connection", "keep-alive")
.requestBody(payload.toJSONString())
.maxBodySize(100)
.timeout(1000 * 10)
.method(Connection.Method.POST);
Response response = connection.execute();
JSONObject payload1 = new JSONObject();
payload.put("username", "username");
payload.put("password", "password");
payload.put("auth_token", "authToken");
Connection.Response logon = Jsoup.connect("http://loginUrl.com/login/")
//.cookies(cookies)
.data("payload", payload.toString())
.headers(headers)
.method(Connection.Method.POST)
.userAgent(USER_AGENT)
.followRedirects(true)
.execute();
// upload file by jsoup
File file1 = new File("C:/dir/file1.txt");
FileInputStream fs1 = new FileInputStream(file1);
Response response1 = Jsoup.connect("http://bluetata.com/")
.data("text","Jsoup upload") // 绑定数据
.data("file1", "uploadTest.txt", fs1) // 上传文件
.method(Method.POST)
.execute();
}
Example #26
| Source File: GITHUBLoginApater.java From crawler-jsoup-maven with Apache License 2.0 | 4 votes |
|
/**
* @param userName 用户名
* @param pwd 密码
* @throws Exception
*/
public static void simulateLogin(String userName, String pwd) throws Exception {
/*
* 第一次请求
* grab login form page first
* 获取登陆提交的表单信息，及修改其提交data数据（login，password）
*/
// get the response, which we will post to the action URL(rs.cookies())
Connection con = Jsoup.connect(LOGIN_URL); // 获取connection
con.header(USER_AGENT, USER_AGENT_VALUE); // 配置模拟浏览器
Response rs = con.execute(); // 获取响应
Document d1 = Jsoup.parse(rs.body()); // 转换为Dom树
System.out.println(d1);
List<Element> eleList = d1.select("form"); // 获取提交form表单，可以通过查看页面源码代码得知
// 获取cooking和表单属性
// lets make data map containing all the parameters and its values found in the form
Map<String, String> datas = new HashMap<>();
// 01/24/2019 17:45 bluetata 更新 -------------------------------------------------------------- Start ----------
// GitHub多次改版更新，最新的提交request data为:
// authenticity_token ll0RJnG1f9XDAaN1DxnyTDzCs+YXweEZWel9kGkq8TvXH83HjCwPG048sJ/VVjDA94YmbF0qvUgcJx8/IKlP8Q==
// commit Sign+in
// login bluetata
// password password123
// utf8 ✓
for(int i = 0; i < eleList.size(); i++) {
for (Element e : eleList.get(i).getAllElements()) {
// 设置用户名
if (e.attr("name").equals("login")) {
e.attr("value", userName);
}
// 设置用户密码
if (e.attr("name").equals("password")) {
e.attr("value", pwd);
}
// 排除空值表单属性
if (e.attr("name").length() > 0) {
datas.put(e.attr("name"), e.attr("value"));
}
}
}
// 旧逻辑 delete 01/24/2019 17:49 bluetata --------------------------------------------start
// for (Element e : eleList.get(0).getAllElements()) {
// // 设置用户名
// if (e.attr("name").equals("login")) {
// e.attr("value", userName);
// }
// // 设置用户密码
// if (e.attr("name").equals("password")) {
// e.attr("value", pwd);
// }
// // 排除空值表单属性
// if (e.attr("name").length() > 0) {
// datas.put(e.attr("name"), e.attr("value"));
// }
// }
// 旧逻辑 delete 01/24/2019 17:49 bluetata --------------------------------------------end
// 01/24/2019 17:45 bluetata 更新 --------------------------------------------------------------- End -----------
/*
* 第二次请求，以post方式提交表单数据以及cookie信息
*/
Connection con2 = Jsoup.connect("https://github.com/session");
con2.header(USER_AGENT, USER_AGENT_VALUE);
// 设置cookie和post上面的map数据
Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST).data(datas).cookies(rs.cookies()).execute();
// 打印，登陆成功后的信息
System.out.println(login.body());
// 登陆成功后的cookie信息，可以保存到本地，以后登陆时，只需一次登陆即可
Map<String, String> map = login.cookies();
for (String s : map.keySet()) {
System.out.println(s + " : " + map.get(s));
}
}
Example #27
| Source File: CentralnicLoinApater.java From crawler-jsoup-maven with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
String url = "https://registrar-console.centralnic.com/dashboard/login";
String userAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36";
Response res = Jsoup.connect(url).userAgent(userAgent).timeout(1000 * 3).execute();
Document d1 = Jsoup.parse(res.body());
List<Element> et = d1.select("#login_form");// 获取form表单
String XSRF = null;
// 获取，cooking和表单属性，下面map存放post时的数据
Map<String, String> datas = new java.util.HashMap<>();
for (Element e : et.get(0).getAllElements()) {
if (e.attr("name").equals("regid")) {
e.attr("value", "*****");// 设置id
}
if (e.attr("name").equals("user")) {
e.attr("value", "*****");// 设置用户名
}
if (e.attr("name").equals("password")) {
e.attr("value", "*****"); // 设置用户密码
}
if (e.attr("name").equals("XSRF")) {
XSRF = e.attr("value");
}
if (e.attr("name").length() > 0) {// 排除空值表单属性
datas.put(e.attr("name"), e.attr("value"));
}
}
System.out.println(datas);
System.out.println("======================");
System.out.println(res.url());
System.out.println(res.statusCode());
Map<String, String> cookies = res.cookies();
cookies.put("remember_me", "H2454482%3Atoddhan");
cookies.put("lang", "en");
System.out.println(res.cookies());
/*
* 第二次请求，以post方式提交表单数据以及cookie信息
*/
Connection con2 = Jsoup.connect("https://registrar-console.centralnic.com/dashboard/login_target");
con2.header("User-Agent", userAgent);
// 设置cookie和post上面的map数据
Response login = con2.ignoreContentType(true).followRedirects(true).method(Method.POST)
.data(datas).cookies(cookies).referrer("https://registrar-console.centralnic.com/dashboard/login")
.header("host", "registrar-console.centralnic.com")
.execute();
Map<String, String> cookies2 = login.cookies();
cookies2.put("remember_me", "H2454482%3Atoddhan");
cookies2.put("lang", "en");
System.out.println(cookies2);
System.out.println(login.statusCode());
System.err.println(JsoupUtil.getDocumentWithCookies("https://registrar-console.centralnic.com/", cookies2));
System.out.println(JsoupUtil.getDocumentWithCookies("https://registrar-console.centralnic.com/graphs/file_browser/droplist", cookies2));
}
Example #28
| Source File: JDLoginApater.java From crawler-jsoup-maven with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) {
// grab login form page first
try {
//lets make data map containing all the parameters and its values found in the form
Map<String, String> mapParamsData = new HashMap<String, String>();
mapParamsData.put("loginType", "f");
mapParamsData.put("loginname", "18241141433");
mapParamsData.put("nloginpwd", "password");
mapParamsData.put("eid", "2QE5VJVZUBCRYD7LQZBJBTEFRNKPMQQA5OXKXNY7AAN4A3DKDTR7IN3GXHE5C6B4GTMW3Z53B4RGORB6YG5LUWF2UA");
mapParamsData.put("fp", "ae5baf289624644fced3f921c6a3792c");
mapParamsData.put("pubKey", "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDC7kw8r6tq43pwApYvkJ5laljaN9BZb21TAIfT/vexbobzH7Q8SUdP5uDPXEBKzOjx2L28y7Xs1d9v3tdPfKI2LR7PAzWBmDMn8riHrDDNpUpJnlAGUqJG9ooPn8j7YNpcxCa1iybOlc2kEhmJn5uwoanQq+CA6agNkqly2H4j6wIDAQAB");
mapParamsData.put("sa_token", "B68C442BE645754F33277E701208059080DD726A94A73F76DEC3053A838549C06EB7D3797CE1C5BBE7C2B2EF9CA7D4676F3D489984B517943EA13575FA80C7E73160F85EDB2705D145C52621D18B86C98B49AAF0DA97A2A7A964A78DDDA048AE592CF17C2AD8BF442AD743460B9316993DCDF5924AD8536FD4392C95A998E1C4C4EEDF76BF8FF03AAC145E449EAB889368EE1E7DA72B18881D527D9F51BAD9E2678DDEAFB33A647DD6D48B2A3BE1BC51DDC55AB1EAAEE2DE9A3CEA3702F93AAD1EC8EF740B632F5A4EC102498CDB31AF91CEA15DB3B6DF6FAC6CA31473ACC5E2CD727F80D2746F504A85379E7F3971086C13BA743F21731CEBFEC558E54E8D5D486CC3A19266E238F539A59C2F8630964981217DCC3B31B324F7DBF41FAEA47CA363904F06816BA9730B31BDD9FFA5498C1D0C36D67F315BA4F9236AC77BAFD5");
mapParamsData.put("seqSid", "5844668515208615000");
mapParamsData.put("uuid", "5653262a-5ef1-47c6-8ac2-427f519fdcfa");
Response loginResponse = Jsoup.connect(LOGIN_URI)
.userAgent(USER_AGENT)
.timeout(TIMEOUT_UNIT * TIMEOUT_TIMES)
.data(mapParamsData)
.method(Method.POST)
.followRedirects(true)
.execute();
System.out.println("Fetched login page");
// System.out.println(loginResponse.toString());
//get the cookies from the response, which we will post to the action URL
Map<String, String> mapLoginPageCookies = loginResponse.cookies();
System.out.println(mapLoginPageCookies);
Document doc = Jsoup.connect("http://order.jd.com/center/list.action")
.cookies(mapLoginPageCookies)
.timeout(30000)
.get();
System.out.println(doc.toString());
} catch (Exception e) {
e.printStackTrace();
}
}
Example #29
| Source File: PostSend2.java From crawler-jsoup-maven with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws Exception {
String url = "http://jzsc.mohurd.gov.cn/dataservice/query/comp/list";
Response res = Jsoup.connect(url).execute();
Map<String, String> cookies = res.cookies();
System.out.println(cookies);
Map<String, String> datas = new HashMap<>();
datas.put("$total", "297267");
datas.put("$reload", "1");
datas.put("$pg", "31&&1=1");
datas.put("$pgsz", "15");
Response res1 = Jsoup.connect(url).cookies(cookies).data(datas).method(Method.POST).execute();
String html = res1.body();
System.out.println(html);
// String jsonBody = "{pg:30,ps:15,tt:297267,pn:5,pc:19818,id:'',st:true}";
//
// Connection connection = Jsoup.connect("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")
// .userAgent("Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36") // User-Agent of Chrome 55
// .referrer("http://jzsc.mohurd.gov.cn/dataservice/query/comp/list")
// .header("Content-Type", "application/x-www-form-urlencoded")
// .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
// .header("Accept-Encoding", "gzip, deflate")
// .header("Accept-Language", "zh-CN,en-US;q=0.8,zh;q=0.6,ja;q=0.4,en;q=0.2")
// .header("Cookie", "filter_comp=; JSESSIONID=F53A68FBA45CCF750F9F32ACF090DC9B")
// .header("Host", "jzsc.mohurd.gov.cn")
// .header("Connection", "keep-alive")
// .header("X-Requested-With", "XMLHttpRequest")
// .header("Upgrade-Insecure-Requests", "1")
// .requestBody(jsonBody)
// .maxBodySize(100)
// .timeout(1000 * 10)
// .method(Connection.Method.POST);
//
// Response response = connection.execute();
//
// String body = response.body();
// System.out.println(body);
}
Example #30
| Source File: WebConnector.java From JavaSkype with MIT License | 4 votes |
|
public void block(User user) throws IOException {
sendRequest(Method.PUT, "/users/self/contacts/" + user.getUsername() + "/block", "reporterIp", "127.0.0.1");
}
