Java Code Examples for com.gargoylesoftware.htmlunit.WebClient

The following are top voted examples for showing how to use com.gargoylesoftware.htmlunit.WebClient. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ats-framework   File: HiddenHtmlPrompt.java   View source code 10 votes vote down vote up
public HiddenHtmlPrompt( UiDriver uiDriver ) {

        super(uiDriver);

        HiddenBrowserDriver browserDriver = (HiddenBrowserDriver) uiDriver;
        HtmlUnitDriver driver = (HtmlUnitDriver) browserDriver.getInternalObject(InternalObjectsEnum.WebDriver.name());
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {

            TargetLocator targetLocator = driver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            webClient = (WebClient) webClientField.get(targetLocator.defaultContent());

        } catch (Exception e) {

            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {

            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
 
Example 2
Project: duck-feed-2   File: DuckScrape.java   View source code 7 votes vote down vote up
public static List<String> searchDuck (String keyword) {
    List<String> searchResults = new ArrayList<>();
    try{
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        HtmlPage page = webClient.getPage("https://duckduckgo.com/html/?q=" + keyword);
        List<HtmlAnchor> l = page.getByXPath("//a[@class='result__url']");
        for(HtmlAnchor a: l) {
            searchResults.add(a.getHrefAttribute());
        }


    }
    catch(Exception e){
        System.err.println(e);
    }
    return searchResults;
}
 
Example 3
Project: Forum-Notifier   File: StatViewController.java   View source code 7 votes vote down vote up
private String getXenToken(final Account account) { // TODO: Re-write this if it ever gets used
    final WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);

    final HtmlPage page;
    final HtmlInput token;

    account.getCookies().forEach(c -> webClient.getCookieManager().addCookie(c));

    try {
        page = webClient.getPage(account.getForum().getProtocol() + "://" + account.getForum());
        token = page.getFirstByXPath("//*[@id='XenForo']/body/div[1]/aside[2]/div/div/div[1]/div[2]/form/div/input[2]");

        webClient.close();
        return token.getValueAttribute();
    } catch (Exception e) {
        e.printStackTrace();
    }
    webClient.close();
    return null;
}
 
Example 4
Project: ats-framework   File: HiddenHtmlAlert.java   View source code 6 votes vote down vote up
public HiddenHtmlAlert( UiDriver uiDriver ) {

        super(uiDriver);

        HiddenBrowserDriver browserDriver = (HiddenBrowserDriver) uiDriver;
        HtmlUnitDriver driver = (HtmlUnitDriver) browserDriver.getInternalObject(InternalObjectsEnum.WebDriver.name());
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {

            TargetLocator targetLocator = driver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            webClient = (WebClient) webClientField.get(targetLocator.defaultContent());
        } catch (Exception e) {

            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {

            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
 
Example 5
Project: ats-framework   File: HiddenHtmlConfirm.java   View source code 6 votes vote down vote up
public HiddenHtmlConfirm( UiDriver uiDriver ) {

        super(uiDriver);

        HiddenBrowserDriver browserDriver = (HiddenBrowserDriver) uiDriver;
        HtmlUnitDriver driver = (HtmlUnitDriver) browserDriver.getInternalObject(InternalObjectsEnum.WebDriver.name());
        Field webClientField = null;
        boolean fieldAccessibleState = false;
        try {

            TargetLocator targetLocator = driver.switchTo();
            webClientField = targetLocator.getClass().getDeclaringClass().getDeclaredField("webClient");
            fieldAccessibleState = webClientField.isAccessible();
            webClientField.setAccessible(true);
            webClient = (WebClient) webClientField.get(targetLocator.defaultContent());

        } catch (Exception e) {

            throw new SeleniumOperationException("Error retrieving internal Selenium web client", e);
        } finally {

            if (webClientField != null) {
                webClientField.setAccessible(fieldAccessibleState);
            }
        }
    }
 
Example 6
Project: pdfdbscrap   File: HtmlUnitExperiments.java   View source code 6 votes vote down vote up
/**
	 * Scraping base experiment. To be used as starting point.
	 *
	 * First things first:
	 * <ol>
	 * <li>
	 * Make sure you're able to retrieve the page. Try again with JS disabled if
	 * the client crashes.
	 * </li>
	 * <li>
	 * Make sure you're actually getting what you want, and that there are no
	 * redirect-, proxy-, or login shenanigans or what not. Have a look at the
	 * retrieved text.
	 * </li>
	 * <li>
	 * Also consider disabling CSS to speed things up a bit.
	 * </li>
	 * </ol>
	 */
	@Test
	public void scrapingBaseExperiment() {
		assertOutputDirectoryExists();

		String url = "http://www.google.com";

		try (WebClient client = new WebClient(BrowserVersion.CHROME)) {
//			client.getOptions().setCssEnabled(false);
//			client.getOptions().setJavaScriptEnabled(false);

			try {
				HtmlPage page = client.getPage(url);
				System.out.println("HtmlPage:");
				System.out.println(page.asText());
				System.out.print("\n");
			} catch (IOException ex) {
				System.err.println("WARNING: failed to visit: " + url);
				ex.printStackTrace(System.err);
			}
		}
	}
 
Example 7
Project: XC2   File: Server.java   View source code 6 votes vote down vote up
public Server()throws IOException{
    browser = new WebClient(BrowserVersion.CHROME);
    problems = new ArrayList<>();
    done = false;
    isSubmitting = false;
    user_name = "XC2";
    pass_word = "donthackme";
    work_list = new ArrayList<>();
    users_data = new HashMap<>();
    application = Executors.newCachedThreadPool();
    //the buffer can be 100 at max (may be adjusted if needed)
    buffer_queue = new LinkedBlockingQueue<>(100);
    
    
    //some browser intitialization to increase efficiency (the cookies part is essential)
    browser.getOptions().setUseInsecureSSL(true);
    browser.getOptions().setJavaScriptEnabled(false);
    browser.getOptions().setCssEnabled(false);
    browser.getOptions().setThrowExceptionOnScriptError(false);
    browser.getCookieManager().setCookiesEnabled(true);
    browser.setAjaxController(new NicelyResynchronizingAjaxController());
}
 
Example 8
Project: spring4-understanding   File: DelegatingWebConnectionTests.java   View source code 6 votes vote down vote up
@Test
public void verifyExampleInClassLevelJavadoc() throws Exception {
	Assume.group(TestGroup.PERFORMANCE);

	WebClient webClient = new WebClient();

	MockMvc mockMvc = MockMvcBuilders.standaloneSetup(TestController.class).build();
	MockMvcWebConnection mockConnection = new MockMvcWebConnection(mockMvc);
	mockConnection.setWebClient(webClient);

	WebRequestMatcher cdnMatcher = new UrlRegexRequestMatcher(".*?//code.jquery.com/.*");
	WebConnection httpConnection = new HttpWebConnection(webClient);
	WebConnection webConnection = new DelegatingWebConnection(mockConnection, new DelegateWebConnection(cdnMatcher, httpConnection));

	webClient.setWebConnection(webConnection);

	Page page = webClient.getPage("http://code.jquery.com/jquery-1.11.0.min.js");
	assertThat(page.getWebResponse().getStatusCode(), equalTo(200));
	assertThat(page.getWebResponse().getContentAsString(), not(isEmptyString()));
}
 
Example 9
Project: xxl-incubator   File: Demo.java   View source code 6 votes vote down vote up
public static void main(String[] args) throws IOException {

        // 浏览器
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setUseInsecureSSL(true);//支持https
        webClient.getOptions().setJavaScriptEnabled(true); // 启用JS解释器,默认为true
        webClient.getOptions().setCssEnabled(false); // 禁用css支持
        webClient.getOptions().setThrowExceptionOnScriptError(false); // js运行错误时,是否抛出异常
        webClient.getOptions().setTimeout(10000); // 设置连接超时时间 ,这里是10S。如果为0,则无限期等待
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.setJavaScriptTimeout(8000);//设置js运行超时时间
        webClient.waitForBackgroundJavaScript(500);//设置页面等待js响应时间,

        // proxy
        //webClient.getOptions().setProxyConfig(new ProxyConfig("IP", 80));

        HtmlPage page = webClient.getPage("http://---");
        String pageXml = page.asXml(); //以xml的形式获取响应文本
        System.out.println(pageXml);

    }
 
Example 10
Project: Forum-Notifier   File: LoginViewController.java   View source code 6 votes vote down vote up
public static Cookie completeCloudflareBrowserCheck(final String url) {
    WebClient completeClient = new WebClient(BrowserVersion.CHROME);
    completeClient.getOptions().setCssEnabled(false);
    completeClient.getOptions().setThrowExceptionOnFailingStatusCode(false);

    final HtmlPage page;
    final HtmlElement submitButton;
    final HtmlForm challengeForm;

    try {
        page = completeClient.getPage(url);
        completeClient.waitForBackgroundJavaScript(5000);

        submitButton = (HtmlElement) page.createElement("button");
        submitButton.setAttribute("type", "submit");

        challengeForm = (HtmlForm) page.getElementById("challenge-form");
        challengeForm.appendChild(submitButton);
        submitButton.click();

        return completeClient.getCookieManager().getCookie("cf_clearance");
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}
 
Example 11
Project: alimama   File: HtmlUnitUtil.java   View source code 6 votes vote down vote up
public static WebClient create(String host,int port) {
	LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
			"org.apache.commons.logging.impl.NoOpLog");
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(
			Level.OFF);
	java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(
			Level.OFF);

	// LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
	// java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);

	WebClient webClient = new WebClient(BrowserVersion.FIREFOX_17,host,port);
	webClient.getOptions().setUseInsecureSSL(true);
	webClient.getOptions().setJavaScriptEnabled(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	webClient.getOptions().setCssEnabled(true);
	webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	webClient.getOptions().setTimeout(60000);
	webClient.setJavaScriptTimeout(60000);
	webClient.waitForBackgroundJavaScript(120000);
	return webClient;
}
 
Example 12
Project: alimama   File: HtmlUnitUtil.java   View source code 6 votes vote down vote up
public static WebClient create() {
	LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
			"org.apache.commons.logging.impl.NoOpLog");
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(
			Level.OFF);
	java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(
			Level.OFF);

	// LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
	// java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);

	WebClient webClient = new WebClient(BrowserVersion.CHROME);
	webClient.getOptions().setUseInsecureSSL(true);
	webClient.getOptions().setJavaScriptEnabled(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	webClient.getOptions().setCssEnabled(true);
	webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	webClient.getOptions().setTimeout(60000);
	webClient.setJavaScriptTimeout(60000);
	webClient.waitForBackgroundJavaScript(120000);
	return webClient;
}
 
Example 13
Project: jscoverproxy-maven-plugin   File: JSCoverProxyMavenMojo.java   View source code 6 votes vote down vote up
private WebDriver getWebClient(int portForJSCoverProxy) {
    Proxy proxy = new Proxy().setHttpProxy("localhost:" + portForJSCoverProxy);
    DesiredCapabilities caps = new DesiredCapabilities();
    caps.setCapability(CapabilityType.PROXY, proxy);
    caps.setJavascriptEnabled(true);
    caps.setBrowserName(BrowserType.HTMLUNIT);
    return new HtmlUnitDriver(caps) {
        @Override
        protected WebClient modifyWebClient(WebClient client) {
            client.setScriptPreProcessor((htmlPage, sourceCode, sourceName, lineNumber, htmlElement) -> {
                if(removeJsSnippets != null && !removeJsSnippets.isEmpty()) {
                    for(String toReplace : removeJsSnippets) {
                        sourceCode = sourceCode.replace(toReplace, "");
                    }
                    return sourceCode;
                } else {
                    return sourceCode;
                }

            });
            return client;
        }
    };
}
 
Example 14
Project: mesfavoris   File: HtmlUnitAuthorizationCodeInstalledApp.java   View source code 6 votes vote down vote up
private HtmlPage allowAccess(WebClient webClient, HtmlPage allowAccessPage) throws IOException {
	HtmlButton allowAccessButton = (HtmlButton) allowAccessPage.getElementById("submit_approve_access");
	if (allowAccessButton == null) {
		throw new RuntimeException("Cannot find allow access button in html page :\n" + allowAccessPage.asXml());
	}
	webClient.waitForBackgroundJavaScriptStartingBefore(WAIT_DELAY_MS);
	// allowAccessButton.click() does not work because
	// allowAccessButton.isVisible() is false
	// for some reason (click() was working with htmlunit 2.23)
	HtmlPage tokenPage = clickButtonIgnoringVisibility(allowAccessButton);
	return tokenPage;
}
 
Example 15
Project: Mastering-Java-EE-Development-with-WildFly   File: ApplicationTestCase.java   View source code 5 votes vote down vote up
@Before
public void setup() throws IOException {
	webClient = new WebClient();

	DefaultCredentialsProvider creds = new DefaultCredentialsProvider();
	creds.addCredentials("root", "p1");
	webClient.setCredentialsProvider(creds);
	resourceBundle = getBundle("ResourceJSF");
}
 
Example 16
Project: MMDownloader   File: Downloader.java   View source code 5 votes vote down vote up
/**
 * HtmlUnit을 이용한 HTML 코드 파싱.
 * @param eachArchiveAddress 실제 만화가 담긴 아카이브 주소
 * @return 성공 시 html 코드를 리턴
 */
private String getHtmlPageHtmlUnit(String eachArchiveAddress) throws Exception {
	/* 필수! 로그 메세지 출력 안함 -> HtmlUnit 이용시 Verbose한 로그들이 너무 많아서 다 끔 */
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF); 
	System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
	
	System.out.print("일반 연결 시도중 ... ");
	
	WebClient webClient = new WebClient();
	webClient.getOptions().setRedirectEnabled(true);
	
	WebRequest req = new WebRequest(new URL(eachArchiveAddress));
	req.setHttpMethod(HttpMethod.POST);
	req.setAdditionalHeader("User-Agent", UserAgent.getUserAgent());
	req.setAdditionalHeader("Accept-Encoding", "gzip"); //20171126 gzip 추가
	req.getRequestParameters().add(new NameValuePair("pass", PASSWORD)); //비밀번호 post 방식 전송
	
	HtmlPage page = webClient.getPage(req);
	
	//Html코드를 포함한 페이지 소스코드가 담길 스트링
	String pageSource = page.asXml();
	
	/** 여기도 페이지 파싱 실패 시 검증하는 코드 들어가야 됨 **/
	
	webClient.close();
	System.out.println("성공");
	return pageSource;
}
 
Example 17
Project: qfii-tracker   File: Spider.java   View source code 5 votes vote down vote up
/**
 * Download pages from url with the specific date, and save to the directory
 *
 * @param directory Directory
 * @param queryDate Query Date
 * @param url Query URL
 */
protected void download(String directory, Date queryDate, String url) throws IOException {
  try (final WebClient webClient = getWebClient()) {
    // Get the query page
    final HtmlPage queryPage = webClient.getPage(url);

    // Get the date select list elements and the search button
    final HtmlSelect year = queryPage.getElementByName("ddlShareholdingYear");
    final HtmlSelect month = queryPage.getElementByName("ddlShareholdingMonth");
    final HtmlSelect day = queryPage.getElementByName("ddlShareholdingDay");
    final HtmlImageInput button = queryPage.getElementByName("btnSearch");

    // Change the value of the date select list
    year.setSelectedAttribute(DateFormatUtils.format(queryDate, "yyyy"), true);
    month.setSelectedAttribute(DateFormatUtils.format(queryDate, "MM"), true);
    day.setSelectedAttribute(DateFormatUtils.format(queryDate, "dd"), true);

    // Now submit the query by clicking the button and get back the result page
    final HtmlPage resultPage = (HtmlPage) button.click();
    File outputFile = new File(directory,
        DateFormatUtils.format(queryDate, "yyyyMMdd") + ".html");
    FileUtils.writeStringToFile(outputFile, resultPage.asXml(), "UTF-8");

    logger.info("Successfully downloaded HKEX web page for market {} and date {}",
        url.substring(url.length() - 2), DateFormatUtils.format(queryDate, "yyyy/MM/dd"));
  }
}
 
Example 18
Project: qfii-tracker   File: Spider.java   View source code 5 votes vote down vote up
protected WebClient getWebClient() {
  final WebClient webClient = new WebClient();
  webClient.getOptions().setJavaScriptEnabled(true);
  webClient.getOptions().setThrowExceptionOnFailingStatusCode(true);
  webClient.getOptions().setThrowExceptionOnScriptError(false);
  webClient.getOptions().setTimeout(10000);
  return webClient;
}
 
Example 19
Project: mvc-tck   File: SimpleAnnotatedTest.java   View source code 5 votes vote down vote up
@Test
public void shouldDoStuff() throws IOException {

    Page page = new WebClient()
            .getPage(baseUrl.toString() + "mvc/simple");

    Assert.assertThat(
            page.getWebResponse().getContentAsString(),
            CoreMatchers.containsString("Hello world!")
    );

}
 
Example 20
Project: Fav-Track   File: MPC.java   View source code 5 votes vote down vote up
private void getData(){
    MPC_host = new GetPropetries().getMPC_host();
    MPC_port = new GetPropetries().getMPC_port();
    try (final WebClient webClient = new WebClient()) {
        final HtmlPage page = webClient.getPage("http://"+MPC_host+":"+MPC_port+"/variables.html");
        filename = page.getElementById("file").getTextContent();
        position = parseInt((page.getElementById("position").getTextContent().substring(0,
                            page.getElementById("position").getTextContent().length()-3)));
        duration = parseInt((page.getElementById("duration").getTextContent().substring(0,
                page.getElementById("duration").getTextContent().length()-3)));
    } catch (IOException e){System.err.println(e); logger(logger,"error", e.getMessage());}
}
 
Example 21
Project: ProBOT   File: Bookie.java   View source code 5 votes vote down vote up
private HtmlPage navigateToPage( User user, String uri, boolean enableJS ) throws Exception, IOException, MalformedURLException
{
    final WebClient webClient = new WebClient();
    webClient.getOptions().setCssEnabled( false );
    webClient.getOptions().setJavaScriptEnabled( enableJS );
    addCredentials( user, webClient );

    String pageUrl = new StringBuilder( "http://" ).append( WEBSITE ).append( uri ).toString();
    HtmlPage page = webClient.getPage( pageUrl );
    return page;
}
 
Example 22
Project: ProBOT   File: Bookie.java   View source code 5 votes vote down vote up
public void cancelBooking( User user, Meeting meeting ) throws Exception
{
    final WebClient webClient = new WebClient();
    addCredentials( user, webClient );

    String uri = CANCEL_BOOKING + "/" + meeting.getMeetingId();
    String pageUrl = new StringBuilder( "http://" ).append( WEBSITE ).append( uri ).toString();

    URL url = new URL( pageUrl );
    WebRequest requestSettings = new WebRequest( url, HttpMethod.POST );

    Page redirectPage = webClient.getPage( requestSettings );
    logger.debug( "Confirmed Cancellation " + redirectPage.getWebResponse().getContentAsString() );
}
 
Example 23
Project: ProBOT   File: HtmlUnitTests.java   View source code 5 votes vote down vote up
@Test
public void bookRoom() throws Exception {
    try (final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_52)) {

        //set proxy username and password
        final DefaultCredentialsProvider credentialsProvider = (DefaultCredentialsProvider) webClient.getCredentialsProvider();
        credentialsProvider.addNTLMCredentials("abhisheks", "abhiProJul17", "apps.prorigo.com", 80, "", "");

        final HtmlPage page = webClient.getPage("http://apps.prorigo.com/conference/Booking");

        final HtmlForm form = page.getForms().get(0);
        final HtmlButton button = form.getFirstByXPath("//*[@id=\"Submit\"]");

        HtmlSelect select = (HtmlSelect) page.getElementById("ConferenceRooms");
        HtmlOption option = select.getOptionByValue("dedfcc1d-d413-e311-9c19-0025648b200e");
        select.setSelectedAttribute(option, true);

        HtmlInput inputStartTime = form.getInputByName("StartTime");
        inputStartTime.setValueAttribute("11:00 PM");

        HtmlInput inputEndTime = form.getInputByName("EndTime");
        inputEndTime.setValueAttribute("11:30 PM");

        HtmlInput inputReason = form.getInputByName("Title");
        inputReason.type("Test Booking By HtmlUnit");

        final HtmlPage page2 = button.click();
    }
}
 
Example 24
Project: crawler-jsoup-maven   File: htmlunitTest.java   View source code 5 votes vote down vote up
public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
    
    // 屏蔽HtmlUnit等系统 log
    LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
    java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
    java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(Level.OFF);
    
    String url = "https://www.douyin.com/share/video/6496703951436516621/?mid=6484356820260686606";
    System.out.println("Loading page now-----------------------------------------------: "+url);
    
    /* HtmlUnit 模拟浏览器 */
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setJavaScriptEnabled(true);              // 启用JS解释器,默认为true  
    webClient.getOptions().setCssEnabled(false);                    // 禁用css支持  
    webClient.getOptions().setThrowExceptionOnScriptError(false);   // js运行错误时,是否抛出异常
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setTimeout(10 * 1000);                   // 设置连接超时时间
    HtmlPage page = webClient.getPage(url);
    webClient.waitForBackgroundJavaScript(30 * 1000);               // 等待js后台执行30秒

    String pageAsXml = page.asXml();
    
    /* Jsoup解析处理 */
    // Document doc = Jsoup.parse(pageAsXml, "https://bluetata.com/");
    Document doc = Jsoup.parse(pageAsXml);  
    Elements pngs = doc.select("img[src$=.png]");                   // 获取所有图片元素集
    // 其他操作
    System.out.println(doc.toString());
}
 
Example 25
Project: poe-ladder-tracker-java   File: CSVLinkCreator.java   View source code 5 votes vote down vote up
/**
 * Starts the CSV link creation process.
 * 
 * @throws Exception If there was an error while getting the CSV specific information from the forum thread.
 */
public void create() throws Exception{
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);
	
	WebClient webClient = new WebClient(BrowserVersion.CHROME);
    HtmlPage page = (HtmlPage) webClient.getPage(forumThreadLink);
    webClient.waitForBackgroundJavaScriptStartingBefore(waitForJavaScript);
    HtmlAnchor anchor = (HtmlAnchor) page.getByXPath(xPath).get(0);
    csvFileLink = anchor.getHrefAttribute();
    csvFileLink = "http://www.pathofexile.com" + csvFileLink.subSequence(0, csvFileLink.length()-1);
    webClient.close();  
}
 
Example 26
Project: XC2   File: Server.java   View source code 5 votes vote down vote up
public Server()throws IOException{
    browser = new WebClient(BrowserVersion.CHROME);
    problems = new ArrayList<>();
    done = false;
    isSubmitting = false;
    user_name = "XC2";
    pass_word = "donthackme";
    application = Executors.newCachedThreadPool();
    //the buffer can be 100 at max (may be adjusted if needed)
    buffer_queue = new LinkedBlockingQueue<>(100);
    network = new Network_Server(this, this.port, this.max_clients);
    
    //some browser intitialization to increase efficiency (the cookies part is essential)
    browser.getOptions().setUseInsecureSSL(true);
    browser.getOptions().setJavaScriptEnabled(false);
    browser.getOptions().setCssEnabled(false);
    browser.getOptions().setThrowExceptionOnScriptError(false);
    browser.getCookieManager().setCookiesEnabled(true);
    browser.setAjaxController(new NicelyResynchronizingAjaxController());
}
 
Example 27
Project: apache-archiva   File: AbstractRepositoryServletProxiedTestCase.java   View source code 5 votes vote down vote up
protected void assertServerSetupCorrectly( RemoteRepoInfo remoteRepo )
    throws Exception
{

    WebClient client = newClient();
    int status = client.getPage( remoteRepo.url ).getWebResponse().getStatusCode();
    assertThat( status ).isEqualTo( HttpServletResponse.SC_OK );

}
 
Example 28
Project: apache-archiva   File: AbstractRepositoryServletTestCase.java   View source code 5 votes vote down vote up
protected static WebClient newClient()
{
    final WebClient webClient = new WebClient();
    webClient.getOptions().setJavaScriptEnabled( false );
    webClient.getOptions().setCssEnabled( false );
    webClient.getOptions().setAppletEnabled( false );
    webClient.getOptions().setThrowExceptionOnFailingStatusCode( false );
    webClient.setAjaxController( new NicelyResynchronizingAjaxController() );
    return webClient;
}
 
Example 29
Project: gecco-htmlunit   File: HtmlUnitDownloder.java   View source code 5 votes vote down vote up
public HtmlUnitDownloder() {
	this.webClient = new WebClient(BrowserVersion.CHROME);
	this.webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	this.webClient.getOptions().setThrowExceptionOnScriptError(false);
	this.webClient.getOptions().setRedirectEnabled(false);
	this.webClient.getOptions().setCssEnabled(false);
	this.webClient.setJavaScriptTimeout(1000);
	//this.webClient.setJavaScriptErrorListener(new GeccoJavaScriptErrorListener());
}
 
Example 30
Project: spring4-understanding   File: HtmlUnitRequestBuilder.java   View source code 5 votes vote down vote up
/**
 * Construct a new {@code HtmlUnitRequestBuilder}.
 * @param sessions a {@link Map} from session {@linkplain HttpSession#getId() IDs}
 * to currently managed {@link HttpSession} objects; never {@code null}
 * @param webClient the WebClient for retrieving cookies
 * @param webRequest the {@link WebRequest} to transform into a
 * {@link MockHttpServletRequest}; never {@code null}
 */
public HtmlUnitRequestBuilder(Map<String, MockHttpSession> sessions, WebClient webClient, WebRequest webRequest) {
	Assert.notNull(sessions, "Sessions Map must not be null");
	Assert.notNull(webClient, "WebClient must not be null");
	Assert.notNull(webRequest, "WebRequest must not be null");

	this.sessions = sessions;
	this.webClient = webClient;
	this.webRequest = webRequest;
}
 
Example 31
Project: spring4-understanding   File: MockMvcWebClientBuilderTests.java   View source code 5 votes vote down vote up
@Test
public void mockMvcSetupWithCustomWebClientDelegate() throws Exception {
	WebClient preconfiguredWebClient = new WebClient();
	this.webClient = mockMvcSetup(this.mockMvc).withDelegate(preconfiguredWebClient).build();

	assertMvcProcessed("http://localhost/test");
	Assume.group(TestGroup.PERFORMANCE, () -> assertDelegateProcessed("http://example.com/"));
}
 
Example 32
Project: JSearcher   File: PostDemo.java   View source code 5 votes vote down vote up
public PostDemo() {
	webClient = new WebClient(BrowserVersion.CHROME);  
	webClient.getOptions().setCssEnabled(false);
	webClient.getOptions().setJavaScriptEnabled(false);
	
	cookieMan = webClient.getCookieManager();
	cookieMan.setCookiesEnabled(true);
}
 
Example 33
Project: JSearcher   File: LoginSimulator.java   View source code 5 votes vote down vote up
public LoginSimulator() {
	webClient = new WebClient(BrowserVersion.CHROME);  
	webClient.getOptions().setCssEnabled(false);
	webClient.getOptions().setJavaScriptEnabled(false);
	cookieMan = webClient.getCookieManager();
	cookieMan.setCookiesEnabled(true);
}
 
Example 34
Project: JSearcher   File: PostDownloader.java   View source code 5 votes vote down vote up
public PostDownloader() {
	webClient = new WebClient(BrowserVersion.CHROME);  
	webClient.getOptions().setCssEnabled(false); // disable css loader
	webClient.getOptions().setJavaScriptEnabled(false); // disable js loader
	cookieMan = webClient.getCookieManager();
	cookieMan.setCookiesEnabled(true); // enable cookie
}
 
Example 35
Project: Fetchino   File: TempContext.java   View source code 5 votes vote down vote up
/**
 * {@inheritDoc}
 * Returns the {@link WebClient} of this context. If this context does not have a web client of its own, the parent's web client will be returned.
 */
@Override
public WebClient getWebClient()
{
	if(webClient != null)
		return webClient;
	else
		return parent.getWebClient();
}
 
Example 36
Project: Fetchino   File: Util.java   View source code 5 votes vote down vote up
/**
 * Creates a new web client.
 *
 * @return A new {@link WebClient} instance.
 */
public static WebClient createWebClient()
{
	WebClient webClient = new WebClient(BrowserVersion.BEST_SUPPORTED);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	return webClient;
}
 
Example 37
Project: ozark   File: ExceptionsIT.java   View source code 5 votes vote down vote up
@Before
public void setUp() {
    webUrl = System.getProperty("integration.url");
    webClient = new WebClient();
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setPrintContentOnFailingStatusCode(false);
}
 
Example 38
Project: ozark   File: CsrfIT.java   View source code 5 votes vote down vote up
@Before
public void setUp() {
    webUrl = System.getProperty("integration.url");
    if (webUrl == null) {
        webUrl = "http://localhost:8080/test-csrf-property/";
    }
    webClient = new WebClient();
    webClient.getOptions().setRedirectEnabled(true);        // enable redirect!
}
 
Example 39
Project: ozark   File: ViewAnnotationIT.java   View source code 5 votes vote down vote up
@Before
public void setUp() {
    webUrl = System.getProperty("integration.url");
    webClient = new WebClient();
    // we explicitly want to test some status codes
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setPrintContentOnFailingStatusCode(false);
}
 
Example 40
Project: spring-security-stateless   File: ApplicationTest.java   View source code 5 votes vote down vote up
@Before
public void setup() throws Exception {
    application = new TestApplication().start();
    base = "http://localhost:" + application.getPort();
    baseUrl = new URL(base);
    webClient = new WebClient();
}