Java Code Examples for com.gargoylesoftware.htmlunit.BrowserVersion

The following are top voted examples for showing how to use com.gargoylesoftware.htmlunit.BrowserVersion. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: duck-feed-2   File: DuckScrape.java   View source code 8 votes vote down vote up
public static List<String> searchDuck (String keyword) {
    List<String> searchResults = new ArrayList<>();
    try{
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        HtmlPage page = webClient.getPage("https://duckduckgo.com/html/?q=" + keyword);
        List<HtmlAnchor> l = page.getByXPath("//a[@class='result__url']");
        for(HtmlAnchor a: l) {
            searchResults.add(a.getHrefAttribute());
        }


    }
    catch(Exception e){
        System.err.println(e);
    }
    return searchResults;
}
 
Example 2
Project: Forum-Notifier   File: StatViewController.java   View source code 7 votes vote down vote up
private String getXenToken(final Account account) { // TODO: Re-write this if it ever gets used
    final WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setCssEnabled(false);
    webClient.getOptions().setJavaScriptEnabled(false);

    final HtmlPage page;
    final HtmlInput token;

    account.getCookies().forEach(c -> webClient.getCookieManager().addCookie(c));

    try {
        page = webClient.getPage(account.getForum().getProtocol() + "://" + account.getForum());
        token = page.getFirstByXPath("//*[@id='XenForo']/body/div[1]/aside[2]/div/div/div[1]/div[2]/form/div/input[2]");

        webClient.close();
        return token.getValueAttribute();
    } catch (Exception e) {
        e.printStackTrace();
    }
    webClient.close();
    return null;
}
 
Example 3
Project: pdfdbscrap   File: HtmlUnitExperiments.java   View source code 6 votes vote down vote up
/**
	 * Scraping base experiment. To be used as starting point.
	 *
	 * First things first:
	 * <ol>
	 * <li>
	 * Make sure you're able to retrieve the page. Try again with JS disabled if
	 * the client crashes.
	 * </li>
	 * <li>
	 * Make sure you're actually getting what you want, and that there are no
	 * redirect-, proxy-, or login shenanigans or what not. Have a look at the
	 * retrieved text.
	 * </li>
	 * <li>
	 * Also consider disabling CSS to speed things up a bit.
	 * </li>
	 * </ol>
	 */
	@Test
	public void scrapingBaseExperiment() {
		assertOutputDirectoryExists();

		String url = "http://www.google.com";

		try (WebClient client = new WebClient(BrowserVersion.CHROME)) {
//			client.getOptions().setCssEnabled(false);
//			client.getOptions().setJavaScriptEnabled(false);

			try {
				HtmlPage page = client.getPage(url);
				System.out.println("HtmlPage:");
				System.out.println(page.asText());
				System.out.print("\n");
			} catch (IOException ex) {
				System.err.println("WARNING: failed to visit: " + url);
				ex.printStackTrace(System.err);
			}
		}
	}
 
Example 4
Project: XC2   File: Server.java   View source code 6 votes vote down vote up
public Server()throws IOException{
    browser = new WebClient(BrowserVersion.CHROME);
    problems = new ArrayList<>();
    done = false;
    isSubmitting = false;
    user_name = "XC2";
    pass_word = "donthackme";
    work_list = new ArrayList<>();
    users_data = new HashMap<>();
    application = Executors.newCachedThreadPool();
    //the buffer can be 100 at max (may be adjusted if needed)
    buffer_queue = new LinkedBlockingQueue<>(100);
    
    
    //some browser intitialization to increase efficiency (the cookies part is essential)
    browser.getOptions().setUseInsecureSSL(true);
    browser.getOptions().setJavaScriptEnabled(false);
    browser.getOptions().setCssEnabled(false);
    browser.getOptions().setThrowExceptionOnScriptError(false);
    browser.getCookieManager().setCookiesEnabled(true);
    browser.setAjaxController(new NicelyResynchronizingAjaxController());
}
 
Example 5
Project: xxl-incubator   File: Demo.java   View source code 6 votes vote down vote up
public static void main(String[] args) throws IOException {

        // 浏览器
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        webClient.getOptions().setUseInsecureSSL(true);//支持https
        webClient.getOptions().setJavaScriptEnabled(true); // 启用JS解释器,默认为true
        webClient.getOptions().setCssEnabled(false); // 禁用css支持
        webClient.getOptions().setThrowExceptionOnScriptError(false); // js运行错误时,是否抛出异常
        webClient.getOptions().setTimeout(10000); // 设置连接超时时间 ,这里是10S。如果为0,则无限期等待
        webClient.getOptions().setDoNotTrackEnabled(false);
        webClient.setJavaScriptTimeout(8000);//设置js运行超时时间
        webClient.waitForBackgroundJavaScript(500);//设置页面等待js响应时间,

        // proxy
        //webClient.getOptions().setProxyConfig(new ProxyConfig("IP", 80));

        HtmlPage page = webClient.getPage("http://---");
        String pageXml = page.asXml(); //以xml的形式获取响应文本
        System.out.println(pageXml);

    }
 
Example 6
Project: Forum-Notifier   File: LoginViewController.java   View source code 6 votes vote down vote up
public static Cookie completeCloudflareBrowserCheck(final String url) {
    WebClient completeClient = new WebClient(BrowserVersion.CHROME);
    completeClient.getOptions().setCssEnabled(false);
    completeClient.getOptions().setThrowExceptionOnFailingStatusCode(false);

    final HtmlPage page;
    final HtmlElement submitButton;
    final HtmlForm challengeForm;

    try {
        page = completeClient.getPage(url);
        completeClient.waitForBackgroundJavaScript(5000);

        submitButton = (HtmlElement) page.createElement("button");
        submitButton.setAttribute("type", "submit");

        challengeForm = (HtmlForm) page.getElementById("challenge-form");
        challengeForm.appendChild(submitButton);
        submitButton.click();

        return completeClient.getCookieManager().getCookie("cf_clearance");
    } catch (Exception e) {
        e.printStackTrace();
        return null;
    }
}
 
Example 7
Project: alimama   File: HtmlUnitUtil.java   View source code 6 votes vote down vote up
public static WebClient create(String host,int port) {
	LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
			"org.apache.commons.logging.impl.NoOpLog");
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(
			Level.OFF);
	java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(
			Level.OFF);

	// LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
	// java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);

	WebClient webClient = new WebClient(BrowserVersion.FIREFOX_17,host,port);
	webClient.getOptions().setUseInsecureSSL(true);
	webClient.getOptions().setJavaScriptEnabled(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	webClient.getOptions().setCssEnabled(true);
	webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	webClient.getOptions().setTimeout(60000);
	webClient.setJavaScriptTimeout(60000);
	webClient.waitForBackgroundJavaScript(120000);
	return webClient;
}
 
Example 8
Project: alimama   File: HtmlUnitUtil.java   View source code 6 votes vote down vote up
public static WebClient create() {
	LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",
			"org.apache.commons.logging.impl.NoOpLog");
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(
			Level.OFF);
	java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(
			Level.OFF);

	// LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
	// java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);

	WebClient webClient = new WebClient(BrowserVersion.CHROME);
	webClient.getOptions().setUseInsecureSSL(true);
	webClient.getOptions().setJavaScriptEnabled(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	webClient.getOptions().setCssEnabled(true);
	webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	webClient.getOptions().setTimeout(60000);
	webClient.setJavaScriptTimeout(60000);
	webClient.waitForBackgroundJavaScript(120000);
	return webClient;
}
 
Example 9
Project: awesome-agile   File: AwesomeAgileFunctionalTest.java   View source code 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  System.out.println("Fake Google OAuth2 server up at: " + fakeGoogleServer.getEndpoint());
  System.out.println("Fake Hackpad server up at: " + fakeHackpadServer.getEndpoint());
  System.out.println("AwesomeAgile web application up at: " + getEndpoint());
  fakeGoogleServer.setClientId(CLIENT_ID);
  fakeGoogleServer.setClientSecret(CLIENT_SECRET);
  fakeGoogleServer.setRedirectUriPrefixes(
      ImmutableList.of("http://localhost:" + port + "/"));
  fakeGoogleServer.setPerson(createUser());
  fakeHackpadServer.setClientId(HACKPAD_CLIENT_ID);
  fakeHackpadServer.setClientSecret(HACKPAD_CLIENT_SECRET);
  fakeHackpadServer.getHackpads().clear();
  fakeHackpadServer.addHackpad(
      new PadIdentity(DEFINITION_OF_READY_TEMPLATE_ID),
      DEFINITION_OF_READY_CONTENTS);
  fakeHackpadServer.addHackpad(
      new PadIdentity(DEFINITION_OF_DONE_TEMPLATE_ID),
      DEFINITION_OF_DONE_CONTENTS);

  driver = new HtmlUnitDriver(BrowserVersion.CHROME);
  driver.setJavascriptEnabled(true);
}
 
Example 10
Project: awesome-agile   File: AwesomeAgileFunctionalTest.java   View source code 6 votes vote down vote up
@Test
public void testDashboardReopenBrowser() throws Exception {
  LandingPage landingPage = PageFactory.initElements(driver, LandingPage.class);
  landingPage.loginWithGoogle(getEndpoint());
  assertThat(driver.getWindowHandles(), hasSize(1));
  landingPage.createDefinitionOfReady();

  HtmlUnitDriver driverTwo = new HtmlUnitDriver(BrowserVersion.CHROME);
  driverTwo.setJavascriptEnabled(true);

  // Open a completely new browser with no cookies
  // Verify that view button is visible for this same user,
  // and we're able to open his Definition of ready hackpad
  LandingPage landingPageTwo = PageFactory.initElements(driverTwo, LandingPage.class);
  landingPageTwo.loginWithGoogle(getEndpoint());
  landingPageTwo.waitForDefinitionOfReady();
  assertTrue(landingPageTwo.isDefinitionOfReadyViewable());
  landingPageTwo.viewDefinitionOfReady();
  String newWindow = Iterables.getFirst(Sets.difference(
      driverTwo.getWindowHandles(),
      ImmutableSet.of(driverTwo.getWindowHandle())), null);
  driverTwo.switchTo().window(newWindow);
  HackpadPage hackpadPage = PageFactory.initElements(driverTwo, HackpadPage.class);
  assertEquals(DEFINITION_OF_READY_CONTENTS, hackpadPage.getContent());
}
 
Example 11
Project: oiosaml.java   File: LoginHandlerTest.java   View source code 6 votes vote down vote up
@Test
public void testPost() throws Exception {
	cfg.setProperty(WSFedConstants.PROP_USE_REDIRECT, false);

	StringWriter sw = new StringWriter();
	when(res.getWriter()).thenReturn(new PrintWriter(sw));
	
	LoginHandler lh = new LoginHandler();
	lh.handleGet(rc);
	
	WebWindow win = mock(WebWindow.class);
	when(win.getScriptObject()).thenThrow(new RuntimeException("test"));
	when(win.getWebClient()).thenReturn(new WebClient(BrowserVersion.FIREFOX_2));
	
	DOMParser parser = new DOMParser();
	parser.parse(new InputSource(new ByteArrayInputStream(sw.toString().getBytes())));
	HTMLElement e = (HTMLElement) parser.getDocument().getDocumentElement();
	
	NodeList forms = e.getElementsByTagName("form");
	assertEquals(1, forms.getLength());
	Element form = (Element)forms.item(0);
	assertEquals("loginform", form.getAttribute("name"));
	assertEquals(rc.getIdpMetadata().getFirstMetadata().getSingleSignonServiceLocation(WSFedConstants.WSFED_PROTOCOL), form.getAttribute("action"));
	
	verify(res, never()).sendRedirect(anyString());
}
 
Example 12
Project: webmonitor   File: Configuration.java   View source code 6 votes vote down vote up
@SuppressWarnings("deprecation")
public BrowserVersion getBrowserEnum() {
       if (browserName.equals("Mozilla Firefox 3.6")) {
           return BrowserVersion.FIREFOX_3_6;
       }
       else if (browserName.equals("Mozilla Firefox 10")) {
           return BrowserVersion.FIREFOX_10;
       }
       else if (browserName.equals("Mozilla Firefox 17")) {
           return BrowserVersion.FIREFOX_17;
       }
       else if (browserName.equals("IE 6")) {
           return BrowserVersion.INTERNET_EXPLORER_6;
       } else if (browserName.equals("IE 7")) {
           return BrowserVersion.INTERNET_EXPLORER_7;
       } else if (browserName.equals("IE 8")) {
           return BrowserVersion.INTERNET_EXPLORER_8;
       }
       else if (browserName.equals("IE 9")) {
           return BrowserVersion.INTERNET_EXPLORER_9;
       }
       else if (browserName.equals("IE 10")) {
           return BrowserVersion.INTERNET_EXPLORER_10;
       }        
       return BrowserVersion.getDefault();
   }
 
Example 13
Project: tool.accelerate.core   File: PageFunctionTest.java   View source code 6 votes vote down vote up
@Ignore
@Test
// TODO: This method of testing does not work for angular, need to find an alternative method of testing
public void techFormTest() {
    final WebClient webClient = new WebClient(BrowserVersion.CHROME);
    HtmlPage page;
    String port = System.getProperty("liberty.test.port");
    try {
        page = webClient.getPage("http://localhost:" + port + "/start/");
        DomElement techForm = page.getElementById("techTable");
        DomElement formBody = techForm.getFirstElementChild();
        int count = formBody.getChildElementCount();
        // We expect there to be more than one child element, otherwise the 
        // javascript has not created the tech table properly.
        assertTrue("Expected more than one element in the tech table, instead found " + count, count > 1);
    } catch (Exception e){
        org.junit.Assert.fail("Caught exception: " + e.getCause().toString());
    } finally {
        webClient.close();
    }
}
 
Example 14
Project: Project-D2L   File: D2LHook.java   View source code 6 votes vote down vote up
public List<LockerObject> getLocker(boolean previewLink) {
	List<LockerObject> list = new ArrayList<>();
	try {
		WebClient webClient = new WebClient(BrowserVersion.FIREFOX_38);
		webClient.setCookieManager(cookieManager);

		HtmlPage page = webClient.getPage("https://pdsb.elearningontario.ca/d2l/lms/locker/locker.d2l?ou=8340");

		for (Object object : page.getByXPath("//a")) {
			DomElement de = (DomElement) object;
			if (de.asXml().toString().contains("/d2l/common/viewFile.d2lfile/Database/")) {
				if (!previewLink) {
					list.add(new LockerObject(de.getAttribute("title").replace("Open ", ""),  "https://pdsb.elearningontario.ca" + de.getAttribute("href").replace("&display=1", "")));
				} else {
					list.add(new LockerObject(de.getAttribute("title").replace("Open ", ""),  "https://pdsb.elearningontario.ca" + de.getAttribute("href")));

				}
			}
		}

		webClient.close();
	} catch (Exception e) {
		e.printStackTrace();
	}
	return list;
}
 
Example 15
Project: Project-D2L   File: D2LHook.java   View source code 6 votes vote down vote up
public List<NotificationObject> getNotifications(String ID) throws InvaildCourseException {
	List<NotificationObject> list = new ArrayList<>();
	try {
		WebClient webClient = new WebClient(BrowserVersion.FIREFOX_38);
		webClient.setCookieManager(cookieManager);

		UnexpectedPage page = webClient.getPage("https://pdsb.elearningontario.ca/d2l/MiniBar/" + ID + "/ActivityFeed/GetAlerts?Category=1&_d2l_prc%24headingLevel=2&_d2l_prc%24scope=&_d2l_prc%24hasActiveForm=false&isXhr=true&requestId=2");
		NotificationFormater notificationFormater = new NotificationFormater(page.getWebResponse().getContentAsString());
		
		webClient.close();
		
		for (Object object: notificationFormater.getNotifications()) {
			DomElement de = (DomElement) object;
			String[] split = de.asText().split("\\n");
			
			list.add(new NotificationObject(split[0].trim(), split[1].trim(), split[2].trim()));
		}
	} catch (Exception e) {
		throw new InvaildCourseException("Invaild Course ID");
	}
	
	return list;
}
 
Example 16
Project: Leboncoin   File: Client.java   View source code 6 votes vote down vote up
private static WebClient getLeboncoin()
{
    final WebClient client = new WebClient(BrowserVersion.FIREFOX_17);
    try
    {
        client.getOptions().setJavaScriptEnabled(true);
        client.getOptions().setAppletEnabled(false);
        client.getOptions().setCssEnabled(false);
        client.getOptions().setPrintContentOnFailingStatusCode(true);
        client.getOptions().setPopupBlockerEnabled(true);
        client.getOptions().setThrowExceptionOnScriptError(false);
        client.getOptions().setUseInsecureSSL(true);
    }
    catch (Exception e)
    {
        Logger.traceERROR(e);
    }

    return client;
}
 
Example 17
Project: reviki   File: WebTestSupport.java   View source code 6 votes vote down vote up
private WebClient setupClient(final String username, final String password) {
  final WebClient client = new WebClient(BrowserVersion.FIREFOX_24);
  final WebClientOptions options = client.getOptions();
  DefaultCredentialsProvider credentials = new DefaultCredentialsProvider();
  credentials.addCredentials(username, password);
  client.setCredentialsProvider(credentials);
  options.setRedirectEnabled(true);
  options.setThrowExceptionOnFailingStatusCode(true);
  client.addWebWindowListener(new ValidateOnContentChange());
  client.getCookieManager().setCookiesEnabled(false);

  // Try to log only "interesting" things:
  // Don't log errors we can't fix due to browser bugs etc.
  client.setIncorrectnessListener(new SuppressingIncorrectnessListener());

  return client;
}
 
Example 18
Project: gwt-sl   File: BaseIntegrationTest.java   View source code 6 votes vote down vote up
@Before
public void setupBrowser() {
	webClient = new WebClient(BrowserVersion.CHROME);
	webClient.getOptions().setCssEnabled(true);
	webClient.setCssErrorHandler(new SilentCssErrorHandler());
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(true);
	webClient.getOptions().setThrowExceptionOnScriptError(true);
	webClient.getOptions().setRedirectEnabled(true);
	webClient.getOptions().setAppletEnabled(false);
	webClient.getOptions().setJavaScriptEnabled(true);
	webClient.getOptions().setPopupBlockerEnabled(true);
	webClient.getOptions().setTimeout(TIMEOUT);
	webClient.getOptions().setPrintContentOnFailingStatusCode(false);
	webClient.setAjaxController(new NicelyResynchronizingAjaxController());
	webClient.setAlertHandler(new AlertHandler() {

		public void handleAlert(Page page, String message) {
			System.err.println("[alert] " + message);
		}

	});
	webClient.waitForBackgroundJavaScript(TIMEOUT);
}
 
Example 19
Project: MALSignatureDesigner   File: Http.java   View source code 6 votes vote down vote up
/**
 * Returns the Url of image for the animeId provided. The file type extension is not included.
 * @param animeId
 * @return null if not found
 * @throws FailingHttpStatusCodeException
 * @throws MalformedURLException
 * @throws IOException
 */
public static String getAnimeImageUrl(int animeId) throws FailingHttpStatusCodeException, MalformedURLException, IOException{
	String theReturn = null;
	final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
	webClient.getOptions().setThrowExceptionOnScriptError(false);//Will give alot of errors
	new IgnoreRandomJScripts(webClient);
	java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit").setLevel(java.util.logging.Level.SEVERE);//to ignore those errors
	try{
		Page page = webClient.getPage("http://myanimelist.net/anime.php?id="+animeId);
		if(page.isHtmlPage()){
			Pattern p = Pattern.compile("http://cdn.myanimelist.net/images/anime/([0-9]+)/([0-9]+).jpg");
			Matcher m = p.matcher(page.getWebResponse().getContentAsString());
			if(m.find()){
				if(m.groupCount() == 2){
					theReturn = "http://cdn.myanimelist.net/images/anime/"+m.group(1)+"/"+m.group(2);
				}
			}
		}
	}
	catch(com.gargoylesoftware.htmlunit.ScriptException e){}
	webClient.closeAllWindows();
	java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit").setLevel(java.util.logging.Level.ALL);
	return theReturn;
}
 
Example 20
Project: MALSignatureDesigner   File: Http.java   View source code 6 votes vote down vote up
/**
 * Downloads and saves Jpeg from url and stores as file
 * @param url
 * @param saveLoc
 * @return true on success
 * @throws FailingHttpStatusCodeException
 * @throws MalformedURLException
 * @throws IOException
 */
public static boolean downloadImage(String url, String saveLoc) throws FailingHttpStatusCodeException, MalformedURLException, IOException{
	if(url != null && saveLoc != null){
		if(!url.isEmpty() && !saveLoc.isEmpty()){
			final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_24);
			Page page = webClient.getPage(url);
			webClient.closeAllWindows();
			if(page.getWebResponse().getContentType().equals("image/jpeg")){//grabbed the xml file
				InputStream in = page.getWebResponse().getContentAsStream();
				OutputStream out; out = new FileOutputStream(new File(System.getProperty("user.dir") + saveLoc));
				IOUtils.copy(in,out);
				return true;
			}
		}
	}
	return false;
}
 
Example 21
Project: RoMessenger   File: LoginManager.java   View source code 6 votes vote down vote up
public static void removeUser(String user) {
	Link.manager.closeAllWindows();
	Link.manager = new WebClient(BrowserVersion.FIREFOX_3_6);
	Link.manager.setCssEnabled(false);
	Link.manager.setJavaScriptEnabled(true);
	Link.manager.setRedirectEnabled(true);
	Link.manager.setThrowExceptionOnFailingStatusCode(false);
	Link.manager.setThrowExceptionOnScriptError(false);
	Link.manager.setPrintContentOnFailingStatusCode(false);

	loggedInToDesktop = false;
	loggedInToMobile = false;

	userLoggingIn = "";
	userPass = "";

	currentUser = "";

	try {
		Link.mobileManager.getPage("http://m.roblox.com/Account/LogOff");
	} catch (Exception e) {
		e.printStackTrace();
	}

	users.remove(user);
}
 
Example 22
Project: RoMessenger   File: LoginManager.java   View source code 6 votes vote down vote up
public static void changeUser(String user) {
	if (users.containsKey(user)) {
		userLoggingIn = user;
		userPass = users.get(user);

		Link.manager.closeAllWindows();
		Link.manager = new WebClient(BrowserVersion.FIREFOX_3_6);
		Link.manager.setCssEnabled(false);
		Link.manager.setJavaScriptEnabled(true);
		Link.manager.setRedirectEnabled(true);
		Link.manager.setThrowExceptionOnFailingStatusCode(false);
		Link.manager.setThrowExceptionOnScriptError(false);
		Link.manager.setPrintContentOnFailingStatusCode(false);
		loggedInToDesktop = false;
		loggedInToMobile = false;

		loginToRobloxMobile(user, users.get(user));
	}
	System.gc();
}
 
Example 23
Project: datasponge   File: SpiderThread.java   View source code 6 votes vote down vote up
/**
 * initializes the WebClient object that will be used to fetch and parse web
 * pages
 *
 * @return new instance of WebClient that can be used to load and parse
 * pages
 */
private WebClient initializeWebClient(boolean minimal) {

    WebClient client = null;
    if (proxy != null && !proxy.trim().isEmpty()) {
        client = new WebClient(BrowserVersion.CHROME, proxy, port);
    } else {
        client = new WebClient(BrowserVersion.CHROME);
    }
    if (minimal) {
        client.getOptions().setAppletEnabled(false);
        client.getOptions().setJavaScriptEnabled(false);
        client.getOptions().setCssEnabled(false);
    }
    return client;
}
 
Example 24
Project: jfunk   File: JFunkHtmlUnitDriverImpl.java   View source code 6 votes vote down vote up
protected JFunkHtmlUnitDriverImpl(final BrowserVersion browserVersion, final HtmlUnitWebDriverParams webDriverParams,
		final AjaxController ajaxController, final HtmlUnitSSLParams sslParams,
		final Map<String, CredentialsProvider> credentialsProviderMap,
		final Provider<DumpFileCreator> htmlFileCreatorProvider, final Provider<File> moduleArchiveDirProvider,
		final Provider<Set<WebWindowListener>> listenersProvider, final Proxy proxy) {

	super(browserVersion);

	this.webDriverParams = webDriverParams;
	this.sslParams = sslParams;
	this.ajaxController = ajaxController;
	this.credentialsProviderMap = credentialsProviderMap;
	this.htmlFileCreatorProvider = htmlFileCreatorProvider;
	this.moduleArchiveDirProvider = moduleArchiveDirProvider;
	this.listenersProvider = listenersProvider;

	setProxy(proxy);
	setJavascriptEnabled(webDriverParams.isJavascriptEnabled());

	// cannot override modifyWebClient because it is called in the super class' constructor before our params are set
	configureWebClient(getWebClient());
}
 
Example 25
Project: jfunk   File: HtmlUnitDriverProvider.java   View source code 6 votes vote down vote up
@Inject
protected HtmlUnitDriverProvider(final Configuration config, final Set<WebDriverEventListener> eventListeners,
		final HtmlUnitWebDriverParams webDriverParams, final HtmlUnitSSLParams sslParams,
		final BrowserVersion browserVersion, final AjaxController ajaxController,
		final Map<String, CredentialsProvider> credentialsProviderMap,
		final Provider<DumpFileCreator> htmlFileCreatorProvider,
		@ModuleArchiveDir final Provider<File> moduleArchiveDirProvider,
		final Provider<Set<WebWindowListener>> listenersProvider, final Map<String, DesiredCapabilities> capabilitiesMap,
		final EventBus eventBus) {
	super(config, eventListeners, capabilitiesMap, eventBus);
	this.webDriverParams = webDriverParams;
	this.sslParams = sslParams;
	this.browserVersion = browserVersion;
	this.ajaxController = ajaxController;
	this.credentialsProviderMap = credentialsProviderMap;
	this.htmlFileCreatorProvider = htmlFileCreatorProvider;
	this.moduleArchiveDirProvider = moduleArchiveDirProvider;
	this.listenersProvider = listenersProvider;
}
 
Example 26
Project: wikidata   File: BasicCrawler.java   View source code 6 votes vote down vote up
/**
 * Makes a new WebClient object and gives it the default values.
 * The Newspapers1 program needs to call this every once in a while to clean up
 * the lingering connections. Without this the connections will stay in a 
 * CLOSE_WAIT state.
 */
public void reset() {
	webClient = new WebClient(BrowserVersion.INTERNET_EXPLORER_7);
	// There are a lot of broken JavaScripts so disabling it saves some errors:
    webClient.setJavaScriptEnabled(false);
    webClient.setTimeout(18000); // Note: the timeout seems to be in ms
    
    // java.lang.RuntimeException: Refresh Aborted by HtmlUnit: Attempted to refresh a page
	// using an ImmediateRefreshHandler which could have caused an OutOfMemoryError Please use
	// WaitingRefreshHandler or ThreadedRefreshHandler instead.
	// Caused by: <META HTTP-EQUIV="Refresh" CONTENT="3300; URL=http://newslink.org">
	// A ThreadedRefreshHandler avoids the above exception, but introduces the new problem
    // that the connections will stay open since they stay in a Threads that don't seem to
    // stop until the end of the program.
    // The solution seems to be to use a ThreadedRefreshHandler except for in certain cases.
    // Newspapers1 and Newspapers3 use setNullRefresh() and setThreadRefresh() to switch
    // between those RefreshHandlers.
    webClient.setRefreshHandler(new ThreadedRefreshHandler());
    
    System.runFinalization();
    System.gc();
}
 
Example 27
Project: jClipCorn   File: HTTPUtilities.java   View source code 6 votes vote down vote up
public static String getJavascriptHTML(String urlToRead, int jsTimeout) {
	//Otherwise we will get flooded with log warnings -.-
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);

	try (WebClient webClient = new WebClient(BrowserVersion.FIREFOX_38)) {
		webClient.getOptions().setJavaScriptEnabled(true);
		webClient.getOptions().setThrowExceptionOnFailingStatusCode(true);
		webClient.getOptions().setThrowExceptionOnScriptError(false);

		HtmlPage page = webClient.getPage(urlToRead);

		webClient.waitForBackgroundJavaScript(jsTimeout);

		return page.asXml();
	} catch (FailingHttpStatusCodeException | IOException e) {
		CCLog.addError(LocaleBundle.getFormattedString("LogMessage.CouldNotGetHTML", urlToRead), e);
		return "";
	}
}
 
Example 28
Project: Crawler_Self_Analysis   File: ApiUse.java   View source code 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	//ģ��chorme�������������������޸�BrowserVersion.����
	WebClient  webClient=new WebClient(BrowserVersion.CHROME);
	webClient.getOptions().setCssEnabled(false);
	webClient.getOptions().setJavaScriptEnabled(false);
	HtmlPage page=webClient.getPage("http://www.yanyulin.info");
	//�������ָ��������ϻ�ȡ��ǩhed������
	//HtmlDivision div=(HtmlDivision)page.getElementById("hed");
	//System.out.println(div.asXml());
	//ͬ�����Դ�ӡ��hed������,//div��//��ʾ���������ĵ��е�div,������Щdiv
	//����list�У�Ȼ���ȡ��һ��div
	//final HtmlDivision div = (HtmlDivision) page.getByXPath("//div").get(0);
	//System.out.println(div.asXml());
	java.util.List<HtmlAnchor> achList=page.getAnchors();
	for(HtmlAnchor ach:achList){
		System.out.println(ach.getHrefAttribute());
	}
    webClient.closeAllWindows();		
	
	
}
 
Example 29
Project: ats-framework   File: UiDriver.java   View source code 5 votes vote down vote up
/**
 *
 * @param url the URL to open
 * @param browserVersion a {@link BrowserVersion} to emulate
 * @return new HiddenBrowserDriver instance
 */
@PublicAtsApi
public static HiddenBrowserDriver getHiddenBrowserDriver(
                                                          String url,
                                                          BrowserVersion browserVersion ) {

    return new HiddenBrowserDriver(url, browserVersion);
}
 
Example 30
Project: Cognizant-Intelligent-Test-Scripter   File: WebDriverFactory.java   View source code 5 votes vote down vote up
private static WebDriver create(RunContext context, ProjectSettings settings, Boolean isGrid, String remoteUrl) {
    DesiredCapabilities caps = new DesiredCapabilities();
    caps.setPlatform(context.Platform);
    if (!context.BrowserVersion.equalsIgnoreCase("default")) {
        caps.setVersion(context.BrowserVersion);
    }
    caps.merge(getCapability(context.BrowserName, settings));
    return create(context.BrowserName, caps, settings, isGrid, remoteUrl);
}
 
Example 31
Project: ProBOT   File: HtmlUnitTests.java   View source code 5 votes vote down vote up
@Test
public void bookRoom() throws Exception {
    try (final WebClient webClient = new WebClient(BrowserVersion.FIREFOX_52)) {

        //set proxy username and password
        final DefaultCredentialsProvider credentialsProvider = (DefaultCredentialsProvider) webClient.getCredentialsProvider();
        credentialsProvider.addNTLMCredentials("abhisheks", "abhiProJul17", "apps.prorigo.com", 80, "", "");

        final HtmlPage page = webClient.getPage("http://apps.prorigo.com/conference/Booking");

        final HtmlForm form = page.getForms().get(0);
        final HtmlButton button = form.getFirstByXPath("//*[@id=\"Submit\"]");

        HtmlSelect select = (HtmlSelect) page.getElementById("ConferenceRooms");
        HtmlOption option = select.getOptionByValue("dedfcc1d-d413-e311-9c19-0025648b200e");
        select.setSelectedAttribute(option, true);

        HtmlInput inputStartTime = form.getInputByName("StartTime");
        inputStartTime.setValueAttribute("11:00 PM");

        HtmlInput inputEndTime = form.getInputByName("EndTime");
        inputEndTime.setValueAttribute("11:30 PM");

        HtmlInput inputReason = form.getInputByName("Title");
        inputReason.type("Test Booking By HtmlUnit");

        final HtmlPage page2 = button.click();
    }
}
 
Example 32
Project: crawler-jsoup-maven   File: htmlunitTest.java   View source code 5 votes vote down vote up
public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
    
    // 屏蔽HtmlUnit等系统 log
    LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log","org.apache.commons.logging.impl.NoOpLog");
    java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
    java.util.logging.Logger.getLogger("org.apache.http.client").setLevel(Level.OFF);
    
    String url = "https://www.douyin.com/share/video/6496703951436516621/?mid=6484356820260686606";
    System.out.println("Loading page now-----------------------------------------------: "+url);
    
    /* HtmlUnit 模拟浏览器 */
    WebClient webClient = new WebClient(BrowserVersion.CHROME);
    webClient.getOptions().setJavaScriptEnabled(true);              // 启用JS解释器,默认为true  
    webClient.getOptions().setCssEnabled(false);                    // 禁用css支持  
    webClient.getOptions().setThrowExceptionOnScriptError(false);   // js运行错误时,是否抛出异常
    webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
    webClient.getOptions().setTimeout(10 * 1000);                   // 设置连接超时时间
    HtmlPage page = webClient.getPage(url);
    webClient.waitForBackgroundJavaScript(30 * 1000);               // 等待js后台执行30秒

    String pageAsXml = page.asXml();
    
    /* Jsoup解析处理 */
    // Document doc = Jsoup.parse(pageAsXml, "https://bluetata.com/");
    Document doc = Jsoup.parse(pageAsXml);  
    Elements pngs = doc.select("img[src$=.png]");                   // 获取所有图片元素集
    // 其他操作
    System.out.println(doc.toString());
}
 
Example 33
Project: poe-ladder-tracker-java   File: CSVLinkCreator.java   View source code 5 votes vote down vote up
/**
 * Starts the CSV link creation process.
 * 
 * @throws Exception If there was an error while getting the CSV specific information from the forum thread.
 */
public void create() throws Exception{
	java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(java.util.logging.Level.OFF);
	
	WebClient webClient = new WebClient(BrowserVersion.CHROME);
    HtmlPage page = (HtmlPage) webClient.getPage(forumThreadLink);
    webClient.waitForBackgroundJavaScriptStartingBefore(waitForJavaScript);
    HtmlAnchor anchor = (HtmlAnchor) page.getByXPath(xPath).get(0);
    csvFileLink = anchor.getHrefAttribute();
    csvFileLink = "http://www.pathofexile.com" + csvFileLink.subSequence(0, csvFileLink.length()-1);
    webClient.close();  
}
 
Example 34
Project: XC2   File: Server.java   View source code 5 votes vote down vote up
public Server()throws IOException{
    browser = new WebClient(BrowserVersion.CHROME);
    problems = new ArrayList<>();
    done = false;
    isSubmitting = false;
    user_name = "XC2";
    pass_word = "donthackme";
    application = Executors.newCachedThreadPool();
    //the buffer can be 100 at max (may be adjusted if needed)
    buffer_queue = new LinkedBlockingQueue<>(100);
    network = new Network_Server(this, this.port, this.max_clients);
    
    //some browser intitialization to increase efficiency (the cookies part is essential)
    browser.getOptions().setUseInsecureSSL(true);
    browser.getOptions().setJavaScriptEnabled(false);
    browser.getOptions().setCssEnabled(false);
    browser.getOptions().setThrowExceptionOnScriptError(false);
    browser.getCookieManager().setCookiesEnabled(true);
    browser.setAjaxController(new NicelyResynchronizingAjaxController());
}
 
Example 35
Project: gecco-htmlunit   File: HtmlUnitDownloder.java   View source code 5 votes vote down vote up
public HtmlUnitDownloder() {
	this.webClient = new WebClient(BrowserVersion.CHROME);
	this.webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
	this.webClient.getOptions().setThrowExceptionOnScriptError(false);
	this.webClient.getOptions().setRedirectEnabled(false);
	this.webClient.getOptions().setCssEnabled(false);
	this.webClient.setJavaScriptTimeout(1000);
	//this.webClient.setJavaScriptErrorListener(new GeccoJavaScriptErrorListener());
}
 
Example 36
Project: JSearcher   File: PostDemo.java   View source code 5 votes vote down vote up
public PostDemo() {
	webClient = new WebClient(BrowserVersion.CHROME);  
	webClient.getOptions().setCssEnabled(false);
	webClient.getOptions().setJavaScriptEnabled(false);
	
	cookieMan = webClient.getCookieManager();
	cookieMan.setCookiesEnabled(true);
}
 
Example 37
Project: JSearcher   File: LoginSimulator.java   View source code 5 votes vote down vote up
public LoginSimulator() {
	webClient = new WebClient(BrowserVersion.CHROME);  
	webClient.getOptions().setCssEnabled(false);
	webClient.getOptions().setJavaScriptEnabled(false);
	cookieMan = webClient.getCookieManager();
	cookieMan.setCookiesEnabled(true);
}
 
Example 38
Project: JSearcher   File: PostDownloader.java   View source code 5 votes vote down vote up
public PostDownloader() {
	webClient = new WebClient(BrowserVersion.CHROME);  
	webClient.getOptions().setCssEnabled(false); // disable css loader
	webClient.getOptions().setJavaScriptEnabled(false); // disable js loader
	cookieMan = webClient.getCookieManager();
	cookieMan.setCookiesEnabled(true); // enable cookie
}
 
Example 39
Project: Fetchino   File: Util.java   View source code 5 votes vote down vote up
/**
 * Creates a new web client.
 *
 * @return A new {@link WebClient} instance.
 */
public static WebClient createWebClient()
{
	WebClient webClient = new WebClient(BrowserVersion.BEST_SUPPORTED);
	webClient.getOptions().setThrowExceptionOnFailingStatusCode(true);
	webClient.getOptions().setThrowExceptionOnScriptError(false);
	return webClient;
}
 
Example 40
Project: XPathBuilder   File: AuthenticatedHtmlUnitDriver.java   View source code 5 votes vote down vote up
@Override
protected WebClient newWebClient(BrowserVersion browserVersion) {
	WebClient client = super.newWebClient(browserVersion);
	DefaultCredentialsProvider provider = new DefaultCredentialsProvider();
	provider.addCredentials(USERNAME, PASSWORD);
	client.setCredentialsProvider(provider);
	return client;
}