Java Code Examples for org.jsoup.select.Elements

private void parsingOffers(String urlC, List<MagicPrice> list) throws IOException {
	Elements trs = URLTools.extractHtml(urlC).select("table.table tr[data-key]");
	for(Element tr : trs)
	{
		Elements tds = tr.select("td");
		if(tds.isEmpty())
		{
			logger.debug(getName() + " found no offer");
			return;
		}
		
		MagicPrice mp = new MagicPrice();
			mp.setCountry("Brazil");
			mp.setCurrency(Currency.getInstance("BRL"));
			mp.setSite(getName());
			mp.setSeller(tds.get(1).text());
			mp.setFoil(tds.get(2).html().equalsIgnoreCase("foil"));
			mp.setQuality(tds.get(3).html());
			mp.setValue(Double.parseDouble(tds.get(5).text().replaceAll("R\\$ ", "").replace(",", ".")));
			mp.setUrl(urlC);
			list.add(mp);
	}
	logger.debug(getName() + " found " + list.size() + " offers");
}
 

@Test
public void a_block_processor_instance_should_be_executed_twice() {

    JavaExtensionRegistry javaExtensionRegistry = this.asciidoctor.javaExtensionRegistry();

    Map<String, Object> config = new HashMap<>();
    config.put(Contexts.KEY, Arrays.asList(Contexts.PARAGRAPH));
    config.put(ContentModel.KEY, ContentModel.SIMPLE);
    YellBlock yellBlock = new YellBlock("yell", config);
    javaExtensionRegistry.block(yellBlock);

    for (int i = 0; i < 2; i++) {
        String content = asciidoctor.convertFile(
                classpath.getResource("sample-with-yell-block.ad"),
                options().toFile(false).get());
        org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
        Elements elements = doc.getElementsByClass("paragraph");
        assertThat(elements.size(), is(1));
        assertThat(elements.get(0).text(), is("THE TIME IS NOW. GET A MOVE ON."));
    }
}
 

@Override
public String getDetailNext(String baseUrl, String currentUrl, byte[] result) throws UnsupportedEncodingException {
    Document document = Jsoup.parse(new String(result, "gb2312"));
    Elements elements = document.select("div.pagelist a:contains(下一页)");
    if (elements.size() > 0) {
        String tempUrl = elements.get(0).attr("href");
        if (tempUrl == null || "".equals(tempUrl))
            return "";
        Pattern pattern = Pattern.compile("http.*/");
        Matcher matcher = pattern.matcher(currentUrl);
        if (matcher.find()) {
            return matcher.group() + tempUrl;
        }
    }
    return "";
}
 

public static ArrayList<String> loadSystemR(String system){
	Path datapath = Paths.get("./src/main/resources/QALD6MultilingualLogs/multilingual_" + system + ".html");
	ArrayList<String> result = Lists.newArrayList();

	try{
		String loadedData = Files.lines(datapath).collect(Collectors.joining()); 
		Document doc = Jsoup.parse(loadedData);
		Element table = doc.select("table").get(5);
		Elements tableRows = table.select("tr");
		for(Element row: tableRows){
			Elements tableEntry = row.select("td");
			result.add(tableEntry.get(1).ownText());
		}
		result.remove(0); //remove the head of the table
		return result;
	}catch(IOException e){
		e.printStackTrace();
		log.debug("loading failed.");
		return result;
	}
}
 

/**
 * 
 * @param sspHandler
 * @param elementHandler 
 * @param elementHandlerWithoutDataTableMarkup
 */
private void extractTableWithDataTableMarkup(
            ElementHandler<Element> elementHandler, 
            ElementHandler<Element> elementHandlerWithoutDataTableMarkup) {
    
    Elements elementsWithMarkup = new Elements();
    
    for (Element el : elementHandler.get()) {
        if (el.select(DATA_TABLE_MARKUP_CSS_LIKE_QUERY).size() > 0) {
            elementsWithMarkup.add(el);
        } else if (elementHandlerWithoutDataTableMarkup != null) {
            elementHandlerWithoutDataTableMarkup.add(el);
        }
    }
    elementHandler.clean().addAll(elementsWithMarkup);
}
 

@Test
@org.jvnet.hudson.test.Issue("JENKINS-55679")
void shouldRenderXmlFiles() {
    SourcePrinter printer = new SourcePrinter();

    IssueBuilder builder = new IssueBuilder();
    Issue issue = builder.build();

    Document document = Jsoup.parse(printer.render(asStream("format.xml"), issue,
            NO_DESCRIPTION, ICON_URL));
    String expectedFile = toString("format.xml");

    assertThat(document.text()).isEqualToIgnoringWhitespace(expectedFile);

    Elements pre = document.getElementsByTag("pre");
    assertThat(pre.text()).isEqualToIgnoringWhitespace(expectedFile);
}
 

@Test
public void testGenerateBasicNumberPropertyWithMinConstraint() throws Exception
{
   Map<String, Object> root = createInspectionResultWrapper(ENTITY_NAME, NUMBER_PROP_WITH_MIN_VAL);

   Resource<URL> templateResource = resourceFactory.create(getClass().getResource(
            Deployments.BASE_PACKAGE_PATH + Deployments.BASIC_PROPERTY_DETAIL_INCLUDE));
   Template processor = processorFactory.create(templateResource, FreemarkerTemplate.class);
   String output = processor.process(root);
   Document html = Jsoup.parseBodyFragment(output);
   assertThat(output.trim(), not(equalTo("")));

   Elements container = html.select("div.form-group");
   assertThat(container, notNullValue());
   assertThat(container.attr("ng-class"), not(equalTo("")));

   Elements formInputElement = html.select("div.form-group input");
   assertThat(formInputElement.attr("id"), equalTo("score"));
   assertThat(formInputElement.attr("type"), equalTo("number"));
   assertThat(formInputElement.attr("min"), equalTo("0"));
   assertThat(formInputElement.attr("ng-model"), equalTo(StringUtils.camelCase(ENTITY_NAME) + "." + "score"));
}
 

public static String getCollectionItemImageUrl(Collection item){
    String content;
    if (!Strings.isNullOrEmpty(item.getContent())) {
        content =  item.getContent();
    }else {

        content =  item.getSummary();
    }
    if (content!=null && !content.equals("")){
        Document doc = Jsoup.parse(content);
        if (doc != null) {
            Elements images = doc.select("img");
            if (images.size() > 0) {
                return images.get(0).attr("src");
            }
        }
    }
    return null;
}
 

@Test
public void a_extension_registered_as_class_can_get_its_asciidoctor_instance() {

    JavaExtensionRegistry javaExtensionRegistry = this.asciidoctor.javaExtensionRegistry();
    TestBlock.asciidoctor = asciidoctor;
    Map<String, Object> config = new HashMap<>();
    config.put(Contexts.KEY, Arrays.asList(Contexts.LISTING));
    config.put(ContentModel.KEY, ContentModel.SIMPLE);

    javaExtensionRegistry.block("quiet", TestBlock.class);
    String content = asciidoctor.convert(
        "[quiet]\nHello World",
        options().toFile(false).get());
    org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
    Elements elements = doc.getElementsByClass("paragraph");
    assertThat(elements.size(), is(1));
    assertThat(elements.get(0).text(), is("hello world"));

    TestBlock.asciidoctor = null;
}
 

@Override
public List<Proxy> parse(String html) {
    Document document = Jsoup.parse(html);
    Elements elements = document.select("div[id=j-tab-newprd] table tbody tr");
    List<Proxy> proxyList = new ArrayList<>();
    for (Element element : elements){
        String ip_port = element.select("td:eq(1)").first().text();
        String ip = ip_port.split(":")[0];
        String port  = ip_port.split(":")[1];

        String isAnonymous = element.select("td:eq(3)").first().text();
        String type = element.select("td:eq(4)").first().text();
        if(!anonymousFlag || isAnonymous.contains("匿") || isAnonymous.contains("anonymous")){
            proxyList.add(new Proxy(ip, Integer.valueOf(port), type, Constant.TIME_INTERVAL));
        }
    }
    return proxyList;
}
 

@MatchType(types = "searchEngine")
public void visitSearchEngine(Page page, CrawlDatums next) {
    String keyword = page.meta("keyword");
    int pageIndex = page.metaAsInt("pageIndex");
    System.out.println("成功抓取关键词" + keyword + "的第" + pageIndex + "页搜索结果");
    Elements results = page.select("li.b_algo>h2>a");

    for (int rank = 0; rank < results.size(); rank++) {
        Element result = results.get(rank);
        /*
        我们希望继续爬取每条搜索结果指向的网页，这里统称为外链。
        我们希望在访问外链时仍然能够知道外链处于搜索引擎的第几页、第几条，
        所以将页号和排序信息放入后续的CrawlDatum中，为了能够区分外链和
        搜索引擎结果页面，type设置为outlink，这里的值完全由
        用户定义，可以设置一个任意的值
        */
        String href = result.attr("abs:href");
        next.addAndReturn(href)
                .type("outlink")
                .meta("keyword", keyword)
                .meta("pageIndex", pageIndex)
                .meta("rank", rank);
    }
}
 

/**
 * 函数具体逻辑
 *
 * @param scope 上下文
 * @return 计算好的节点
 */
@Override
public XValue call(Scope scope) {
    Elements context = scope.context();
    List<String> res = new LinkedList<>();
    if (context!=null&&context.size()>0){
        if (scope.isRecursion()){
            NodeTest allTextFun = Scanner.findNodeTestByName("allText");
            return allTextFun.call(scope);
        }else {
            for (Element e:context){
                if ("script".equals(e.nodeName())){
                    res.add(e.data());
                }else {
                    res.add(e.ownText());
                }
            }
        }
    }
    return XValue.create(res);
}
 

/**
 * Statically analyze the DOM of the State s to create comprehensive Form
 * objects
 * 
 * @param s
 * @return
 */
public static Set<Form> createFormObjects(State s) {

	Set<Form> fl = new HashSet<Form>();
	String dom = s.getDom();

	Document doc = Jsoup.parse(dom, "UTF-8");
	Elements forms = doc.getElementsByTag("form");
	System.out.println("[LOG]\t" + forms.size() + " form(s) found in " + s.getName());

	for (org.jsoup.nodes.Element currentForm : forms) {
		Form formObject = new Form();

		formObject.setAttributes(currentForm.attributes().asList());
		formObject.setFormName(getFormName(formObject, currentForm));
		formObject.setReturnValue("void");
		formObject.setFormFieldList(analyzeFormBody(s, currentForm));

		fl.add(formObject);
	}

	return fl;
}
 

private List<WebDavFile> parseDir(String s) {
    List<WebDavFile> list = new ArrayList<>();
    Document document = Jsoup.parse(s);
    Elements elements = document.getElementsByTag("d:response");
    String baseUrl = getUrl().endsWith("/") ? getUrl() : getUrl() + "/";
    for (Element element : elements) {
        String href = element.getElementsByTag("d:href").get(0).text();
        if (!href.endsWith("/")) {
            String fileName = href.substring(href.lastIndexOf("/") + 1);
            WebDavFile webDavFile;
            try {
                webDavFile = new WebDavFile(baseUrl + fileName);
                webDavFile.setDisplayName(fileName);
                webDavFile.setUrlName(href);
                list.add(webDavFile);
            } catch (MalformedURLException e) {
                e.printStackTrace();
            }
        }
    }
    return list;
}
 

@Test
public void a_block_processor_instance_should_be_executed_when_registered_block_is_found_in_document() {

    JavaExtensionRegistry javaExtensionRegistry = this.asciidoctor.javaExtensionRegistry();

    Map<String, Object> config = new HashMap<>();
    config.put(Contexts.KEY, Arrays.asList(Contexts.PARAGRAPH));
    config.put(ContentModel.KEY, ContentModel.SIMPLE);
    YellBlock yellBlock = new YellBlock("yell", config);
    javaExtensionRegistry.block(yellBlock);
    String content = asciidoctor.convertFile(
            classpath.getResource("sample-with-yell-block.ad"),
            options().toFile(false).get());
    org.jsoup.nodes.Document doc = Jsoup.parse(content, "UTF-8");
    Elements elements = doc.getElementsByClass("paragraph");
    assertThat(elements.size(), is(1));
    assertThat(elements.get(0).text(), is("THE TIME IS NOW. GET A MOVE ON."));

}
 

/**
 * Extracts script nodes from web page HTML document
 *
 * @param url
 * @throws IOException
 */
private void getScriptNodes(URL url) throws IOException {

    // Document doc = getHTML(url);// this required another connexion

    Elements scripts = HTMLDoc.select("script");

    for (int i = 0; i < scripts.size(); i++) {

        Element script = scripts.get(i);

        if (script.hasAttr("src")) {
            // System.out.println("script = "+scripts.get(i)+"");
            this.scripts.add(script);
        }
        // System.out.println("-----------------------");
    }
}
 

public static ArrayList<String> loadSystemR(String system){
	Path datapath = Paths.get("./src/main/resources/QALD6MultilingualLogs/multilingual_" + system + ".html");
	ArrayList<String> result = Lists.newArrayList();

	try{
		String loadedData = Files.lines(datapath).collect(Collectors.joining()); 
		Document doc = Jsoup.parse(loadedData);
		Element table = doc.select("table").get(5);
		Elements tableRows = table.select("tr");
		for(Element row: tableRows){
			Elements tableEntry = row.select("td");
			result.add(tableEntry.get(1).ownText());
		}
		result.remove(0); //remove the head of the table
		return result;
	}catch(IOException e){
		e.printStackTrace();
		log.debug("loading failed.");
		return result;
	}
}
 

/**
 * 提取网页中的表单数据
 * @param source 网页源代码
 * @return 表单数据
 */
public static Map<String, String> extractForm(String source) {
    Map<String, String> map = new HashMap<>();

    if (StringUtils.isNotBlank(source)) {
        Document doc = Jsoup.parse(source);
        Elements elements = doc.select("input");
        for (Element element : elements) {
            String key;
            if (org.apache.commons.lang3.StringUtils.isNotBlank(element.attr("name"))) {
                key = element.attr("name");
            } else if (org.apache.commons.lang3.StringUtils.isNotBlank(element.id())) {
                key = element.id();
            } else {
                continue;
            }
            map.put(key, element.val());
        }
    }

    return map;
}
 

@Test
public void batchFilterNoBackfill() {
  Map<String, Object> context = ImmutableMap.of(
    "items",
    (Object) Lists.newArrayList("1", "2", "3", "4", "5", "6")
  );

  Document dom = Jsoup.parseBodyFragment(render("batch-filter", context));
  assertThat(dom.select("tr")).hasSize(2);

  Elements trs = dom.select("tr");
  assertThat(trs.get(0).select("td")).hasSize(3);
  assertThat(trs.get(0).select("td").get(0).text()).isEqualTo("1");
  assertThat(trs.get(0).select("td").get(1).text()).isEqualTo("2");
  assertThat(trs.get(0).select("td").get(2).text()).isEqualTo("3");
  assertThat(trs.get(1).select("td")).hasSize(3);
  assertThat(trs.get(1).select("td").get(0).text()).isEqualTo("4");
  assertThat(trs.get(1).select("td").get(1).text()).isEqualTo("5");
  assertThat(trs.get(1).select("td").get(2).text()).isEqualTo("6");
}
 

public static ArrayList<String> loadSystemP(String system){

		Path datapath = Paths.get("./src/main/resources/QALD6MultilingualLogs/multilingual_" + system + ".html");
		ArrayList<String> result = Lists.newArrayList();

		try{
			String loadedData = Files.lines(datapath).collect(Collectors.joining()); 
			Document doc = Jsoup.parse(loadedData);
			Element table = doc.select("table").get(5);
			Elements tableRows = table.select("tr");
			for(Element row: tableRows){
				Elements tableEntry = row.select("td");
				result.add(tableEntry.get(2).ownText());
			}
			result.remove(0); //remove the head of the table
			return result;
		}catch(IOException e){
			e.printStackTrace();
			log.debug("loading failed.");
			return result;
		}
	}
 

@Override
public boolean extractTopicsFrom(Document d, String u, TopicMap t) throws Exception {
    
    this.tm = t;
    this.wandoraClass = getWandoraClassTopic(tm);
    
    Topic documentType = getOrCreateTopic(tm, DOC_TYPE, "Document");
    makeSubclassOf(tm, documentType, wandoraClass);
    
    Topic docTopic = getOrCreateTopic(tm, u);
    docTopic.addType(documentType);
    
    Elements links = d.select("a");
    
    for(Element link: links){
        try {
            parseLink(link,docTopic);
        } 
        catch (TopicMapException tme) {
            log(tme.getMessage()); 
        }
    }
    
    return true;

}
 

private void parseCssSelector(Document document) {
    Elements elements = document.select("item");
    for (Element element : elements) {
        Item item = new Item();
        Elements title = element.select("title");
        Elements link = element.select("link");
        if (!title.isEmpty()) {
            item.title = title.get(0).text();
        }
        if (!link.isEmpty()) {
            item.url = link.get(0).text();
        }
        if (mList == null) {
            mList = new RssList();
        }
        mList.addItem(item);
    }
}
 

@Override
public List<Proxy> parse(String content) {
    Document document = Jsoup.parse(content);
    Elements elements = document.select("div[class='table-responsive'] table[class='table'] tbody tr");
    List<Proxy> proxyList = new ArrayList<>(elements.size());
    for (Element element : elements){
        String ip = element.select("td:eq(0)").first().text();
        String port  = element.select("td:eq(2)").first().text();
        String type = element.select("td:eq(5)").first().text();
        String isAnonymous = element.select("td:eq(7)").first().text();
        System.out.println("ip:"+ip);
        log.debug("parse result = "+type+"://"+ip+":"+port+"  "+isAnonymous);
        if(!anonymousFlag || isAnonymous.contains("匿")){
            proxyList.add(new Proxy(ip, Integer.valueOf(port), type, Constant.TIME_INTERVAL));
        }
    }
    return proxyList;
}
 

/**
 * Get this element's child elements.
 * <p>
 * This is effectively a filter on {@link #childNodes()} to get Element nodes.
 * </p>
 * @return child elements. If this element has no children, returns an
 * empty list.
 * @see #childNodes()
 */
public Elements children() {
    // create on the fly rather than maintaining two lists. if gets slow, memoize, and mark dirty on change
    List<Element> elements = new ArrayList<Element>(childNodes.size());
    for (Node node : childNodes) {
        if (node instanceof Element)
            elements.add((Element) node);
    }
    return new Elements(elements);
}
 

private static void prependBaseHref(VaadinRequest request,
        Document indexDocument) {
    Elements base = indexDocument.head().getElementsByTag("base");
    String baseHref = getServiceUrl(request);
    if (base.isEmpty()) {
        indexDocument.head().prependElement("base").attr("href", baseHref);
    } else {
        base.first().attr("href", baseHref);
    }
}
 

@Override
public ArrayList<DownloadBean> getDownloadurls(final String url) throws InterruptedException {
    latch = new CountDownLatch(1);
    final ArrayList<DownloadBean> urls = new ArrayList<>();
    mExecutorService.submit(new Runnable() {
        @Override
        public void run() {
            try {
                Elements elements = Jsoup.connect(url)
                        .timeout(10000)
                        .ignoreContentType(true)
                        .ignoreHttpErrors(true)
                        .userAgent(Url.MOBBILE_AGENT)
                        .get()
                        .select("body > div:nth-child(5)");
                String u1 = elements.select("li:nth-child(2) > a").attr("abs:href");
                String u1n = elements.select("li:nth-child(2) > a").text();
                String u2 = elements.select("li:nth-child(3) > a").attr("abs:href");
                String u2n = elements.select("li:nth-child(3) > a").text();
                urls.add(new DownloadBean(u1n, u1));
                urls.add(new DownloadBean(u2n, u2));
            } catch (IOException e) {
                e.printStackTrace();
            }
            latch.countDown();
        }
    });
    latch.await();
    return urls;
}
 

private static void accumulateParents(Element el, Elements parents) {
    Element parent = el.parent();
    if (parent != null && !parent.tagName().equals("#root")) {
        parents.add(parent);
        accumulateParents(parent, parents);
    }
}
 

@Override
protected List<FriendData> doInBackground(String... params) {
    final List<FriendData> temp = new ArrayList<>();
    HttpUtil.syncGet(FriendActivity.this, params[0], new TextResponseHandler() {
        @Override
        public void onSuccess(String response) {
            Document document = Jsoup.parse(response);
            Elements lists = document.select("ul.buddy").select("li");
            if (lists.size() <= 0) {
                isHaveMore = false;
            } else {
                for (Element element : lists) {
                    String imgurl = UrlUtils.getFullUrl(element.select("img").attr("src"));
                    Element usernameNode = element.select("h4").select("a[href^=home.php?mod=space&uid=]").get(0);
                    String userName = usernameNode.text();
                    Integer usernameColor = GetId.getColor(usernameNode.attr("style"));
                    if (usernameColor == null) {
                        usernameColor = ContextCompat.getColor(FriendActivity.this, R.color.text_color_pri);
                    }
                    String uid = GetId.getId("uid=", imgurl);
                    String info = element.select("p.maxh").text();
                    boolean online = element.select("em.gol").text().contains("在线");
                    //userName,imgUrl,info,uid
                    temp.add(new FriendData(userName, usernameColor, imgurl, info, uid, online));
                }
            }
        }

        @Override
        public void onFailure(Throwable e) {
            super.onFailure(e);
        }
    });
    return temp;
}
 

public Gallery(Context context, String json, Elements related, boolean isFavorite) throws IOException{
    LogUtility.d("Found JSON: "+json);
    JsonReader reader=new JsonReader(new StringReader(json));
    this.related =new ArrayList<>(related.size());
    for(Element e:related) this.related.add(new SimpleGallery(context,e));
    galleryData=new GalleryData(reader);
    calculateSizes(galleryData);
    onlineFavorite=isFavorite;
}
 

private void fuckIPLocation(Task task) {
    String domain = task.getResult().getHostInfo().getDomain();
    task.getExecutor().submit(
            new Runnable() {
                @Override
                public void run() {

                    try {
                        HttpResponse resp = requests.url("http://ip.tool.chinaz.com/" + domain).get();
                        Document parse = resp.parse();
                        if (parse!=null ){
                            Elements select = parse.select("p.WhwtdWrap:contains(物理位置) +p > span");
                            if (select.size() > 0)
                            {
                                String ip = select.get(1).text();
                                String location = select.last().text();
                                if (location!=null && location.toLowerCase().contains("cdn"))
                                    task.getResult().getHostInfo().setCdn(true);
                                task.getResult().getAdditional().put("IP物理位置",ip + " --- " + location);
                            }
                        }
                    } catch (MalformedURLException e) {
                    }
                }
            }
    );
}