Java Code Examples for org.jsoup.nodes.Element

public void UpdateSign() {
	try {
		mTime = "";
		mSign = "";
		String token = ZenUtils.getToken();
		if (token != null) {
			ZenURLConnection connection = new ZenURLConnection(ZEN_UPDATE_SIGN_URL + mFid);
			connection.addRequestHeader("Cookie", "u=" + URLEncoder.encode(token, "utf-8") + ";");
			connection.addRequestHeader("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.9; rv:25.0) Gecko/20100101 Firefox/25.0");
			String response = connection.startSychronous();
			if (response != null) {
				Document doc = Jsoup.parse(response);
				
				Element signEle = doc.select("input[name=sign]").first();
				mSign = signEle.attr("value");
				Element timeEle = doc.select("input[name=time]").first();
				mTime = timeEle.attr("value");
				
				System.out.println("time: " + mTime + " sign: " + mSign);
			}
		}

	} catch (Exception e) {
		e.printStackTrace();
	}
}
 

private void parseCssSelector(Document document) {
    Elements elements = document.select("item");
    for (Element element : elements) {
        Item item = new Item();
        Elements title = element.select("title");
        Elements link = element.select("link");
        if (!title.isEmpty()) {
            item.title = title.get(0).text();
        }
        if (!link.isEmpty()) {
            item.url = link.get(0).text();
        }
        if (mList == null) {
            mList = new RssList();
        }
        mList.addItem(item);
    }
}
 

private void parseTopic(Element classElement, Topic classTopic) throws TopicMapException {
    String name = classElement.text().trim();
    
    if(name.length() == 0) return;
    
    Topic t = getOrCreateTopic(tm, null , name);
    
    if(classTopic == null) classTopic = wandoraClass;
    makeSubclassOf(tm, t, classTopic);
    
    // See if the next element is a list (of instances)
    Element listWrapper = classElement.nextElementSibling();
    if(listWrapper != null && !listWrapper.children().isEmpty()) {
        for(Element listCandidate: listWrapper.children()){
            if(listCandidate.tagName().equals("ul"))
                parseList(listCandidate, t);
        }
    }
}
 

/**
 * 
 * @param sspHandler
 * @param elementHandler 
 * @param elementHandlerWithoutDataTableMarkup
 */
private void extractTableWithDataTableMarkup(
            ElementHandler<Element> elementHandler, 
            ElementHandler<Element> elementHandlerWithoutDataTableMarkup) {
    
    Elements elementsWithMarkup = new Elements();
    
    for (Element el : elementHandler.get()) {
        if (el.select(DATA_TABLE_MARKUP_CSS_LIKE_QUERY).size() > 0) {
            elementsWithMarkup.add(el);
        } else if (elementHandlerWithoutDataTableMarkup != null) {
            elementHandlerWithoutDataTableMarkup.add(el);
        }
    }
    elementHandler.clean().addAll(elementsWithMarkup);
}
 

private void writeTableAXml(String name, String source, String filename, List<Stuff> stuff) throws IOException {
  org.jdom2.Element rootElem = new org.jdom2.Element("tableA");
  org.jdom2.Document doc = new org.jdom2.Document(rootElem);
  rootElem.addContent(new org.jdom2.Element("title").setText(name));
  rootElem.addContent(new org.jdom2.Element("source").setText(source));

  for (Stuff p : stuff) {
    org.jdom2.Element paramElem = new org.jdom2.Element("parameter");
    paramElem.setAttribute("code", Integer.toString(p.no));
    paramElem.addContent(new org.jdom2.Element("description").setText(p.desc));
    rootElem.addContent(paramElem);
  }

  XMLOutputter fmt = new XMLOutputter(Format.getPrettyFormat());
  String x = fmt.outputString(doc);

  try (FileOutputStream fout = new FileOutputStream(dirOut + filename)) {
    fout.write(x.getBytes(StandardCharsets.UTF_8));
  }

  if (show)
    System.out.printf("%s%n", x);
}
 

/**
 * Weights current element. By matching it with positive candidates and
 * weighting child nodes. Since it's impossible to predict which exactly
 * names, ids or class names will be used in HTML, major role is played by
 * child nodes
 *
 * @param e Element to weight, along with child nodes
 */
private int getWeight(Element e, boolean checkextra) {
    int weight = calcWeight(e);
    int ownTextWeight = (int) Math.round(e.ownText().length() / 100.0 * 10);
    weight += ownTextWeight;
    int childrenWeight = weightChildNodes(e);
    weight += childrenWeight;

    // add additional weight using possible 'extragravityscore' attribute
    if (checkextra) {
        Element xelem = e.select("[extragravityscore]").first();
        if (xelem != null) {
            //                System.out.println("HERE found one: " + xelem.toString());
            weight += Integer.parseInt(xelem.attr("extragravityscore"));
            //                System.out.println("WITH WEIGHT: " + xelem.attr("extragravityscore"));
        }
    }

    return weight;
}
 

private void appendTextSkipHidden(Element e, StringBuilder accum, int indent) {
    for (Node child : e.childNodes()) {
        if (unlikely(child)) {
            continue;
        }
        if (child instanceof TextNode) {
            TextNode textNode = (TextNode) child;
            String txt = textNode.text();
            accum.append(txt);
        } else if (child instanceof Element) {
            Element element = (Element) child;
            if (accum.length() > 0 && element.isBlock()
                    && !lastCharIsWhitespace(accum))
                accum.append(' ');
            else if (element.tagName().equals("br"))
                accum.append(' ');
            appendTextSkipHidden(element, accum, indent + 1);
        }
    }
}
 

@Override
protected void check(
        SSPHandler sspHandler, 
        TestSolutionHandler testSolutionHandler) {
    if (getLinkElementSelector().isEmpty()) {
        testSolutionHandler.addTestSolution(TestSolution.NOT_APPLICABLE);
        return;
    }
    
    prs = sspHandler.getProcessRemarkService();
    setServicesToChecker(titlePertinenceElementChecker);
    
    if (! getLinkElementSelector().getDecidableElements().isEmpty()) {
        setServicesToChecker(getDecidableElementsChecker());
        for (Element el : getLinkElementSelector().getDecidableElements().get()) {
            testSolutionHandler.addTestSolution(testLink(sspHandler, el));
        }
    }
    // reset service and aggregate all the remarks collected locally
    // for further save
    prs.resetService();
    prs.getRemarkList().addAll(remarks);
}
 

@Override
public List<Proxy> parse(String html) {
    Document document = Jsoup.parse(html);
    Elements elements = document.select("table tr:gt(1)");
    List<Proxy> proxyList = new ArrayList<>(elements.size());
    for (Element element : elements){
        String ip = element.select("td:eq(0)").first().text();
        String port  = element.select("td:eq(1)").first().text();
        String isAnonymous = element.select("td:eq(3)").first().text();
        log.debug("parse result = http://"+ip+":"+port+"  "+isAnonymous);
        if(!anonymousFlag || isAnonymous.contains("匿")){
            proxyList.add(new Proxy(ip, Integer.valueOf(port), "http", Constant.TIME_INTERVAL));
        }
    }
    return proxyList;
}
 

/**
 * Sets the unordered list numbering.
 * 
 * @param context
 *            the {@link Context}
 * @param element
 *            the ol {@link Element}
 */
private void setUnorderedListNumbering(Context context, Element element) {
    final String symbol;
    if (element.hasAttr(TYPE_ATTR)) {
        final String type = element.attr(TYPE_ATTR);
        if ("disc".equals(type)) {
            symbol = DISC_SYMBOL;
        } else if ("square".equals(type)) {
            symbol = SQUARE_SYMBOL;
        } else if ("circle".equals(type)) {
            symbol = CIRCLE_SYMBOL;
        } else {
            symbol = DISC_SYMBOL;
        }
    } else {
        symbol = DISC_SYMBOL;
    }

    if (context.numbering == null) {
        createNumbering(context);
    }
    context.numberingLevel = incrementNumberingLevel(context.numbering, context.numberingLevel,
            STNumberFormat.BULLET, 1, symbol, false);
}
 

public Map<String,String> aizhanIcp(String domain){
    HashMap<String, String> map = new HashMap<>();
    HttpClient httpClient = new HttpClient();
    String url = "https://icp.aizhan.com/%s/";
    try {
        ResponseStatus responseStatus = httpClient.get(String.format(url, domain));
        String html = responseStatus.getContent();
        if (!html.contains("未找到") && html.contains("该单位备案网站") && html.contains("缓存于"))
        {
            Document doc = Jsoup.parse(html);

            Elements trs = doc.select("div#company .table-s1 tbody tr");
            for (Element tr : trs) {
                String title = tr.select("td").get(1).text();
                String dom = tr.select("td").get(2).text();
                map.put(dom,title);
            }
        }
    } catch (Exception e) {
        task.getExceptions().add(e);
    }
    SysLog.info("ICP反查结束");
    return map;
}
 

@Override
protected void select(SSPHandler sspHandler) {
    totalNumberOfElements = sspHandler.getTotalNumberOfElements();
    // retrieve element from the nomenclature
    Nomenclature deprecatedHtmlAttr = nomenclatureLoaderService.
            loadByCode(PRESENTATION_ATTR_NOM);
    for (String deprecatedAttr : deprecatedHtmlAttr.getValueList()) {
        SimpleElementSelector sec = 
                    new SimpleElementSelector(buildQuery(deprecatedAttr));
        ElementHandler<Element> eh = new ElementHandlerImpl();
        sec.selectElements(sspHandler, eh);
        
        attrElementHandlerMap.put(deprecatedAttr, eh);
    }   
    
    // elements with width attribute that are not img
    SimpleElementSelector secWidthAttrNotImg = 
            new SimpleElementSelector(ELEMENT_WITH_WITDH_ATTR_NOT_IMG_V2);
    ElementHandler<Element> ehWithAttrNotImg = new ElementHandlerImpl();
    secWidthAttrNotImg.selectElements(sspHandler, ehWithAttrNotImg);
        
    attrElementHandlerMap.put(WIDTH_ATTR, ehWithAttrNotImg);
    
    // elements with width attribute that are not img
    SimpleElementSelector secHeightAttrNotImg = 
            new SimpleElementSelector(ELEMENT_WITH_HEIGHT_ATTR_NOT_IMG_V2);
    ElementHandler<Element> ehHeightAttrNotImg = new ElementHandlerImpl();
    secHeightAttrNotImg.selectElements(sspHandler, ehHeightAttrNotImg);
        
    attrElementHandlerMap.put(HEIGHT_ATTR, ehHeightAttrNotImg);
}
 

private boolean isInsideTemplate(org.jsoup.nodes.Element element,
        org.jsoup.nodes.Element templateRoot) {
    if (element == templateRoot) {
        return false;
    }
    if ("template".equalsIgnoreCase(element.tagName())) {
        return true;
    }
    return isInsideTemplate(element.parent(), templateRoot);
}
 

public boolean matches(Element root, Element element) {
    if (root == element)
        return false;

    Element prev = element.previousElementSibling();

    while (prev != null) {
        if (evaluator.matches(root, prev))
            return true;

        prev = prev.previousElementSibling();
    }
    return false;
}
 

@Override
protected List<String> getURLsFromPage(Document page) {
    List<String> pageURLs = new ArrayList<>();

    for (Element thumb : page.select("div.thumb-container a.img-container")) {
        if (isStopped()) {
            break;
        }
        String thumbURL = thumb.attr("href");
        if (thumbURL.contains("pornmd.com")) {
            continue;
        }

        String url;
        if (!thumbURL.startsWith("http")) {
            url = "https://" + DOMAIN + thumbURL;
        } else {
            url = thumbURL;
        }
        pageURLs.add(url);

        if (isThisATest()) {
            break;
        }
    }

    return pageURLs;
}
 

public void parseContests(String html) {
    Document doc = Jsoup.parse(html);
    Elements h3 = doc.select("h3");
    Elements tables = doc.select(".dataTable");
    int flag = 0;
    contestArrayList.clear();
    for (Element table : tables) {
        Elements tbody = table.getElementsByTag("tbody");
        Elements contests = tbody.get(0).children();
        for (Element contest : contests) {
            Element code = contest.child(0);
            Element name = contest.child(1);
            Element startDate = contest.child(2);
            Element endDate = contest.child(3);
            String cCode = code.text();
            String cName = name.text();
            String sDate = startDate.text();
            String eDate = endDate.text();
            Contest contest1 = new Contest(cCode, cName, sDate, eDate, flag);
            contestArrayList.add(contest1);
        }
        flag++;
    }
    contestsListAdapter = new ContestsListAdapter(contestArrayList, getActivity());
    recyclerView.swapAdapter(contestsListAdapter, false);
    if (filter != null)
        contestsListAdapter.setFilter(filter);

    if (contestArrayList == null || contestArrayList.isEmpty()) {
        recyclerView.setVisibility(View.GONE);
        contestsPlaceholder.setVisibility(View.VISIBLE);
    } else {
        recyclerView.setVisibility(View.VISIBLE);
        contestsPlaceholder.setVisibility(View.GONE);
    }
}
 

public static void parseRegistrationNumber(String data) {
  Document secData = Jsoup.parse(data);
  Element body = secData.selectFirst("body");
  Element section = body.selectFirst("section.main > section");
  Elements sections = section.select("div");
  for (Element element : sections) {
    if (element.text().equals("Results") || element.text().equals("Okay")) {
      continue;
    }
    System.out.println(element.text());
  }
}
 

@Test
public void testOnZero() throws Exception {
    Elements context = new Elements();
    Element el = new Element("V");
    el.appendText("test 69.");
    context.add(el);
    Num n = new Num();
    XValue v = n.call(Scope.create(context));
    logger.info("v = {}",v);
    Assert.assertEquals(69,v.asDouble(),0.00000000000001);
}
 

/**
 * Extract first element according to a query
 */
private static Element element(Element container, String query) {

  Elements select = container.select(query);

  if (select.size() == 0) {
    return null;
  }

  return select.first();
}
 

private List<Area> parseProvince(String url) {

        String htmlStr = HttpUtil.get(url, CHARSET);
        Document document = Jsoup.parse(htmlStr);

        // 获取 class='provincetr' 的元素
        Elements elements = document.getElementsByClass("provincetr");
        List<Area> provinces = new LinkedList<Area>();
        int sort = 1;
        for (Element element : elements) {
            // 获取 elements 下属性是 href 的元素
            Elements links = element.getElementsByAttribute("href");
            for (Element link : links) {
                String provinceName = link.text();
                String href = link.attr("href");
                String provinceCode = href.substring(0, 2);

                Area provinceArea = Area.builder().code(provinceCode + "0000")
                        .label(provinceName).source(url)
                        .sortValue(sort++)
                        .level(new RemoteData<>("PROVINCE"))
                        .fullName(provinceName)
                        .build();
                provinceArea.setChildren(parseCity(provinceName, COMMON_URL + href));

                StaticLog.info("省级数据:  {}  ", provinceArea);

                provinces.add(provinceArea);
            }
        }
        return provinces;
    }
 

@Override
public XElements evaluate(Element element) {
    List<XElements> xElementses = new ArrayList<XElements>();
    for (XPathEvaluator xPathEvaluator : xPathEvaluators) {
        xElementses.add(xPathEvaluator.evaluate(element));
    }
    return new CombiningDefaultXElements(xElementses);
}
 

/**
     * 从搜索html中得到书列表
     *
     * @param html
     * @return
     */
    public static ArrayList<Book> getBooksFromSearchHtml(String html) {
        ArrayList<Book> books = new ArrayList<>();
        Document doc = Jsoup.parse(html);
//        Element node = doc.getElementById("results");
//        for (Element div : node.children()) {
        Elements divs = doc.getElementsByClass("result-list");
        Element div = divs.get(0);
//        if (!StringHelper.isEmpty(div.className()) && div.className().equals("result-list")) {
        for (Element element : div.children()) {
            Book book = new Book();
            Element img = element.child(0).child(0).child(0);
            book.setImgUrl(img.attr("src"));
            Element title = element.getElementsByClass("result-item-title result-game-item-title").get(0);
            book.setName(title.child(0).attr("title"));
            book.setChapterUrl(title.child(0).attr("href"));
            Element desc = element.getElementsByClass("result-game-item-desc").get(0);
            book.setDesc(desc.text());
            Element info = element.getElementsByClass("result-game-item-info").get(0);
            for (Element element1 : info.children()) {
                String infoStr = element1.text();
                if (infoStr.contains("作者：")) {
                    book.setAuthor(infoStr.replace("作者：", "").replace(" ", ""));
                } else if (infoStr.contains("类型：")) {
                    book.setType(infoStr.replace("类型：", "").replace(" ", ""));
                } else if (infoStr.contains("更新时间：")) {
                    book.setUpdateDate(infoStr.replace("更新时间：", "").replace(" ", ""));
                } else {
                    Element newChapter = element1.child(1);
                    book.setNewestChapterUrl(newChapter.attr("href"));
                    book.setNewestChapterTitle(newChapter.text());
                }
            }
            book.setSource(BookSource.tianlai.toString());
            books.add(book);

        }

        return books;
    }
 

@Override
public List<RetrievableDeck> getDeckList() throws IOException {

	int nbPage = getInt(MAX_PAGE);
	List<RetrievableDeck> list = new ArrayList<>();

	for (int i = 1; i <= nbPage; i++) {
		Document d = URLTools.extractHtml(getString(URL) + "/" + getString(FORMAT) + "/?lng=fr&page=" + i);

		Elements e = d.select("tr.touch_row");

		for (Element cont : e) {
			RetrievableDeck deck = new RetrievableDeck();
			Element info = cont.select("a").get(0);
			String idColor = cont.select("img").get(0).attr("src");
			idColor = idColor.substring(idColor.lastIndexOf('/') + 1, idColor.lastIndexOf('.'));
			String name = info.text();
			String url = info.attr("href") + "/fr?get_code=1&code_type=bb_deck&code_extended=0&code_html_nl=off";
			String auteur = cont.select("a").get(1).text();

			deck.setName(name);
			try {
				deck.setUrl(new URI(url));
			} catch (URISyntaxException e1) {
				deck.setUrl(null);
			}
			deck.setAuthor(auteur);
			deck.setColor(cacheColor.get(Integer.parseInt(idColor)));

			list.add(deck);
		}
	}
	return list;
}
 

public InputField(Element singleElement) {
	super();
	tag = "input";
	this.type = "text"; // default type
	this.variableName = UtilsStaticAnalyzer.getElementName(singleElement);
	this.locator = singleElement.cssSelector();
	this.isMethod = false;
}
 

@Test
    public void testJsoup3() throws IOException {
        Document document = Jsoup.connect("https://android-arsenal.com/search?q=Circle").get();
//        Elements select = document.select("div.container.content");
        Elements select = document.select("div.pc");
        int i = 0;
        for (Element element : select) {
            System.out.println(element.toString());
            i++;
            System.out.println("---------------------------------------");
            if (i == 3) {
                return;
            }
        }
    }
 

private Element createDependencyElement(BootstrapContext context,
        JsonObject dependencyJson) {
    String type = dependencyJson.getString(Dependency.KEY_TYPE);
    if (Dependency.Type.contains(type)) {
        Dependency.Type dependencyType = Dependency.Type.valueOf(type);
        return createDependencyElement(context.getUriResolver(),
                LoadMode.INLINE, dependencyJson, dependencyType);
    }
    return Jsoup.parse(
            dependencyJson.getString(Dependency.KEY_CONTENTS), "",
            Parser.xmlParser());
}
 

@Override
public List<String> getURLsFromPage(Document doc) {
    List<String> imageURLs = new ArrayList<>();
    for (Element thumb : doc.select("div.boxed-content > a > img")) {
        String image = thumb.attr("src").replaceAll("thumbs", "images");
        image = image.replace("_b", "_o");
        image = image.replaceAll("\\d-s", "i");
        imageURLs.add(image);
    }
    return imageURLs;
}
 

/**
 * Remove an attribute from every matched element.
 * @param attributeKey The attribute to remove.
 * @return this (for chaining)
 */
public Elements removeAttr(String attributeKey) {
    for (Element element : this) {
        element.removeAttr(attributeKey);
    }
    return this;
}
 

public static String getHtmlEncode(@NonNull byte[] bytes) {
    try {
        Document doc = Jsoup.parse(new String(bytes, StandardCharsets.UTF_8));
        Elements metaTags = doc.getElementsByTag("meta");
        String charsetStr;
        for (Element metaTag : metaTags) {
            charsetStr = metaTag.attr("charset");
            if (!isEmpty(charsetStr)) {
                return charsetStr;
            }
            String content = metaTag.attr("content");
            String http_equiv = metaTag.attr("http-equiv");
            if (http_equiv.toLowerCase().equals("content-type")) {
                if (content.toLowerCase().contains("charset")) {
                    charsetStr = content.substring(content.toLowerCase().indexOf("charset") + "charset=".length());
                } else {
                    charsetStr = content.substring(content.toLowerCase().indexOf(";") + 1);
                }
                if (!isEmpty(charsetStr)) {
                    return charsetStr;
                }
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return getJavaEncode(bytes);
}
 

public static ArrayList<Topic> loadCategoryThemes(IHttpClient client, String catalogId) throws IOException {
    String pageBody = client.performGet(GAMES_CATALOG_URL).getResponseBody();
    ArrayList<Topic> res = new ArrayList<>();

    Pattern pattern = Pattern.compile("<a name=\"entry" + catalogId + "\">([\\s\\S]*?)</div>(?:<!--Begin Msg Number|<!-- TABLE FOOTER)", Pattern.CASE_INSENSITIVE);
    Matcher m = pattern.matcher(pageBody);

    if (!m.find())
        return res;
    Document doc = Jsoup.parse(m.group(1));
    Elements subCategoryElements = doc.select("ol[type=1]>li");
    for (Element topicElement : subCategoryElements) {
        Elements elements = topicElement.select("a");
        if (elements.size() == 0) continue;

        Element element = elements.get(0);
        Uri uri = Uri.parse(element.attr("href"));

        Topic topic = new Topic(uri.getQueryParameter("showtopic"), element.text());
        m = Pattern.compile("</a>(?:\\s*</b>\\s*-\\s*)(.*)?(?:<br\\s*/>|$)", Pattern.CASE_INSENSITIVE).matcher(topicElement.html());
        if (m.find())
            topic.setDescription(m.group(1));
        res.add(topic);
    }

    return res;
}