Java Code Examples for org.jsoup.nodes.TextNode

The following examples show how to use org.jsoup.nodes.TextNode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: intellij-quarkus   Source File: HtmlToPlainText.java    License: Eclipse Public License 2.0 7 votes vote down vote up
@Override
public void head(Node node, int depth) {
    String name = node.nodeName();
    if (node instanceof TextNode) {
        append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
    } else if (name.equals("ul")) {
        listNesting++;
    } else if (name.equals("li")) {
        append("\n ");
        for (int i = 1; i < listNesting; i++) {
            append("  ");
        }
        if (listNesting == 1) {
            append("* ");
        } else {
            append("- ");
        }
    } else if (name.equals("dt")) {
        append("  ");
    } else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr")) {
        append("\n");
    }
}
 
Example 2
private void generateTextContent(StringBuilder sb, List<Node> nodes) {
    for (Node node : nodes) {
        if (node instanceof Element) {
            Element element = (Element) node;

            switch (element.nodeName()) {
                case "a":
                    sb.append(getTextLink(element));
                    break;

                case "br":
                    sb.append('\n');
                    break;

                default:
                    generateTextContent(sb, element.childNodes());
                    break;
            }
        } else if (node instanceof TextNode) {
            sb.append(((TextNode) node).getWholeText());
        }
    }
}
 
Example 3
Source Project: a   Source File: PageLoaderEpub.java    License: GNU General Public License v3.0 6 votes vote down vote up
@Override
protected String getChapterContent(BookChapterBean chapter) throws Exception {
    Resource resource = epubBook.getResources().getByHref(chapter.getDurChapterUrl());
    StringBuilder content = new StringBuilder();
    Document doc = Jsoup.parse(new String(resource.getData(), mCharset));
    Elements elements = doc.getAllElements();
    for (Element element : elements) {
        List<TextNode> contentEs = element.textNodes();
        for (int i = 0; i < contentEs.size(); i++) {
            String text = contentEs.get(i).text().trim();
            text = StringUtils.formatHtml(text);
            if (elements.size() > 1) {
                if (text.length() > 0) {
                    if (content.length() > 0) {
                        content.append("\r\n");
                    }
                    content.append("\u3000\u3000").append(text);
                }
            } else {
                content.append(text);
            }
        }
    }
    return content.toString();
}
 
Example 4
Source Project: FairEmail   Source File: HtmlHelper.java    License: GNU General Public License v3.0 6 votes vote down vote up
private static String _getText(Document d, boolean full) {
    truncate(d, !full);

    for (Element bq : d.select("blockquote")) {
        bq.prependChild(new TextNode("["));
        bq.appendChild(new TextNode("]"));
    }

    String text = d.text();
    if (full)
        return text;

    String preview = text.substring(0, Math.min(text.length(), PREVIEW_SIZE));
    if (preview.length() < text.length())
        preview += "…";

    return preview;
}
 
Example 5
Source Project: lemminx   Source File: HtmlToPlainText.java    License: Eclipse Public License 2.0 6 votes vote down vote up
@Override
public void head(Node node, int depth) {
	String name = node.nodeName();
	if (node instanceof TextNode) {
		append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
	} else if (name.equals("ul")) {
		listNesting++;
	} else if (name.equals("li")) {
		append("\n ");
		for (int i = 1; i < listNesting; i++) {
			append("  ");
		}
		if (listNesting == 1) {
			append("* ");
		} else {
			append("- ");
		}
	} else if (name.equals("dt")) {
		append("  ");
	} else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr")) {
		append("\n");
	}
}
 
Example 6
Source Project: MyBookshelf   Source File: PageLoaderEpub.java    License: GNU General Public License v3.0 6 votes vote down vote up
@Override
protected String getChapterContent(BookChapterBean chapter) throws Exception {
    Resource resource = epubBook.getResources().getByHref(chapter.getDurChapterUrl());
    StringBuilder content = new StringBuilder();
    Document doc = Jsoup.parse(new String(resource.getData(), mCharset));
    Elements elements = doc.getAllElements();
    for (Element element : elements) {
        List<TextNode> contentEs = element.textNodes();
        for (int i = 0; i < contentEs.size(); i++) {
            String text = contentEs.get(i).text().trim();
            text = StringUtils.formatHtml(text);
            if (elements.size() > 1) {
                if (text.length() > 0) {
                    if (content.length() > 0) {
                        content.append("\r\n");
                    }
                    content.append("\u3000\u3000").append(text);
                }
            } else {
                content.append(text);
            }
        }
    }
    return content.toString();
}
 
Example 7
Source Project: OpenHub   Source File: CollectionsPresenter.java    License: GNU General Public License v3.0 6 votes vote down vote up
private ArrayList<Collection> getBellowCollections(Document doc){
    ArrayList<Collection> collections = new ArrayList<>();
    Elements elements = doc.getElementsByClass(
            "d-flex border-bottom border-gray-light pb-4 mb-5");
    for (Element element : elements) {
        Element titleElement = element.select("div > h2 > a").first();
        Element descElement = element.select("div").last();
        String id = titleElement.attr("href");
        id = id.substring(id.lastIndexOf("/") + 1);
        String title = titleElement.textNodes().get(0).toString();

        List<TextNode> descTextNodes = descElement.textNodes();
        int descIndex = descTextNodes.size() == 0 ? 0 : descTextNodes.size() - 1;
        String desc = descTextNodes.get(descIndex).toString().trim();
        Collection collection = new Collection(id, title, desc);
        collections.add(collection);
    }
    return collections;
}
 
Example 8
Source Project: scava   Source File: HtmlParser.java    License: Eclipse Public License 2.0 6 votes vote down vote up
private static void readNodes(List<Node> nodeList, List<String> textList)
{
	String tempText;
	for(Node node : nodeList)
	{
		if(node.childNodeSize()>0)
		{
			readNodes(node.childNodes(), textList);
		}
		else
		{
			if(node.nodeName().equals("#text"))
			{
				tempText=((TextNode) node).getWholeText();
				tempText=newline.matcher(tempText).replaceAll("");
				if(!tempText.isEmpty())
					textList.add(tempText);
			}
		}
	}
}
 
Example 9
Source Project: scava   Source File: HtmlParser.java    License: Eclipse Public License 2.0 6 votes vote down vote up
private static void readNodesWithTags(List<Node> nodeList, List<Map.Entry<String,String>> textListMap, String tag)
{
	for(Node node : nodeList)
	{
		if(node.childNodeSize()>0)
		{
			readNodesWithTags(node.childNodes(), textListMap, node.nodeName());
		}
		else
		{
			if(node.nodeName().equals("#text"))
			{
				if(tag.equalsIgnoreCase("body"))
					tag="p";
				textListMap.add(new AbstractMap.SimpleEntry<String,String>(tag, ((TextNode) node).getWholeText() ));
			}
		}
	}
}
 
Example 10
Source Project: Xndroid   Source File: OutputFormatter.java    License: GNU General Public License v3.0 6 votes vote down vote up
private void appendTextSkipHidden(Element e, StringBuilder accum, int indent) {
    for (Node child : e.childNodes()) {
        if (unlikely(child)) {
            continue;
        }
        if (child instanceof TextNode) {
            TextNode textNode = (TextNode) child;
            String txt = textNode.text();
            accum.append(txt);
        } else if (child instanceof Element) {
            Element element = (Element) child;
            if (accum.length() > 0 && element.isBlock()
                    && !lastCharIsWhitespace(accum))
                accum.append(' ');
            else if (element.tagName().equals("br"))
                accum.append(' ');
            appendTextSkipHidden(element, accum, indent + 1);
        }
    }
}
 
Example 11
Source Project: dkpro-c4corpus   Source File: Paragraph.java    License: Apache License 2.0 6 votes vote down vote up
public void initRawInfo()
{
    StringBuilder sb = new StringBuilder();
    for (Node n : this) {
        //            NodeHelper.cleanEmptyElements(n);
        if (n instanceof TextNode) {
            this.setTagName(getPath(n));
            String nodeRawText = ((TextNode) n).text();
            sb.append(Utils.normalizeBreaks(nodeRawText).trim());

            if (NodeHelper.isLink(n)) {
                charsCountInLinks += nodeRawText.length();
            }
        }
    }

    rawText = sb.toString();
}
 
Example 12
Source Project: dkpro-c4corpus   Source File: Paragraph.java    License: Apache License 2.0 6 votes vote down vote up
public String getPath(Node n)
{
    String nodePath = "";
    while (n != null) {
        if (n instanceof TextNode) {
            n = n.parent();
        }
        if (NodeHelper.isInnerText(n)) {
            n = n.parent();
        }
        String parentNodeName = n.nodeName();
        nodePath = parentNodeName + "." + nodePath;

        if (!parentNodeName.equalsIgnoreCase("html")) {
            n = n.parent();
        }
        else {
            break;
        }
    }

    return nodePath;
}
 
Example 13
Source Project: eclipse.jdt.ls   Source File: HtmlToPlainText.java    License: Eclipse Public License 2.0 6 votes vote down vote up
@Override
public void head(Node node, int depth) {
	String name = node.nodeName();
	if (node instanceof TextNode) {
		append(((TextNode) node).text()); // TextNodes carry all user-readable text in the DOM.
	} else if (name.equals("ul")) {
		listNesting++;
	} else if (name.equals("li")) {
		append("\n ");
		for (int i = 1; i < listNesting; i++) {
			append("  ");
		}
		if (listNesting == 1) {
			append("* ");
		} else {
			append("- ");
		}
	} else if (name.equals("dt")) {
		append("  ");
	} else if (StringUtil.in(name, "p", "h1", "h2", "h3", "h4", "h5", "tr")) {
		append("\n");
	}
}
 
Example 14
Source Project: kafka-connect-couchbase   Source File: HtmlRenderer.java    License: Apache License 2.0 6 votes vote down vote up
private static void renderAsPlaintext(Node node, StringBuilder out) {
  if (node instanceof TextNode) {
    String text = ((TextNode) node).text();
    if (out.length() == 0 || endsWithWhitespace(out)) {
      text = trimLeft(text);
    }
    out.append(text);
    return;
  }

  if (node instanceof Element) {
    Element e = (Element) node;

    if (e.tagName().equals("p") || e.tagName().equals("br")) {
      trimRight(out);
      if (out.length() > 0) {
        out.append(PARAGRAPH_SEPARATOR);
      }
    }

    for (Node child : e.childNodes()) {
      renderAsPlaintext(child, out);
    }
  }
}
 
Example 15
@Override
public String operate(Element element) {
    int index = 0;
    StringBuilder accum = new StringBuilder();
    for (Node node : element.childNodes()) {
        if (node instanceof TextNode) {
            TextNode textNode = (TextNode) node;
            if (group == 0) {
                accum.append(textNode.text());
            } else if (++index == group) {
                return textNode.text();
            }
        }
    }
    return accum.toString();
}
 
Example 16
@Override
public String buildTextFromElement(Element element) {
    StringBuilder elementText = new StringBuilder();
    if (element.hasAttr(ALT_ATTR)) {
        elementText.append(SPACER);
        elementText.append(altAttrTextBuilder.buildTextFromElement(element));
    }
    for (Node child : element.childNodes()) {
        if (child instanceof TextNode && !((TextNode)child).isBlank()) {
           elementText.append(SPACER);
           elementText.append(StringUtils.trim(((TextNode)child).text()));
        } else if (child instanceof Element){
            elementText.append(SPACER);
            elementText.append(buildTextFromElement((Element)child));
        }
    }
    return StringUtils.trim(elementText.toString());
}
 
Example 17
Source Project: flow   Source File: ElementUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Converts the given element and its children to a JSoup node with
 * children.
 *
 * @param document
 *            A JSoup document
 * @param element
 *            The element to convert
 * @return A JSoup node containing the converted element
 */
public static Node toJsoup(Document document, Element element) {
    if (element.isTextNode()) {
        return new TextNode(element.getText(), document.baseUri());
    }

    org.jsoup.nodes.Element target = document
            .createElement(element.getTag());
    if (element.hasProperty("innerHTML")) {
        target.html((String) element.getPropertyRaw("innerHTML"));
    }

    element.getAttributeNames().forEach(name -> {
        String attributeValue = element.getAttribute(name);
        if ("".equals(attributeValue)) {
            target.attr(name, true);
        } else {
            target.attr(name, attributeValue);
        }
    });

    element.getChildren()
            .forEach(child -> target.appendChild(toJsoup(document, child)));

    return target;
}
 
Example 18
Source Project: flow   Source File: ElementUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Converts a given JSoup {@link org.jsoup.nodes.Node} and its children into
 * a matching {@link com.vaadin.flow.dom.Element} hierarchy.
 * <p>
 * Only nodes of type {@link org.jsoup.nodes.TextNode} and
 * {@link org.jsoup.nodes.Element} are converted - other node types return
 * an empty optional.
 *
 * @param node
 *            JSoup node to convert
 * @return element with the matching hierarchy as the given node, or empty
 */
public static Optional<Element> fromJsoup(Node node) {
    Element ret;
    if (node instanceof TextNode) {
        return Optional.of(Element.createText(((TextNode) node).text()));
    } else if (node instanceof org.jsoup.nodes.Element) {
        ret = new Element(((org.jsoup.nodes.Element)node).tagName());
    } else {
        LoggerFactory.getLogger(ElementUtil.class).error(
                "Could not convert a {}, '{}' into {}!",
                Node.class.getName(), node, Element.class.getName());
        return Optional.empty();
    }

    node.attributes().asList().forEach(attribute -> ret
            .setAttribute(attribute.getKey(), attribute.getValue()));

    List<Node> childNodes = node.childNodes();
    if (!childNodes.isEmpty()) {
        childNodes.forEach(
                child -> fromJsoup(child).ifPresent(ret::appendChild));
    }

    return Optional.of(ret);
}
 
Example 19
Source Project: JumpGo   Source File: OutputFormatter.java    License: Mozilla Public License 2.0 6 votes vote down vote up
private void appendTextSkipHidden(Element e, StringBuilder accum, int indent) {
    for (Node child : e.childNodes()) {
        if (unlikely(child)) {
            continue;
        }
        if (child instanceof TextNode) {
            TextNode textNode = (TextNode) child;
            String txt = textNode.text();
            accum.append(txt);
        } else if (child instanceof Element) {
            Element element = (Element) child;
            if (accum.length() > 0 && element.isBlock()
                    && !lastCharIsWhitespace(accum))
                accum.append(' ');
            else if (element.tagName().equals("br"))
                accum.append(' ');
            appendTextSkipHidden(element, accum, indent + 1);
        }
    }
}
 
Example 20
Source Project: zrlog   Source File: ParseUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static String autoDigest(String str, int size) {
    StringBuilder sb = new StringBuilder();
    Document document = Jsoup.parseBodyFragment(str);
    List<Node> allTextNode = new ArrayList<>();
    getAllTextNode(document.childNodes(), allTextNode);
    int tLength = 0;
    for (Node node : allTextNode) {
        if (node instanceof TextNode) {
            sb.append(node.parent().outerHtml());
            tLength += ((TextNode) node).text().length();
            if (tLength > size) {
                sb.append(" ...");
                break;
            }
        }
    }
    String digest = sb.toString();
    Elements elements = Jsoup.parse(str).body().select("video");
    if (elements != null && !elements.isEmpty()) {
        digest = elements.get(0).toString() + "<br/>" + digest;
    }
    return digest.trim();
}
 
Example 21
private void addTextNode(TextNode tNode) {

        String text = tNode.text().trim();
        if (text.isEmpty()) {
            return;
        }
        String xpath = JsoupHelper.getXpath(tNode);
        tNodeList.add(tNode);
        xpathMap.put(tNode, xpath);

        CountInfo countInfo = new CountInfo(tNode);
        ArrayList<CountInfo> countInfoList = countMap.get(xpath);
        if (countInfoList == null) {
            countInfoList = new ArrayList<CountInfo>();
            countMap.put(xpath, countInfoList);
        }
        countInfoList.add(countInfo);
    }
 
Example 22
Source Project: astor   Source File: Cleaner.java    License: GNU General Public License v2.0 6 votes vote down vote up
public void head(Node source, int depth) {
    if (source instanceof Element) {
        Element sourceEl = (Element) source;

        if (whitelist.isSafeTag(sourceEl.tagName())) { // safe, clone and copy safe attrs
            ElementMeta meta = createSafeElement(sourceEl);
            Element destChild = meta.el;
            destination.appendChild(destChild);

            numDiscarded += meta.numAttribsDiscarded;
            destination = destChild;
        } else if (source != root) { // not a safe tag, so don't add. don't count root against discarded.
            numDiscarded++;
        }
    } else if (source instanceof TextNode) {
        TextNode sourceText = (TextNode) source;
        TextNode destText = new TextNode(sourceText.getWholeText());
        destination.appendChild(destText);
    } else if (source instanceof DataNode && whitelist.isSafeTag(source.parent().nodeName())) {
      DataNode sourceData = (DataNode) source;
      DataNode destData = new DataNode(sourceData.getWholeData());
      destination.appendChild(destData);
    } else { // else, we don't care about comments, xml proc instructions, etc
        numDiscarded++;
    }
}
 
Example 23
Source Project: astor   Source File: Cleaner.java    License: GNU General Public License v2.0 6 votes vote down vote up
public void head(Node source, int depth) {
    if (source instanceof Element) {
        Element sourceEl = (Element) source;

        if (whitelist.isSafeTag(sourceEl.tagName())) { // safe, clone and copy safe attrs
            ElementMeta meta = createSafeElement(sourceEl);
            Element destChild = meta.el;
            destination.appendChild(destChild);

            numDiscarded += meta.numAttribsDiscarded;
            destination = destChild;
        } else if (source != root) { // not a safe tag, so don't add. don't count root against discarded.
            numDiscarded++;
        }
    } else if (source instanceof TextNode) {
        TextNode sourceText = (TextNode) source;
        TextNode destText = new TextNode(sourceText.getWholeText());
        destination.appendChild(destText);
    } else if (source instanceof DataNode && whitelist.isSafeTag(source.parent().nodeName())) {
      DataNode sourceData = (DataNode) source;
      DataNode destData = new DataNode(sourceData.getWholeData());
      destination.appendChild(destData);
    } else { // else, we don't care about comments, xml proc instructions, etc
        numDiscarded++;
    }
}
 
Example 24
Source Project: WordCount   Source File: ContentExtractor.java    License: GNU General Public License v2.0 6 votes vote down vote up
private void addTextNode(TextNode tNode) {

        String text = tNode.text().trim();
        if (text.isEmpty()) {
            return;
        }
        String xpath = JsoupHelper.getXpath(tNode);
        tNodeList.add(tNode);
        xpathMap.put(tNode, xpath);

        CountInfo countInfo = new CountInfo(tNode);
        ArrayList<CountInfo> countInfoList = countMap.get(xpath);
        if (countInfoList == null) {
            countInfoList = new ArrayList<CountInfo>();
            countMap.put(xpath, countInfoList);
        }
        countInfoList.add(countInfo);
    }
 
Example 25
Source Project: jinjava   Source File: TruncateHtmlFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void head(Node node, int depth) {
  if (node instanceof TextNode) {
    TextNode text = (TextNode) node;
    String textContent = text.text();

    if (textLen >= maxTextLen) {
      text.text("");
    } else if (textLen + textContent.length() > maxTextLen) {
      int ptr = maxTextLen - textLen;
      if (!killwords) {
        ptr = Functions.movePointerToJustBeforeLastWord(ptr, textContent) - 1;
      }

      text.text(textContent.substring(0, ptr) + ending);
      textLen = maxTextLen;
    } else {
      textLen += textContent.length();
    }
  }
}
 
Example 26
@Test
public void ruby_treeprocessor_should_be_registered() {

    this.asciidoctor.createGroup()
        .loadRubyClass(getClass().getResourceAsStream("/ruby-extensions/shell-session-tree-processor.rb"))
        .rubyTreeprocessor("ShellSessionTreeProcessor")
        .register();

    String content = this.asciidoctor.convert(
        " $ echo \"Hello, World!\"\n" +
            " > Hello, World!\n" +
            "\n" +
            " $ gem install asciidoctor",
        options().toFile(false).get());

    final Document document = Jsoup.parse(content);
    final TextNode commandElement = document.getElementsByClass("command").get(0).textNodes().get(0);
    assertThat(commandElement.getWholeText(), is("echo \"Hello, World!\""));
    final TextNode commandElement2 = document.getElementsByClass("command").get(1).textNodes().get(0);
    assertThat(commandElement2.getWholeText(), is("gem install asciidoctor"));
}
 
Example 27
@Test
public void ruby_treeprocessor_should_be_registered() {

    final String rubyExtPath = classpath.getResource("ruby-extensions").getAbsolutePath();
    final AsciidoctorJRuby asciidoctor = AsciidoctorJRuby.Factory.create(singletonList(rubyExtPath));
    asciidoctor.rubyExtensionRegistry()
        .requireLibrary("shell-session-tree-processor.rb")
        .treeprocessor("ShellSessionTreeProcessor");

    String content = asciidoctor.convert(
        " $ echo \"Hello, World!\"\n" +
            " > Hello, World!\n" +
            "\n" +
            " $ gem install asciidoctor",
            options().toFile(false).get());

    final Document document = Jsoup.parse(content);
    final TextNode commandElement = document.getElementsByClass("command").get(0).textNodes().get(0);
    assertThat(commandElement.getWholeText(), is("echo \"Hello, World!\""));
    final TextNode commandElement2 = document.getElementsByClass("command").get(1).textNodes().get(0);
    assertThat(commandElement2.getWholeText(), is("gem install asciidoctor"));
}
 
Example 28
Source Project: xsoup   Source File: ElementOperator.java    License: MIT License 6 votes vote down vote up
@Override
public String operate(Element element) {
    int index = 0;
    StringBuilder accum = new StringBuilder();
    for (Node node : element.childNodes()) {
        if (node instanceof TextNode) {
            TextNode textNode = (TextNode) node;
            if (group == 0) {
                accum.append(textNode.text());
            } else if (++index == group) {
                return textNode.text();
            }
        }
    }
    return accum.toString();
}
 
Example 29
private String getText(Element element) {
	return element.childNodes().stream()
		.filter(node -> node instanceof TextNode)
		.map(node -> (TextNode) node)
		.map(TextNode::text)
		.collect(Collectors.joining());
}
 
Example 30
Source Project: FairEmail   Source File: HtmlHelper.java    License: GNU General Public License v3.0 5 votes vote down vote up
private static boolean hasVisibleContent(List<Node> nodes) {
    for (Node node : nodes)
        if (node instanceof TextNode && !((TextNode) node).isBlank())
            return true;
        else if (node instanceof Element) {
            Element element = (Element) node;
            if (!element.isBlock() &&
                    (element.hasText() ||
                            element.selectFirst("a") != null ||
                            element.selectFirst("img") != null))
                return true;
        }
    return false;
}