com.sun.syndication.feed.synd.SyndEntry Java Examples

private Map<Channel, List<Document>> buildChannelMap(long userId) {
    List<Source> sources;
    if (userId == PUB_USER_KEY) {
        sources = articleDao.getPublicSources();
    } else if (articleDao.hasSubscriptions(userId)) {
        sources = articleDao.getSources(userId);
    } else {
        return Collections.emptyMap();
    }

    List<RufusFeed> requests = sources.parallelStream().map(RufusFeed::generate).collect(Collectors.toList());

    Map<Channel, List<Document>> ret = new HashMap<>();
    requests.stream().filter(r -> r.getFeed() != null).forEach(r -> {
        Pair<SyndFeed, List<SyndEntry>> synd = feedPair(r);
        ret.put(Channel.of(
            synd.getKey().getTitle(),
            synd.getKey().getLanguage(),
            synd.getKey().getLink(),
            r.getSource()),
            extractDocuments(synd));
    });

    return ret;
}
 

private static List<Document> extractDocuments(Pair<SyndFeed, List<SyndEntry>> feedEntry) {
    List<Document> ret = new ArrayList<>();
    feedEntry.getRight().forEach(e -> {
        FeedUtils.mergeAuthors(e);
        String description = e.getDescription() != null ? FeedUtils.clean(e.getDescription().getValue()) : StringUtils.EMPTY;
        if (description.length() > FeedConstants.MAX_DESCRIP_LEN) {
            description = FeedUtils.truncate(description, FeedConstants.MAX_DESCRIP_LEN);
        }
        ret.add(Document.of(
            StringEscapeUtils.unescapeHtml4(e.getTitle()),
            e.getPublishedDate(),
            e.getAuthors(),
            description,
            e.getLink(),
            feedEntry.getLeft().getTitle()
        ));
    });
    return ret;
}
 

@Override
protected Object getNextRowData(AdapterRowSet rows) throws AdapterException {
   	SyndEntry entry = feediterator.next();
   	if (feediterator.hasNext() == false) {
   		hasNoMoreRows();
   	}
   	return entry;
}
 

@Override
public void setItems(List<IRssItem> items) {
    if (items == null) {
        syndFeed.setEntries(null);
        return;
    }

    List<SyndEntry> syndEntries = new ArrayList<SyndEntry>(items.size());
    for (Iterator<IRssItem> iterator = items.iterator(); iterator.hasNext(); ) {
        RomeRssItem rssItem = (RomeRssItem)iterator.next();
        syndEntries.add(rssItem.getSyndEntry());
    }

    syndFeed.setEntries(syndEntries);
}
 

@Override
public List<IRssItem> getItems() {
    List<IRssItem> result = new ArrayList<IRssItem>(syndFeed.getEntries().size());

    for (Iterator<?> iterator = syndFeed.getEntries().iterator(); iterator.hasNext(); ) {
        IRssItem rssItem = new RomeRssItem(props, this, (SyndEntry)iterator.next());
        result.add(rssItem);
    }

    return result;
}
 

protected void printHeadlines( GenericMessageEvent event,
                               LinkedHashMap<String, ImmutablePair<String, URL>> newsFeedsByCategory,
                               String category, Logger logger )
{
    SyndFeedInput input = new SyndFeedInput();
    SyndFeed feed = null;

    try
    {
        feed = input.build( new XmlReader( newsFeedsByCategory.get( category ).getRight() ) );
    }
    catch ( FeedException | IOException e )
    {
        String errorMsg = "Error opening RSS feed";
        logger.error( e.getMessage() );
        logger.error( errorMsg );
        event.respondWith( errorMsg );
    }

    List<SyndEntry> entries = feed.getEntries();

    for ( int i = 0; i < NUM_HEADLINES; i++ )
    {
        event.respondWith( Colors.bold( "  " + entries.get( i ).getTitle() ) + " -> "
            + entries.get( i ).getLink() );
    }
}
 

/**
 * Read the items of an external feed url
 * 
 * @param feedURL
 * @return The list of all items
 */
// ROME library uses untyped lists
@SuppressWarnings("unchecked")
private List<Item> getItemsFromFeed(final Feed extFeed) {
    final List<Item> items = new ArrayList<Item>();
    final SyndFeed feed = getSyndFeed(extFeed);
    if (feed != null) {
        final List<SyndEntry> entries = feed.getEntries();
        for (final SyndEntry entry : entries) {
            final Item item = convertToItem(entry);
            items.add(item);
        }
    }
    return items;
}
 

/**
 * Read the items of an external feed url
 * 
 * @param feedURL
 * @return The list of all items
 */
// ROME library uses untyped lists
@SuppressWarnings("unchecked")
private List<Item> getItemsFromFeed(final Feed extFeed) {
    final List<Item> items = new ArrayList<Item>();
    final SyndFeed feed = getSyndFeed(extFeed);
    if (feed != null) {
        final List<SyndEntry> entries = feed.getEntries();
        for (final SyndEntry entry : entries) {
            final Item item = convertToItem(entry);
            items.add(item);
        }
    }
    return items;
}
 

@SuppressWarnings("unchecked")
public void start(String rssurl) throws AdapterException {
    FeedFetcher feedFetcher = new HttpURLFeedFetcher();
	try {
		feed = feedFetcher.retrieveFeed(new URL(rssurl));
	    feediterator = (Iterator<SyndEntry>) feed.getEntries().iterator();
	} catch (IllegalArgumentException | IOException	| FeedException | FetcherException e) {
		feediterator = null;
		feed = null;
		throw new AdapterException(e);
	}
}
 

/**
 * Override equals to tell if the given object is equal to this RomeFeedEntry object.
 * Compares the title and uri of the underlying SyndEntrys of both objects to determine equality.
 *
 * @param o The given object
 * @return Whether the given object is equal to this object or not
 */
@Override
public boolean equals(Object o)
{
  boolean equal = false;
  if (o instanceof RomeFeedEntry) {
    RomeFeedEntry rfe = (RomeFeedEntry)o;
    SyndEntry syndEntry = getComponent();
    equal = syndEntry.getTitle().equals(rfe.getComponent().getTitle())
            && syndEntry.getUri().equals(rfe.getComponent().getUri());
  }
  return equal;
}
 

public static List getAllPostsFromFeed(String urlToGet, String source) throws IOException, FeedException {

        ArrayList<BlogPost> posts = new ArrayList<BlogPost>();

        URL url = new URL(urlToGet);
        SyndFeedInput input = new SyndFeedInput();
        try {
            SyndFeed feed = input.build(new XmlReader(url));

            int items = feed.getEntries().size();

            if (items > 0) {
                log.info("Attempting to parse rss feed: " + urlToGet);
                log.info("This Feed has " + items + " items");
                List<SyndEntry> entries = feed.getEntries();

                for (SyndEntry item : entries) {
                    if (item.getContents().size() > 0) {
                        SyndContentImpl contentHolder = (SyndContentImpl) item.getContents().get(0);
                        String content = contentHolder.getValue();
                        if (content != null && !content.isEmpty()) {
                            BlogPost post = new BlogPost(content, null, null, source, item.getLink(), item.getUri(), null);
                            posts.add(post);
                        }
                    }
                }
            }
            return posts;
        }
        catch(Exception ex){
            log.error(ex);
            return posts;
        }

    }
 

private static String getFeed(HttpUrl host) {
    SyndFeed feed = new SyndFeedImpl();
    feed.setFeedType("rss_2.0");
    feed.setTitle("StubbornJava");
    feed.setLink(host.toString());
    feed.setDescription("Unconventional guides, examples, and blog utilizing modern Java");

    List<PostRaw> posts = Posts.getAllRawPosts();
    List<SyndEntry> entries = Seq.seq(posts)
        .map(p -> {
            SyndEntry entry = new SyndEntryImpl();
            entry.setTitle(p.getTitle());
            entry.setLink(host.newBuilder().encodedPath(p.getUrl()).build().toString());
            entry.setPublishedDate(Date.from(p.getDateCreated()
                                              .toLocalDate()
                                              .atStartOfDay(ZoneId.systemDefault())
                                              .toInstant()));
            entry.setUpdatedDate(Date.from(p.getDateUpdated()
                                            .toLocalDate()
                                            .atStartOfDay(ZoneId.systemDefault())
                                            .toInstant()));
            SyndContentImpl description = new SyndContentImpl();
            description.setType("text/plain");
            description.setValue(p.getMetaDesc());
            entry.setDescription(description);
            return entry;
        }).toList();
    feed.setEntries(entries);

    StringWriter writer = new StringWriter();
    SyndFeedOutput output = new SyndFeedOutput();

    return Unchecked.supplier(() -> {
        output.output(feed, writer);
        return writer.toString();
    }).get();
}
 

private static String getFeed(HttpUrl host) {
    SyndFeed feed = new SyndFeedImpl();
    feed.setFeedType("rss_2.0");
    feed.setTitle("StubbornJava");
    feed.setLink(host.toString());
    feed.setDescription("Unconventional guides, examples, and blog utilizing modern Java");

    List<PostRaw> posts = Posts.getAllRawPosts();
    List<SyndEntry> entries = Seq.seq(posts)
        .map(p -> {
            SyndEntry entry = new SyndEntryImpl();
            entry.setTitle(p.getTitle());
            entry.setLink(host.newBuilder().encodedPath(p.getUrl()).build().toString());
            entry.setPublishedDate(Date.from(p.getDateCreated()
                                              .toLocalDate()
                                              .atStartOfDay(ZoneId.systemDefault())
                                              .toInstant()));
            entry.setUpdatedDate(Date.from(p.getDateUpdated()
                                            .toLocalDate()
                                            .atStartOfDay(ZoneId.systemDefault())
                                            .toInstant()));
            SyndContentImpl description = new SyndContentImpl();
            description.setType("text/plain");
            description.setValue(p.getMetaDesc());
            entry.setDescription(description);
            return entry;
        }).toList();
    feed.setEntries(entries);

    StringWriter writer = new StringWriter();
    SyndFeedOutput output = new SyndFeedOutput();

    return Unchecked.supplier(() -> {
        output.output(feed, writer);
        return writer.toString();
    }).get();
}
 

/**
    * Constructor. The identityKey is needed to generate personal URLs for the corresponding user.
    */
protected RSSFeed(final Feed feed, final Identity identity, final Long courseId, final String nodeId,final Translator translator, RepositoryService repositoryService) {
       super();

       // This helper object is required for generating the appropriate URLs for
       // the given user (identity)
	final FeedViewHelper helper = new FeedViewHelper(feed, identity, courseId, nodeId, translator, repositoryService);

       setFeedType("rss_2.0");
       setEncoding(RSSServlet.DEFAULT_ENCODING);
       setTitle(feed.getTitle());
       // According to the rss specification, the feed channel description is not
       // (explicitly) allowed to contain html tags.
       String strippedDescription = FilterFactory.getHtmlTagsFilter().filter(feed.getDescription());
       strippedDescription = strippedDescription.replaceAll(" ", " "); // TODO: remove when filter
       // does it
       setDescription(strippedDescription);
       setLink(helper.getJumpInLink());

       setPublishedDate(feed.getLastModified());
       // The image
       if (feed.getImageName() != null) {
           final SyndImage image = new SyndImageImpl();
           image.setDescription(feed.getDescription());
           image.setTitle(feed.getTitle());
           image.setLink(getLink());
           image.setUrl(helper.getImageUrl());
           setImage(image);
       }

       final List<SyndEntry> episodes = new ArrayList<SyndEntry>();
       for (final Item item : feed.getPublishedItems()) {
           final SyndEntry entry = new SyndEntryImpl();
           entry.setTitle(item.getTitle());

           final SyndContent itemDescription = new SyndContentImpl();
           itemDescription.setType("text/plain");
           itemDescription.setValue(helper.getItemDescriptionForBrowser(item));
           entry.setDescription(itemDescription);

           // Link will also be converted to the rss guid tag. Except if there's an
           // enclosure, then the enclosure url is used.
           // Use jump-in link far all entries. This will be overriden if the item
           // has an enclosure.
           entry.setLink(helper.getJumpInLink() + "#" + item.getGuid());
           entry.setPublishedDate(item.getPublishDate());
           entry.setUpdatedDate(item.getLastModified());

           // The enclosure is the media (audio or video) file of the episode
           final Enclosure media = item.getEnclosure();
           if (media != null) {
               final SyndEnclosure enclosure = new SyndEnclosureImpl();
               enclosure.setUrl(helper.getMediaUrl(item));
               enclosure.setType(media.getType());
               enclosure.setLength(media.getLength());
               // Also set the item link to point to the enclosure
               entry.setLink(helper.getMediaUrl(item));
               final List<SyndEnclosure> enclosures = new ArrayList<SyndEnclosure>();
               enclosures.add(enclosure);
               entry.setEnclosures(enclosures);
           }

           episodes.add(entry);
       }
       setEntries(episodes);
   }
 

public AbstractMap.SimpleEntry<Integer, Integer> fetchAndSave() throws Exception {

        URL url = new URL(this.url);

        SyndFeedInput input = new SyndFeedInput();
        SyndFeed feed = input.build(new XmlReader(url));


        int items = feed.getEntries().size();

        if(items > 0){
            log.info("Attempting to parse rss feed: "+ this.url );
            log.info("This Feed has "+items +" items");
        }

        List <SyndEntry> entries = feed.getEntries();

        for (SyndEntry item : entries){
            log.info("Title: " + item.getTitle());
            log.info("Link: " + item.getLink());
            SyndContentImpl contentHolder = (SyndContentImpl) item.getContents().get(0);
            String content = contentHolder.getValue();

            //content might contain html data, let's clean it up
            Document doc = Jsoup.parse(content);
            content = doc.text();
            try {
                    Result result = ld.detectLanguage(content, language);
                    if (result.languageCode.equals(language) && result.isReliable) {

                        FileSaver file = new FileSaver(content, this.language, "bs", item.getLink(), item.getUri(), String.valueOf(content.hashCode()));
                        String fileName = file.getFileName();
                        BlogPost post = new BlogPost(content,this.language,null,"bs",item.getLink(),item.getUri(),fileName);
                        if(DAO.saveEntry(post)) {
                            file.save(this.logDb);
                            numOfFiles++;
                            wrongCount = 0;
                        }

                    }

                    else{
                        log.info("Item " + item.getTitle() + "is in a diff languageCode, skipping this post  "+ result.languageCode);
                        wrongCount ++;
                        if(wrongCount > 3){
                            log.info("Already found 3 posts in the wrong languageCode, skipping this blog");
                        }
                        break;
                    }

            }
            catch(Exception e){
                log.error(e);
                break;
            }


        }
        return new AbstractMap.SimpleEntry<>(numOfFiles,wrongCount);
    }
 

/**
 * Returns wrapped SyndEntry object.
 */
public SyndEntry getSyndEntry() {
    return this.syndEntry;
}
 

/**
 * Constructs an item using specified SyndEntry-object.
 */
public RomeRssItem(TypedProperties props, RomeRssChannel parentChannel, SyndEntry syndEntry) {
    // UNSUSED: this.props = props;
    this.parentChannel = parentChannel;
    this.syndEntry = syndEntry;
}
 

public static void mergeAuthors(SyndEntry entry) {
    if (CollectionUtils.isEmpty(entry.getAuthors())) {
        entry.setAuthors(Collections.singletonList(entry.getAuthor()));
    }
}
 

/**
 * Constructor. The identityKey is needed to generate personal URLs for the corresponding user.
 */
protected RSSFeed(final Feed feed, final Identity identity, final Long courseId, final String nodeId, final Translator translator, RepositoryService repositoryService) {
    super();

    // This helper object is required for generating the appropriate URLs for
    // the given user (identity)
    final FeedViewHelper helper = new FeedViewHelper(feed, identity, courseId, nodeId, translator, repositoryService);

    setFeedType("rss_2.0");
    setEncoding(DEFAULT_ENCODING);
    setTitle(feed.getTitle());
    // According to the rss specification, the feed channel description is not
    // (explicitly) allowed to contain html tags.
    String strippedDescription = FilterFactory.getHtmlTagsFilter().filter(feed.getDescription());
    strippedDescription = strippedDescription.replaceAll(" ", " "); // TODO: remove when filter
    // does it
    setDescription(strippedDescription);
    setLink(helper.getJumpInLink());

    setPublishedDate(feed.getLastModified());
    // The image
    if (feed.getImageName() != null) {
        final SyndImage image = new SyndImageImpl();
        image.setDescription(feed.getDescription());
        image.setTitle(feed.getTitle());
        image.setLink(getLink());
        image.setUrl(helper.getImageUrl());
        setImage(image);
    }

    final List<SyndEntry> episodes = new ArrayList<SyndEntry>();
    for (final Item item : feed.getPublishedItems()) {
        final SyndEntry entry = new SyndEntryImpl();
        entry.setTitle(item.getTitle());

        final SyndContent itemDescription = new SyndContentImpl();
        itemDescription.setType("text/plain");
        itemDescription.setValue(helper.getItemDescriptionForBrowser(item));
        entry.setDescription(itemDescription);

        // Link will also be converted to the rss guid tag. Except if there's an
        // enclosure, then the enclosure url is used.
        // Use jump-in link far all entries. This will be overriden if the item
        // has an enclosure.
        entry.setLink(helper.getJumpInLink() + "#" + item.getGuid());
        entry.setPublishedDate(item.getPublishDate());
        entry.setUpdatedDate(item.getLastModified());

        // The enclosure is the media (audio or video) file of the episode
        final Enclosure media = item.getEnclosure();
        if (media != null) {
            final SyndEnclosure enclosure = new SyndEnclosureImpl();
            enclosure.setUrl(helper.getMediaUrl(item));
            enclosure.setType(media.getType());
            enclosure.setLength(media.getLength());
            // Also set the item link to point to the enclosure
            entry.setLink(helper.getMediaUrl(item));
            final List<SyndEnclosure> enclosures = new ArrayList<SyndEnclosure>();
            enclosures.add(enclosure);
            entry.setEnclosures(enclosures);
        }

        episodes.add(entry);
    }
    setEntries(episodes);
}
 

/**
 * Converts a <code>SyndEntry</code> into an <code>Item</code>
 * 
 * @param entry
 *            The SyndEntry
 * @return The Item
 */
private Item convertToItem(final SyndEntry entry) {
    // A SyncEntry can potentially have many attributes like title, description,
    // guid, link, enclosure or content. In OLAT, however, items are limited
    // to the attributes, title, description and one media file (called
    // enclosure in RSS) for simplicity.
    final Item e = new Item();
    e.setTitle(entry.getTitle());
    e.setDescription(entry.getDescription() != null ? entry.getDescription().getValue() : null);
    // Extract content objects from syndication item
    final StringBuffer sb = new StringBuffer();
    for (final SyndContent content : (List<SyndContent>) entry.getContents()) {
        // we don't check for type, assume it is html or txt
        if (sb.length() > 0) {
            sb.append("<p />");
        }
        sb.append(content.getValue());
    }
    // Set aggregated content from syndication item as our content
    if (sb.length() > 0) {
        e.setContent(sb.toString());
    }
    e.setGuid(entry.getUri());
    e.setExternalLink(entry.getLink());
    e.setLastModified(entry.getUpdatedDate());
    e.setPublishDate(entry.getPublishedDate());

    for (final Object enclosure : entry.getEnclosures()) {
        if (enclosure instanceof SyndEnclosure) {
            final SyndEnclosure syndEnclosure = (SyndEnclosure) enclosure;
            final Enclosure media = new Enclosure();
            media.setExternalUrl(syndEnclosure.getUrl());
            media.setType(syndEnclosure.getType());
            media.setLength(syndEnclosure.getLength());
            e.setEnclosure(media);
        }
        // Break after one cycle because only one media file is supported
        break;
    }
    return e;
}
 

@SuppressWarnings("unchecked")
private Pair<SyndFeed, List<SyndEntry>> feedPair(RufusFeed request) {
    SyndFeed feed = request.getFeed();
    return ImmutablePair.of(feed, feed.getEntries());
}
 

/**
 * Converts a <code>SyndEntry</code> into an <code>Item</code>
 * 
 * @param entry
 *            The SyndEntry
 * @return The Item
 */
private Item convertToItem(final SyndEntry entry) {
    // A SyncEntry can potentially have many attributes like title, description,
    // guid, link, enclosure or content. In OLAT, however, items are limited
    // to the attributes, title, description and one media file (called
    // enclosure in RSS) for simplicity.
    final Item e = new Item();
    e.setTitle(entry.getTitle());
    e.setDescription(entry.getDescription() != null ? entry.getDescription().getValue() : null);
    // Extract content objects from syndication item
    final StringBuffer sb = new StringBuffer();
    for (final SyndContent content : (List<SyndContent>) entry.getContents()) {
        // we don't check for type, assume it is html or txt
        if (sb.length() > 0) {
            sb.append("<p />");
        }
        sb.append(content.getValue());
    }
    // Set aggregated content from syndication item as our content
    if (sb.length() > 0) {
        e.setContent(sb.toString());
    }
    e.setGuid(entry.getUri());
    e.setExternalLink(entry.getLink());
    e.setLastModified(entry.getUpdatedDate());
    e.setPublishDate(entry.getPublishedDate());

    for (final Object enclosure : entry.getEnclosures()) {
        if (enclosure instanceof SyndEnclosure) {
            final SyndEnclosure syndEnclosure = (SyndEnclosure) enclosure;
            final Enclosure media = new Enclosure();
            media.setExternalUrl(syndEnclosure.getUrl());
            media.setType(syndEnclosure.getType());
            media.setLength(syndEnclosure.getLength());
            e.setEnclosure(media);
        }
        // Break after one cycle because only one media file is supported
        break;
    }
    return e;
}
 

@Override
protected void setColumnValue(int tablecolumnindex, int returncolumnindex, AdapterRow row, Object o) throws AdapterException {
	SyndEntry entry = (SyndEntry) o; 
	//        col.setName("URI");
	//        col.setName("AUTHOR");
	//        col.setName("LINK");
	//        col.setName("PUBLISHEDDATE");
	//        col.setName("TITLE");
	//        col.setName("UPDATEDATE");
	//        col.setName("DESCRIPTION");
   	switch (tablecolumnindex) {
   	case 0:
   		row.setColumnValue(returncolumnindex, checkStringLength(entry.getUri(), 512));
   		break;
   	case 1:
   		row.setColumnValue(returncolumnindex, checkStringLength(entry.getAuthor(), 255));
   		break;
   	case 2:
   		row.setColumnValue(returncolumnindex, checkStringLength(entry.getLink(), 512));
   		break;
   	case 3:
		if (entry.getPublishedDate() != null)
			row.setColumnValue(returncolumnindex, new Timestamp(entry.getPublishedDate()));
		else
			row.setColumnNull(returncolumnindex);
   		break;
   	case 4:
   		row.setColumnValue(returncolumnindex, checkStringLength(entry.getTitle(), 255));
   		break;
   	case 5:
		if (entry.getUpdatedDate() != null)
			row.setColumnValue(returncolumnindex, new Timestamp(entry.getUpdatedDate()));
		else
			row.setColumnNull(returncolumnindex);
   		break;
   	case 6:
		if (entry.getDescription() != null) {
			row.setColumnValue(returncolumnindex, checkStringLength(entry.getDescription().getValue(), 5000));
		} else
			row.setColumnNull(returncolumnindex);
   		break;
   	}
}
 

@Override
public int compareTo(SyndEntry syndEntry2) {
	return getPublishedDate().compareTo(syndEntry2.getPublishedDate());
}
 

/**
 * Create a RomeFeedEntry using a Rome SyndEntry object.
 *
 * @param syndEntry The Rome SyndEntry object
 */
public RomeFeedEntry(SyndEntry syndEntry)
{
  //this.syndEntry = syndEntry;
  super(syndEntry);
}
 

/**
 * Get a serializable syndEntry for the given syndEntry.
 * Not all implementations of syndEntry are serializable according to rome
 * documentation. This method creates and returns a copy of the original
 * syndEntry that is java serializable.
 *
 * @param syndEntry The syndEntry to create a serializable copy of
 * @return The serializable copy syndEntry
 */
private RomeFeedEntry getSerializableEntry(SyndEntry syndEntry)
{
  SyndEntry serSyndEntry = new SyndEntryImpl();
  serSyndEntry.copyFrom(syndEntry);
  //return serSyndEntry;
  RomeFeedEntry romeFeedEntry = new RomeFeedEntry(serSyndEntry);
  return romeFeedEntry;
}
 

/**
 * Set the Rome SyndEntry object.
 *
 * @param syndEntry The SyndEntry object
 */
public void setSyndEntry(SyndEntry syndEntry)
{
  setComponent(syndEntry);
}
 

/**
 * Get the Rome SyndEntry object.
 *
 * @return The SyndEntry object
 */
public SyndEntry getSyndEntry()
{
  return getComponent();
}