com.sun.syndication.feed.synd.SyndFeed Java Examples

private Map<Channel, List<Document>> buildChannelMap(long userId) {
    List<Source> sources;
    if (userId == PUB_USER_KEY) {
        sources = articleDao.getPublicSources();
    } else if (articleDao.hasSubscriptions(userId)) {
        sources = articleDao.getSources(userId);
    } else {
        return Collections.emptyMap();
    }

    List<RufusFeed> requests = sources.parallelStream().map(RufusFeed::generate).collect(Collectors.toList());

    Map<Channel, List<Document>> ret = new HashMap<>();
    requests.stream().filter(r -> r.getFeed() != null).forEach(r -> {
        Pair<SyndFeed, List<SyndEntry>> synd = feedPair(r);
        ret.put(Channel.of(
            synd.getKey().getTitle(),
            synd.getKey().getLanguage(),
            synd.getKey().getLink(),
            r.getSource()),
            extractDocuments(synd));
    });

    return ret;
}
 

private static List<Document> extractDocuments(Pair<SyndFeed, List<SyndEntry>> feedEntry) {
    List<Document> ret = new ArrayList<>();
    feedEntry.getRight().forEach(e -> {
        FeedUtils.mergeAuthors(e);
        String description = e.getDescription() != null ? FeedUtils.clean(e.getDescription().getValue()) : StringUtils.EMPTY;
        if (description.length() > FeedConstants.MAX_DESCRIP_LEN) {
            description = FeedUtils.truncate(description, FeedConstants.MAX_DESCRIP_LEN);
        }
        ret.add(Document.of(
            StringEscapeUtils.unescapeHtml4(e.getTitle()),
            e.getPublishedDate(),
            e.getAuthors(),
            description,
            e.getLink(),
            feedEntry.getLeft().getTitle()
        ));
    });
    return ret;
}
 

private static String getFeed(HttpUrl host) {
    SyndFeed feed = new SyndFeedImpl();
    feed.setFeedType("rss_2.0");
    feed.setTitle("StubbornJava");
    feed.setLink(host.toString());
    feed.setDescription("Unconventional guides, examples, and blog utilizing modern Java");

    List<PostRaw> posts = Posts.getAllRawPosts();
    List<SyndEntry> entries = Seq.seq(posts)
        .map(p -> {
            SyndEntry entry = new SyndEntryImpl();
            entry.setTitle(p.getTitle());
            entry.setLink(host.newBuilder().encodedPath(p.getUrl()).build().toString());
            entry.setPublishedDate(Date.from(p.getDateCreated()
                                              .toLocalDate()
                                              .atStartOfDay(ZoneId.systemDefault())
                                              .toInstant()));
            entry.setUpdatedDate(Date.from(p.getDateUpdated()
                                            .toLocalDate()
                                            .atStartOfDay(ZoneId.systemDefault())
                                            .toInstant()));
            SyndContentImpl description = new SyndContentImpl();
            description.setType("text/plain");
            description.setValue(p.getMetaDesc());
            entry.setDescription(description);
            return entry;
        }).toList();
    feed.setEntries(entries);

    StringWriter writer = new StringWriter();
    SyndFeedOutput output = new SyndFeedOutput();

    return Unchecked.supplier(() -> {
        output.output(feed, writer);
        return writer.toString();
    }).get();
}
 

/**
*/
  @Override
  public MediaResource createFeedFile(final OLATResourceable ores, final Identity identity, final Long courseId, final String nodeId, final Translator translator) {
      MediaResource media = null;
      final Feed feed = getFeed(ores);

      if (feed != null) {
          final SyndFeed rssFeed = new RSSFeed(feed, identity, courseId, nodeId, translator, repositoryService);
          media = new SyndFeedMediaResource(rssFeed);
      }
      return media;
  }
 

/**
 * @param extFeed
 * @param feed
 */
private void addExternalImageURL(final SyndFeed feed, final Feed extFeed) {
    final SyndImage img = feed.getImage();
    if (img != null) {
        extFeed.setExternalImageURL(img.getUrl());
    } else {
        extFeed.setExternalImageURL(null);
    }
}
 

/**
 * Read the items of an external feed url
 * 
 * @param feedURL
 * @return The list of all items
 */
// ROME library uses untyped lists
@SuppressWarnings("unchecked")
private List<Item> getItemsFromFeed(final Feed extFeed) {
    final List<Item> items = new ArrayList<Item>();
    final SyndFeed feed = getSyndFeed(extFeed);
    if (feed != null) {
        final List<SyndEntry> entries = feed.getEntries();
        for (final SyndEntry entry : entries) {
            final Item item = convertToItem(entry);
            items.add(item);
        }
    }
    return items;
}
 

public SyndFeedMediaResource(final SyndFeed feed) {
    this.feed = feed;

    feedString = null;
    try {
        final SyndFeedOutput output = new SyndFeedOutput();
        feedString = output.outputString(feed);
    } catch (final FeedException e) {
        /* TODO: ORID-1007 ExceptionHandling */
        log.error(e.getMessage());
    }
}
 

/**
 * Creates a personal RSS document
 * 
 * @param pathInfo
 * @return RssDocument
 */
private SyndFeed getPersonalFeed(final String pathInfo) {
    // pathInfo is like /personal/username/tokenid/olat.rss
    final int startIdName = RSSUtil.RSS_PREFIX_PERSONAL.length();
    final int startIdToken = pathInfo.indexOf("/", RSSUtil.RSS_PREFIX_PERSONAL.length());
    final String idName = pathInfo.substring(startIdName, startIdToken);
    final int startUselessUri = pathInfo.indexOf("/", startIdToken + 1);
    final String idToken = pathInfo.substring(startIdToken + 1, startUselessUri);

    // ---- check integrity and user authentication ----
    if (idName == null || idName.equals("")) {
        return null;
    }
    final BaseSecurity baseSecurity = (BaseSecurity) CoreSpringFactory.getBean(BaseSecurity.class);
    final Identity identity = baseSecurity.findIdentityByName(idName);
    if (identity == null) {
        // error - abort
        return null;
    }
    // check if this is a valid authentication
    final Authentication auth = baseSecurity.findAuthentication(identity, RSSUtil.RSS_AUTH_PROVIDER);
    if (auth == null) {
        // error, rss authentication not yet set. user must login first, then the
        // auth provider will be generated on the fly
        return null;
    }
    if (!auth.getCredential().equals(idToken)) {
        // error - wrong authentication
        return null;
    }
    return new PersonalRSSFeed(identity, idToken);
}
 

/**
*/
  @Override
  public MediaResource createFeedFile(final OLATResourceable ores, final Identity identity, final Long courseId, final String nodeId, final Translator translator) {
      MediaResource media = null;
      final Feed feed = getFeed(ores);

      if (feed != null) {
          final SyndFeed rssFeed = new RSSFeed(feed, identity, courseId, nodeId, translator, repositoryService);
          media = new SyndFeedMediaResource(rssFeed);
      }
      return media;
  }
 

/**
 * @param extFeed
 * @param feed
 */
private void addExternalImageURL(final SyndFeed feed, final Feed extFeed) {
    final SyndImage img = feed.getImage();
    if (img != null) {
        extFeed.setExternalImageURL(img.getUrl());
    } else {
        extFeed.setExternalImageURL(null);
    }
}
 

/**
 * Read the items of an external feed url
 * 
 * @param feedURL
 * @return The list of all items
 */
// ROME library uses untyped lists
@SuppressWarnings("unchecked")
private List<Item> getItemsFromFeed(final Feed extFeed) {
    final List<Item> items = new ArrayList<Item>();
    final SyndFeed feed = getSyndFeed(extFeed);
    if (feed != null) {
        final List<SyndEntry> entries = feed.getEntries();
        for (final SyndEntry entry : entries) {
            final Item item = convertToItem(entry);
            items.add(item);
        }
    }
    return items;
}
 

public SyndFeedMediaResource(final SyndFeed feed) {
    this.feed = feed;

    feedString = null;
    try {
        final SyndFeedOutput output = new SyndFeedOutput();
        feedString = output.outputString(feed);
    } catch (final FeedException e) {
        /* TODO: ORID-1007 ExceptionHandling */
        log.error(e.getMessage());
    }
}
 

private static String getFeed(HttpUrl host) {
    SyndFeed feed = new SyndFeedImpl();
    feed.setFeedType("rss_2.0");
    feed.setTitle("StubbornJava");
    feed.setLink(host.toString());
    feed.setDescription("Unconventional guides, examples, and blog utilizing modern Java");

    List<PostRaw> posts = Posts.getAllRawPosts();
    List<SyndEntry> entries = Seq.seq(posts)
        .map(p -> {
            SyndEntry entry = new SyndEntryImpl();
            entry.setTitle(p.getTitle());
            entry.setLink(host.newBuilder().encodedPath(p.getUrl()).build().toString());
            entry.setPublishedDate(Date.from(p.getDateCreated()
                                              .toLocalDate()
                                              .atStartOfDay(ZoneId.systemDefault())
                                              .toInstant()));
            entry.setUpdatedDate(Date.from(p.getDateUpdated()
                                            .toLocalDate()
                                            .atStartOfDay(ZoneId.systemDefault())
                                            .toInstant()));
            SyndContentImpl description = new SyndContentImpl();
            description.setType("text/plain");
            description.setValue(p.getMetaDesc());
            entry.setDescription(description);
            return entry;
        }).toList();
    feed.setEntries(entries);

    StringWriter writer = new StringWriter();
    SyndFeedOutput output = new SyndFeedOutput();

    return Unchecked.supplier(() -> {
        output.output(feed, writer);
        return writer.toString();
    }).get();
}
 

protected void printHeadlines( GenericMessageEvent event,
                               LinkedHashMap<String, ImmutablePair<String, URL>> newsFeedsByCategory,
                               String category, Logger logger )
{
    SyndFeedInput input = new SyndFeedInput();
    SyndFeed feed = null;

    try
    {
        feed = input.build( new XmlReader( newsFeedsByCategory.get( category ).getRight() ) );
    }
    catch ( FeedException | IOException e )
    {
        String errorMsg = "Error opening RSS feed";
        logger.error( e.getMessage() );
        logger.error( errorMsg );
        event.respondWith( errorMsg );
    }

    List<SyndEntry> entries = feed.getEntries();

    for ( int i = 0; i < NUM_HEADLINES; i++ )
    {
        event.respondWith( Colors.bold( "  " + entries.get( i ).getTitle() ) + " -> "
            + entries.get( i ).getLink() );
    }
}
 

public static RufusFeed generate(Source source) {
    SyndFeedInput input = new SyndFeedInput();
    SyndFeed feed = null;
    URL url = source.getUrl();
    try {
        feed = input.build(new XmlReader(url));
    } catch (Exception e) {
        logger.debug("Could not build SyndFeedInput for {}", url, e);
    }
    if (CollectionUtils.isEmpty(source.getTags())) {
        source.setTags(Collections.emptyList()); //never null!
    }

    return new RufusFeed(source, feed);
}
 

@Override
public void writeRss(Writer out, IRssChannel rssChannel) throws RssException, IOException {
    SyndFeed syndFeed = ((RomeRssChannel)rssChannel).getSyndFeed();
    SyndFeedOutput syndFeedOutput = new SyndFeedOutput();
    try {
        syndFeedOutput.output(syndFeed, out);
    } catch (FeedException e) {
        String msg = "Cannot write RSS feed.";
        log.error(msg, e);
        throw new RssException(msg, e);
    }
}
 

public static List getAllPostsFromFeed(String urlToGet, String source) throws IOException, FeedException {

        ArrayList<BlogPost> posts = new ArrayList<BlogPost>();

        URL url = new URL(urlToGet);
        SyndFeedInput input = new SyndFeedInput();
        try {
            SyndFeed feed = input.build(new XmlReader(url));

            int items = feed.getEntries().size();

            if (items > 0) {
                log.info("Attempting to parse rss feed: " + urlToGet);
                log.info("This Feed has " + items + " items");
                List<SyndEntry> entries = feed.getEntries();

                for (SyndEntry item : entries) {
                    if (item.getContents().size() > 0) {
                        SyndContentImpl contentHolder = (SyndContentImpl) item.getContents().get(0);
                        String content = contentHolder.getValue();
                        if (content != null && !content.isEmpty()) {
                            BlogPost post = new BlogPost(content, null, null, source, item.getLink(), item.getUri(), null);
                            posts.add(post);
                        }
                    }
                }
            }
            return posts;
        }
        catch(Exception ex){
            log.error(ex);
            return posts;
        }

    }
 

/**
 * Returns wrapped SyndFeed object.
 */
public SyndFeed getSyndFeed() {
    return this.syndFeed;
}
 

@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {

    String hubtopic = null;
    String hubchallenge = null;
    MessageStatus stsMessage = MessageStatus.ERROR;

    ClassLoader cl = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(SyndFeedInput.class.getClassLoader());

    if (request.getContentType().contains(MediaType.APPLICATION_ATOM_XML)
            || request.getContentType().contains(MediaType.APPLICATION_RSS_XML)) {

        InputStream in = request.getInputStream();

        try {
            SyndFeedInput input = new SyndFeedInput();
            SyndFeed feed = input.build(new XmlReader(in));

            List<SyndLinkImpl> linkList = feed.getLinks();

            for (SyndLinkImpl link : linkList) {

                if ("self".equals(link.getRel())) {
                    hubtopic = link.getHref();
                }
            }

            if (hubtopic == null) {
                hubtopic = feed.getUri();
            }

            topicUpdated(hubtopic, feed);
        } catch (FeedException e) {
            LOG.error("Failed in creating feed response.", e);
        } finally {
            Thread.currentThread().setContextClassLoader(cl);
        }


        stsMessage = MessageStatus.OK;
    }

    response.setContentType(MediaType.APPLICATION_FORM_URLENCODED);

    switch (stsMessage) {
        case OK:
            response.setStatus(HttpStatus.SC_OK);
            break;
        case OK_CHALLENGE:
            response.setStatus(HttpStatus.SC_OK);
            response.getWriter().print(hubchallenge);
            break;
        default:
            response.setStatus(HttpStatus.SC_NOT_FOUND);
            break;
    }

}
 

/**
 * Constructs this feed from specified parsed feed object.
 */
public RomeRssChannel(TypedProperties props, SyndFeed syndFeed) {
    this.props = props;
    this.syndFeed = syndFeed;
}
 

public List<SyndFeed> getFeeds() {
    return this.feeds;
}
 

@SuppressWarnings("unchecked")
private Pair<SyndFeed, List<SyndEntry>> feedPair(RufusFeed request) {
    SyndFeed feed = request.getFeed();
    return ImmutablePair.of(feed, feed.getEntries());
}
 

public SyndFeed getFeed() {
    return feed;
}
 

private RufusFeed(Source source, SyndFeed feed) {
    this.feed = feed;
    this.source = source;
}
 

public static ClientSource ofExisting(Source source) {
    SyndFeed syndFeed = RufusFeed.generate(source).getFeed();
    return new ClientSource(FeedUtils.clean(syndFeed.getTitle()), source.getUrl().toExternalForm(), source.isFrontpage());
}
 

public AbstractMap.SimpleEntry<Integer, Integer> fetchAndSave() throws Exception {

        URL url = new URL(this.url);

        SyndFeedInput input = new SyndFeedInput();
        SyndFeed feed = input.build(new XmlReader(url));


        int items = feed.getEntries().size();

        if(items > 0){
            log.info("Attempting to parse rss feed: "+ this.url );
            log.info("This Feed has "+items +" items");
        }

        List <SyndEntry> entries = feed.getEntries();

        for (SyndEntry item : entries){
            log.info("Title: " + item.getTitle());
            log.info("Link: " + item.getLink());
            SyndContentImpl contentHolder = (SyndContentImpl) item.getContents().get(0);
            String content = contentHolder.getValue();

            //content might contain html data, let's clean it up
            Document doc = Jsoup.parse(content);
            content = doc.text();
            try {
                    Result result = ld.detectLanguage(content, language);
                    if (result.languageCode.equals(language) && result.isReliable) {

                        FileSaver file = new FileSaver(content, this.language, "bs", item.getLink(), item.getUri(), String.valueOf(content.hashCode()));
                        String fileName = file.getFileName();
                        BlogPost post = new BlogPost(content,this.language,null,"bs",item.getLink(),item.getUri(),fileName);
                        if(DAO.saveEntry(post)) {
                            file.save(this.logDb);
                            numOfFiles++;
                            wrongCount = 0;
                        }

                    }

                    else{
                        log.info("Item " + item.getTitle() + "is in a diff languageCode, skipping this post  "+ result.languageCode);
                        wrongCount ++;
                        if(wrongCount > 3){
                            log.info("Already found 3 posts in the wrong languageCode, skipping this blog");
                        }
                        break;
                    }

            }
            catch(Exception e){
                log.error(e);
                break;
            }


        }
        return new AbstractMap.SimpleEntry<>(numOfFiles,wrongCount);
    }
 

public void topicUpdated(String hubtopic, SyndFeed feed) {

        String queue = importDeclaration.getMetadata().get("push.eventAdmin.queue").toString();

        LOG.info("(subscriber), received updated content for {}, sending through eventAdmin in the queue {}", hubtopic, queue);

        Dictionary properties = new Hashtable();
        properties.put("topic", hubtopic);
        properties.put("content", feed.toString());

        Event eventAdminMessage = new Event(queue, properties);

        eventAdmin.sendEvent(eventAdminMessage);

    }
 

void topicUpdated(String hubtopic, SyndFeed feed);