Java Code Examples for com.sun.syndication.io.SyndFeedInput

The following are top voted examples for showing how to use com.sun.syndication.io.SyndFeedInput. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: parabuild-ci   File: FeedReader.java   View source code 6 votes vote down vote up
public static void main(String[] args) {
    boolean ok = false;
    if (args.length==1) {
        try {
            URL feedUrl = new URL(args[0]);
            SyndFeedInput input = new SyndFeedInput();

            SyndFeed feed = input.build(new XmlReader(feedUrl));

            System.out.println(feed);

            ok = true;
        }
        catch (Exception ex) {
            ex.printStackTrace();
            System.out.println("ERROR: "+ex.getMessage());
        }
    }

    if (!ok) {
        System.out.println();
        System.out.println("FeedReader reads and prints any RSS/Atom feed type.");
        System.out.println("The first parameter must be the URL of the feed to read.");
        System.out.println();
    }
}
 
Example 2
Project: dwnewsbot   File: FetchAtomWorker.java   View source code 6 votes vote down vote up
public void execute(JobExecutionContext context) throws JobExecutionException {
    log.info("Start FetchAtomWorker job");

    EntityManagerFactory emf = (EntityManagerFactory) context.getMergedJobDataMap().get("emf");
    DataManager dm = new DataManager (emf);
    ServletContext scontext = (ServletContext) context.getMergedJobDataMap().get("scontext");

    Iterator it = dm.feeds.entrySet().iterator();
    while (it.hasNext()) {
        Map.Entry <String, String[]> pair = (Map.Entry)it.next();
        System.out.println(pair.getKey() + " = " + pair.getValue());

        try {
            URL feedUrl = new URL(pair.getValue()[1]);

            SyndFeedInput input = new SyndFeedInput();
            SyndFeed feed = input.build(new XmlReader(feedUrl));
            for (SyndEntry entry : (List<SyndEntry>) feed.getEntries()) {
                dm.saveNewsItem(entry, pair.getKey());
            }
        } catch (Exception ex) {
            ex.printStackTrace();
        }
    }
}
 
Example 3
Project: FeedExpander   File: FeedCreatorTest.java   View source code 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testCreateNullHeadlineAndLink() throws Exception {
	String createdFeed = new String(createFeedCreator("feeds/valid_page/index.html")
			.createFeed(".headline,.article,.footer", null, null, null)
			.setTitle("Title")
			.build());
	SyndFeedInput input = new SyndFeedInput();
	SyndFeed feed = input.build(new XmlReader(new ByteArrayInputStream(createdFeed.getBytes())));
	assertEquals("Title", feed.getTitle());
	
	List<SyndEntry> entries = feed.getEntries();

	assertEquals(null, entries.get(0).getTitle());
	assertEquals(null, entries.get(0).getLink());
	assertEquals(EMPTY, entries.get(0).getAuthor());
	assertEquals(null, entries.get(1).getTitle());
	assertEquals(null, entries.get(1).getLink());
	assertEquals(EMPTY, entries.get(1).getAuthor());
}
 
Example 4
Project: bingesurfing   File: Feeder.java   View source code 6 votes vote down vote up
public void read() {
	try {
		URL source = new URL(Defaults.FEEDURL);
		SyndFeedInput input = new SyndFeedInput();
		SyndFeed feed = input.build(new XmlReader(source));
		@SuppressWarnings("unchecked")
		List<SyndEntry> entries = feed.getEntries();
		urls = new ArrayList<String>();
		for(SyndEntry entry : entries) {
			String sURL = entry.getLink();
			urls.add(sURL);
		} // for
	} catch (IllegalArgumentException | FeedException | IOException e) {
		e.printStackTrace();
	}

	long seed = System.nanoTime();
	Collections.shuffle(urls, new Random(seed));
}
 
Example 5
Project: PrimefacesShowcase   File: TwitterRSSService.java   View source code 6 votes vote down vote up
public List<String> getTweets(String username) {
	List<String> tweets = new ArrayList<String>();
	
	try {
		URL feedSource = new URL("http://api.twitter.com/1/statuses/user_timeline.rss?count=100&screen_name=" + username);
           
		SyndFeedInput input = new SyndFeedInput();
		SyndFeed feed = input.build(new XmlReader(feedSource));
		for(Object f : feed.getEntries()) {
			SyndEntry entry = (SyndEntry) f;
			tweets.add(entry.getTitle().substring(username.length() + 2));
		}
	}catch(Exception e) {
		logger.severe(e.getMessage());
	}
	
	return tweets;
}
 
Example 6
Project: PrimefacesShowcase   File: YAHOOWeatherService.java   View source code 6 votes vote down vote up
public String getConditions(String city, String unit) {

		try {
			URL feedSource = new URL("http://weather.yahooapis.com/forecastrss?p=" + city + "&u=" + unit);

			SyndFeedInput input = new SyndFeedInput();
			SyndFeed feed = input.build(new XmlReader(feedSource));
			String value = ((SyndEntry) feed.getEntries().get(0)).getDescription().getValue();
			
			return value.split("<a href")[0];		//Remove links
		} catch (Exception e) {
			logger.severe(e.getMessage());
		}
		
		return "Unable to retrieve conditions";
	}
 
Example 7
Project: XPagesToolkit   File: FeedReaderService.java   View source code 6 votes vote down vote up
public List<RSSEntry> getAllEntriesFromURL(String strURL) {
	List<RSSEntry> lstRC = new ArrayList<RSSEntry>();
	if (!StringUtil.isEmpty(strURL)) {
		Thread currentThread = Thread.currentThread();
		ClassLoader clCurrent = currentThread.getContextClassLoader();

		try {
			currentThread.setContextClassLoader(XPTRSSActivator.class.getClassLoader());

			URL feedUrl = new URL(strURL);
			SyndFeedInput input = new SyndFeedInput();
			SyndFeed feed = input.build(new XmlReader(feedUrl));
			lstRC = processFeed2List(feed);

		} catch (Exception e) {
			LoggerFactory.logError(getClass(), "getAllEntriesFromURL", e);
		} finally {
			currentThread.setContextClassLoader(clCurrent);
		}
	}
	return lstRC;
}
 
Example 8
Project: XPagesToolkit   File: FeedReaderService.java   View source code 6 votes vote down vote up
public RSSFeed getFeedFromURL(String strURL) {
	RSSFeed rssFeed = new RSSFeed();
	rssFeed.setURL(strURL);
	Thread currentThread = Thread.currentThread();
	ClassLoader clCurrent = currentThread.getContextClassLoader();
	try {
		currentThread.setContextClassLoader(XPTRSSActivator.class.getClassLoader());
		URL feedUrl = new URL(strURL);
		SyndFeedInput input = new SyndFeedInput();
		SyndFeed feed = input.build(new XmlReader(feedUrl));
		rssFeed.setAuthor(feed.getAuthor());
		rssFeed.setDescription(feed.getDescription());
		rssFeed.setTitle(feed.getTitle());
		if (feed.getImage() != null) {
			rssFeed.setImageURL(feed.getImage().getUrl());
		}
		rssFeed.setEntries(processFeed2List(feed));
	} catch (Exception e) {
		LoggerFactory.logError(getClass(), "getFeedFromURL", e);
	} finally {
		currentThread.setContextClassLoader(clCurrent);
	}
	return rssFeed;
}
 
Example 9
Project: RSS-Mailer   File: RssReader.java   View source code 6 votes vote down vote up
/**
 * Retrieve the last available feed item, null if it's already in memory.
 * @return the last available feed item
 */
public FeedItem getFeed() {
	FeedItem feedItem = null;
	
	try {
		reader = new XmlReader(url);
		SyndFeed feed = new SyndFeedInput().build(reader);
		SyndEntry newEntry = ((SyndEntry)feed.getEntries().get(0));
		
		if(lastEntry == null || newEntry.getPublishedDate().after(lastEntry.getPublishedDate())) {
			log.info("Retrieving new feed.");
			lastEntry = newEntry;
			feedItem = populateFeedItem(lastEntry);
		}
		
	} catch (IllegalArgumentException | FeedException | IOException e) {
		log.error("There's a problem trying to read feed", e.getMessage());
	}
	
	return feedItem;
}
 
Example 10
Project: Babler   File: RSSScraper.java   View source code 5 votes vote down vote up
public static List getAllPostsFromFeed(String urlToGet, String source) throws IOException, FeedException {

        ArrayList<BlogPost> posts = new ArrayList<BlogPost>();

        URL url = new URL(urlToGet);
        SyndFeedInput input = new SyndFeedInput();
        try {
            SyndFeed feed = input.build(new XmlReader(url));

            int items = feed.getEntries().size();

            if (items > 0) {
                log.info("Attempting to parse rss feed: " + urlToGet);
                log.info("This Feed has " + items + " items");
                List<SyndEntry> entries = feed.getEntries();

                for (SyndEntry item : entries) {
                    if (item.getContents().size() > 0) {
                        SyndContentImpl contentHolder = (SyndContentImpl) item.getContents().get(0);
                        String content = contentHolder.getValue();
                        if (content != null && !content.isEmpty()) {
                            BlogPost post = new BlogPost(content, null, null, source, item.getLink(), item.getUri(), null);
                            posts.add(post);
                        }
                    }
                }
            }
            return posts;
        }
        catch(Exception ex){
            log.error(ex);
            return posts;
        }

    }
 
Example 11
Project: rufus   File: RufusFeed.java   View source code 5 votes vote down vote up
public static RufusFeed generate(Source source) {
    SyndFeedInput input = new SyndFeedInput();
    SyndFeed feed = null;
    URL url = source.getUrl();
    try {
        feed = input.build(new XmlReader(url));
    } catch (Exception e) {
        logger.debug("Could not build SyndFeedInput for {}", url, e);
    }
    if (CollectionUtils.isEmpty(source.getTags())) {
        source.setTags(Collections.emptyList()); //never null!
    }

    return new RufusFeed(source, feed);
}
 
Example 12
Project: rufus   File: FeedParser.java   View source code 5 votes vote down vote up
private static FeedResponse validate(String feedRequestUrl) {
    try {
        URL url = new URL(feedRequestUrl);
        SyndFeedInput input = new SyndFeedInput();
        input.build(new XmlReader(url)); //ensure request is a valid rss feed
        return FeedResponse.valid(feedRequestUrl);
    } catch (Exception e) {
        logger.debug("could not parse feed request {}, reason {}", feedRequestUrl, e.getMessage());
        return FeedResponse.invalid(e.getMessage(), feedRequestUrl);
    }
}
 
Example 13
Project: parabuild-ci   File: FeedConverter.java   View source code 5 votes vote down vote up
public static void main(String[] args) {
    boolean ok = false;
    if (args.length==2) {
        try {
            String outputType = args[0];
            URL feedUrl = new URL(args[1]);
            SyndFeedInput input = new SyndFeedInput();

            SyndFeed feed = input.build(new XmlReader(feedUrl));

            feed.setFeedType(outputType);
            SyndFeedOutput output = new SyndFeedOutput();
            output.output(feed,new PrintWriter(System.out));

            ok = true;
        }
        catch (Exception ex) {
            System.out.println("ERROR: "+ex.getMessage());
        }
    }

    if (!ok) {
        System.out.println();
        System.out.println("Converter converts between syndication feeds types.");
        System.out.println("The first parameter must be the feed type to convert to.");
        System.out.println(" [valid values are: rss_0.9, rss_0.91, rss_0.92, rss_0.93, ]");
        System.out.println(" [                  rss_0.94, rss_1.0, rss_2.0 & atom_0.3  ]");
        System.out.println("The second parameter must be the URL of the feed to convert.");
        System.out.println();
    }
}
 
Example 14
Project: FTBCurseBot   File: RssUtils.java   View source code 5 votes vote down vote up
@Nullable
public static SyndFeed getFeed (String url) {
    try {
        SyndFeedInput input = new SyndFeedInput();
        SyndFeed feed = input.build(new XmlReader(new URL(url)));
        return feed;
    } catch (FeedException | IOException e) {
        e.printStackTrace();
        //TODO implement this
    }
    return null;
}
 
Example 15
Project: feedsucker   File: RomeFeedReader.java   View source code 5 votes vote down vote up
public SyndFeed createFeed(String feedUrl) throws IOException, FeedException {        
    SyndFeedInput input = new SyndFeedInput();       
    SyndFeed feed = null; urlConn = null;
    if (userAgent == null) { // use default user agend
        feed = input.build(new XmlReader(new URL(feedUrl)));
    } 
    else { // setup user agend explicitly
        urlConn = new URL(feedUrl).openConnection();            
        urlConn.setRequestProperty("User-Agent", userAgent);        
        feed = input.build(new XmlReader(urlConn));
    }        
    return feed;
}
 
Example 16
Project: Camel   File: RssUtils.java   View source code 5 votes vote down vote up
public static SyndFeed createFeed(String feedUri, ClassLoader classLoader) throws Exception {
    ClassLoader tccl = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(classLoader);
        InputStream in = new URL(feedUri).openStream();
        SyndFeedInput input = new SyndFeedInput();
        return input.build(new XmlReader(in));
    } finally {
        Thread.currentThread().setContextClassLoader(tccl);
    }
}
 
Example 17
Project: Camel   File: RssUtils.java   View source code 5 votes vote down vote up
public static SyndFeed createFeed(String feedUri, String username, String password, ClassLoader classLoader) throws Exception {
    ClassLoader tccl = Thread.currentThread().getContextClassLoader();
    try {
        Thread.currentThread().setContextClassLoader(classLoader);
        URL feedUrl = new URL(feedUri);
        HttpURLConnection httpcon = (HttpURLConnection) feedUrl.openConnection();
        String encoding = Base64.encodeBase64String(username.concat(":").concat(password).getBytes());
        httpcon.setRequestProperty("Authorization", "Basic " + encoding);
        SyndFeedInput input = new SyndFeedInput();
        return input.build(new XmlReader(httpcon));
    } finally {
        Thread.currentThread().setContextClassLoader(tccl);
    }
}
 
Example 18
Project: applets-java-api   File: RssNewsFetcher.java   View source code 5 votes vote down vote up
@Override
public void fetchSources() {

    if (!this.sourceDao.isExisting(this.key)) {
        try {

            SyndFeedInput input = new SyndFeedInput();
            SyndFeed feed = input.build(new XmlReader(new URL(this.value)));

            Source source = new Source();

            String imageUrl = feed.getImage().getUrl(); // Return null if no image

            source.setType("rss");

            source.setKey(this.key);
            source.setName(this.key);
            source.setValue(this.value);
            source.setUrlImage(imageUrl);

            this.sourceDao.add(source);

        } catch (IOException | FeedException e) {
            e.printStackTrace();
        }
    }
}
 
Example 19
Project: joyrest   File: FeedDownloader.java   View source code 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public List<SyndEntry> apply(URL url) {
    try (XmlReader xmlReader = new XmlReader(url)) {
        SyndFeedInput input = new SyndFeedInput();
        SyndFeed feed = input.build(xmlReader);
        return feed.getEntries();
    } catch (Exception e) {
        LOG.log(Level.WARNING, "An error during downloading and parsing a given feed: " + url, e);
    }
    return Collections.emptyList();
}
 
Example 20
Project: FeedExpander   File: FeedCreatorTest.java   View source code 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testCreateSuccess() throws Exception {
	String createdFeed = new String(createFeedCreator("feeds/valid_page/index.html")
			.createFeed(".headline,.article,.footer", ".headline", "a", ".author").build());
	SyndFeedInput input = new SyndFeedInput();
	SyndFeed feed = input.build(new XmlReader(new ByteArrayInputStream(createdFeed.getBytes())));
	List<SyndEntry> entries = feed.getEntries();

	assertEquals("headline1", entries.get(0).getTitle());
	assertEquals("http://dummy.url", entries.get(0).getLink());
	assertEquals("Jon Doe", entries.get(0).getAuthor());

	assertEquals("headline2", entries.get(1).getTitle());
	assertEquals("http://dummy.url", entries.get(1).getLink());
	assertEquals("Anna Smith", entries.get(1).getAuthor());
	
	String content1 = entries.get(0).getDescription().getValue();
	assertTrue(content1.contains("headline1"));
	assertTrue(content1.contains("text1"));
	assertTrue(content1.contains("footer1"));
	assertTrue(content1.contains("http://dummy.url"));

	String content2 = entries.get(1).getDescription().getValue();
	assertTrue(content2.contains("headline2"));
	assertTrue(content2.contains("text2"));
	assertTrue(content2.contains("footer2"));
	assertTrue(content2.contains("http://dummy.url"));
}
 
Example 21
Project: concourseconnect-community   File: RssListViewer.java   View source code 5 votes vote down vote up
public String doView(RenderRequest request, RenderResponse response) throws Exception {
  // The JSP to show upon success
  String defaultView = VIEW_PAGE;

  // General display preferences
  request.setAttribute(TITLE, request.getPreferences().getValue(PREF_TITLE, "Review"));
  request.setAttribute(SHOW_DESCRIPTION, request.getPreferences().getValue(PREF_SHOW_DESCRIPTION, "true"));
  request.setAttribute(LIMIT, String.valueOf(Integer.parseInt(request.getPreferences().getValue(PREF_LIMIT, "10"))-1));

  // Determine the feed
  String feedValue = request.getPreferences().getValue(PREF_FEED, null);
  if (feedValue == null) {
    return null;
  }

  // Process the feed
  URL feedUrl = new URL(feedValue);
  SyndFeedInput input = new SyndFeedInput();
  SyndFeed feed = input.build(new XmlReader(feedUrl));
  request.setAttribute(RSS_FEED, feed);

  if (LOG.isDebugEnabled()) {
    LOG.debug(feed);
  }

  // Skip displaying if there are no entries
  if (feed.getEntries().size() == 0) {
    return null;
  }

  // JSP view
  return defaultView;
}
 
Example 22
Project: HackBulgaria-Java   File: NewsAggregator.java   View source code 5 votes vote down vote up
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
    resp.setContentType("text/html");
    resp.setCharacterEncoding("UTF-8");
    PrintWriter out = resp.getWriter();

    URL feedUrl = new URL("http://localhost");
    String category = req.getPathInfo();
    if (category.equals("/Bulgaria")) {
        feedUrl = new URL("http://www.dnevnik.bg/rssc/?rubrid=1657");
        out.printf("<h1>Новини от България</h1><hr>");
    } else if (category.equals("/Technology")) {
        feedUrl = new URL("http://www.dnevnik.bg/rssc/?rubrid=1660");
        out.printf("<h1>Новини от Технологии</h1><hr>");
    } else if (category.equals("/Sports")) {
        feedUrl = new URL("http://www.dnevnik.bg/rssc/?rubrid=2368");
        out.printf("<h1>Новини от Спорт</h1><hr>");
    }

    HttpURLConnection conn = (HttpURLConnection)feedUrl.openConnection();
    try {
        SyndFeed feed = new SyndFeedInput().build(new XmlReader(conn));
        List entries = feed.getEntries();
        Iterator it = entries.iterator();
        while (it.hasNext()) {
            SyndEntry entry = (SyndEntry) it.next();
            out.printf("<a href=\"%s\"><h2>%s</h2></a>", entry.getLink(), entry.getTitle());
            out.printf("<h4>%s пише на %s</h4>", entry.getAuthor(), entry.getPublishedDate());
            out.printf("<article>%s</article>", entry.getDescription().getValue());
            out.printf("<hr/>");
        }
    } catch (FeedException e) {
        e.printStackTrace();
    }
}
 
Example 23
Project: chitanka4kindle   File: OpdsPage.java   View source code 5 votes vote down vote up
private SyndFeed parse(String address) throws IOException {
	try {
		SyndFeedInput input = new SyndFeedInput();
		return input.build(new XmlReader(Network.getInputStream(address)));
	} catch (Exception e) {
		throw new RuntimeException("Failed parsing: " + address, e);
	}
}
 
Example 24
Project: marmotta   File: SleUtility.java   View source code 5 votes vote down vote up
/** This method will take a SyndFeed object with a SimpleListExtension on it and populate the entries
 * with current SleEntry values for sorting and grouping.
 * <b>NB</b>: This basically does this by re-generating the XML for all the entries then re-parsing them
 * into the SLE data structures. It is a very heavy operation and should not be called frequently!
 */
public static void initializeForSorting( SyndFeed feed )throws FeedException {
    List syndEntries = feed.getEntries();

    // TODO: the null parameter below will break delegating parsers and generators
    ModuleGenerators g =
            new ModuleGenerators( feed.getFeedType() + ITEM_MODULE_GENERATORS_POSFIX_KEY,
                    null);
    SyndFeedOutput o = new SyndFeedOutput();
    Document d = o.outputJDom( feed );
    SyndFeed feed2 = new SyndFeedInput().build( d );
    feed.copyFrom( feed2 );
}
 
Example 25
Project: swingx-ws   File: YahooRSS.java   View source code 5 votes vote down vote up
/**
 * Reads a feed and returns the ROME SyndFeed object that was created by
 * reading the feed. Note that this method blocks, and should be called
 * on a background thread.
 *
 * @return the feed
 * @throws RSSException if something goes wrong while accessing the RSS feed
 */
public SyndFeed readFeed() throws RSSException {
    try {
        URL feedUrl = constructUrl();
        SyndFeedInput input = new SyndFeedInput();
        return input.build(new XmlReader(feedUrl));
    } catch (Exception e) {
        throw new RSSException(e);
    }
}
 
Example 26
Project: PrimefacesShowcase   File: YAHOONewsService.java   View source code 5 votes vote down vote up
public List<NewsGroup> fetchNews() {
	List<NewsGroup> news = new ArrayList<NewsGroup>();
	try {
		for(String key : feeds.keySet()) {
			URL feedSource = new URL(feeds.get(key));
			SyndFeedInput input = new SyndFeedInput();
			SyndFeed feed = input.build(new XmlReader(feedSource));
			List<NewsEntry> entries = new ArrayList<NewsEntry>();
			
			int i = 0;
			for(Object f : feed.getEntries()) {
				SyndEntry entry = (SyndEntry) f;
                   String title = entry.getTitle();
                   title = title.length() <= 25 ? title : title.substring(0, 25);
				entries.add(new NewsEntry(i, title + "...", entry.getDescription().getValue()));
				i++;
			}
			
			news.add(new NewsGroup(key, entries)); 
		}
	
	} catch(Exception exception) {
		logger.log(Level.SEVERE, "Error in retrieving news from YAHOO: {0}", exception.getMessage());
	}
	
	return news;
}
 
Example 27
Project: betfair-horses   File: ResultsAPI.java   View source code 5 votes vote down vote up
private static SyndFeed fetchFeed(String url) throws BetfairException {
    SyndFeed feed;
    try {
        URL feedSource = new URL(url);
        SyndFeedInput input = new SyndFeedInput();
        feed = input.build(new XmlReader(feedSource));
    } catch (Exception e) {
        throw new BetfairException("Problem fetching results from URL " +
                url, e);
    }
    return feed;
}
 
Example 28
Project: griffon-swingx-ws-plugin   File: YahooRSS.java   View source code 5 votes vote down vote up
/**
 * Reads a feed and returns the ROME SyndFeed object that was created by
 * reading the feed. Note that this method blocks, and should be called
 * on a background thread.
 *
 * @return the feed
 * @throws RSSException if something goes wrong while accessing the RSS feed
 */
public SyndFeed readFeed() throws RSSException {
    try {
        URL feedUrl = constructUrl();
        SyndFeedInput input = new SyndFeedInput();
        return input.build(new XmlReader(feedUrl));
    } catch (Exception e) {
        throw new RSSException(e);
    }
}
 
Example 29
Project: GridSphere   File: RssServiceImpl.java   View source code 5 votes vote down vote up
public SyndFeed getFeed(String url) throws FeedException {
    SyndFeedInput input = new SyndFeedInput();
    SyndFeed feed = null;
    long diff = CACHE_TIME + 1;
    // check if the url is in the cachedStore
    if (cachedStore.containsKey(url)) {
        Long cachedTime = (Long) this.cachedTime.get(url);
        diff = System.currentTimeMillis() - cachedTime.longValue();
    }


    if (diff > CACHE_TIME) {
        try {
            URL rssUrl = new URL(url);
            URLConnection c = rssUrl.openConnection();
            c.setConnectTimeout(2000);
            c.setReadTimeout(2000);
            c.connect();
            InputSource src = new InputSource(c.getInputStream());
            feed = input.build(src);
        } catch (IOException e) {
            throw new FeedException("Invalid URL.");
        }
        cachedStore.put(url, feed);
        cachedTime.put(url, new Long(System.currentTimeMillis()));
    }
    return (SyndFeed) cachedStore.get(url);
}
 
Example 30
Project: Babler   File: RSSScraper.java   View source code 4 votes vote down vote up
public AbstractMap.SimpleEntry<Integer, Integer> fetchAndSave() throws Exception {

        URL url = new URL(this.url);

        SyndFeedInput input = new SyndFeedInput();
        SyndFeed feed = input.build(new XmlReader(url));


        int items = feed.getEntries().size();

        if(items > 0){
            log.info("Attempting to parse rss feed: "+ this.url );
            log.info("This Feed has "+items +" items");
        }

        List <SyndEntry> entries = feed.getEntries();

        for (SyndEntry item : entries){
            log.info("Title: " + item.getTitle());
            log.info("Link: " + item.getLink());
            SyndContentImpl contentHolder = (SyndContentImpl) item.getContents().get(0);
            String content = contentHolder.getValue();

            //content might contain html data, let's clean it up
            Document doc = Jsoup.parse(content);
            content = doc.text();
            try {
                    Result result = ld.detectLanguage(content, language);
                    if (result.languageCode.equals(language) && result.isReliable) {

                        FileSaver file = new FileSaver(content, this.language, "bs", item.getLink(), item.getUri(), String.valueOf(content.hashCode()));
                        String fileName = file.getFileName();
                        BlogPost post = new BlogPost(content,this.language,null,"bs",item.getLink(),item.getUri(),fileName);
                        if(DAO.saveEntry(post)) {
                            file.save(this.logDb);
                            numOfFiles++;
                            wrongCount = 0;
                        }

                    }

                    else{
                        log.info("Item " + item.getTitle() + "is in a diff languageCode, skipping this post  "+ result.languageCode);
                        wrongCount ++;
                        if(wrongCount > 3){
                            log.info("Already found 3 posts in the wrong languageCode, skipping this blog");
                        }
                        break;
                    }

            }
            catch(Exception e){
                log.error(e);
                break;
            }


        }
        return new AbstractMap.SimpleEntry<>(numOfFiles,wrongCount);
    }
 
Example 31
Project: AtomMatrikel   File: Downloader.java   View source code 4 votes vote down vote up
public void read() throws Exception {
    FTPClient ftpClient = new FTPClient();
    ftpClient.setBufferSize(1024 * 1024);
    ftpClient.setDefaultTimeout(1000);
    ftpClient.setControlEncoding("UTF-8");
    ftpClient.setDataTimeout(10000);
    ZipFile zipFile = new ZipFile();
    Parser parser = new Parser();
    URL feedUrl = new URL("https://download.kortforsyningen.dk/sites/default/files/feeds/MATRIKELKORT_GML.xml");
    SyndFeedInput input = new SyndFeedInput();
    System.out.println("Henter Atom feed....\n");

    SyndFeed feed = input.build(new XmlReader(feedUrl));
    System.out.println("Feed Titel: " + feed.getTitle());
    Date lastPublishedDate;
    Date currentDate;
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    String fileName;
    Integer elavsKode;

    Track track = new Track();
    String lastStr = track.getLastFromDb();
    lastPublishedDate = sdf.parse(lastStr);

    System.out.println("\nSidste entry: " + lastStr + "\n");

    this.login(ftpClient);

    for (SyndEntry entry : (List<SyndEntry>) feed.getEntries()) {

        System.out.println("\n--------------\n");

        currentDate = entry.getPublishedDate();

        if (currentDate.before(lastPublishedDate) || currentDate.equals(lastPublishedDate)) {
            break;
        }

        System.out.println("Published Date: " + entry.getPublishedDate() + "\n");

        // Get the Links
        for (SyndLinkImpl link : (List<SyndLinkImpl>) entry.getLinks()) {
            System.out.println("Link: " + link.getHref());
            fileName = link.getHref().split("/")[7];
            elavsKode = Integer.valueOf(fileName.split("_")[0]);
            track.storeInDb(currentDate.toString(), elavsKode);

            // Get the file
            this.c(ftpClient, fileName, zipFile);

            // Get xml stream from zip-file
            ZipInputStream zin = zipFile.getGml(fileName);

            // Parse the xml and insert in database
            parser.build(zin, elavsKode);

            // Delete the file
            File file = new File(fileName);
            file.delete();
        }


    }
    // logout the user, returned true if logout successfully
    ftpClient.disconnect();
    System.out.println("Alt hentet...");
    System.out.println("FTP forbindelse afbrydes.");


}
 
Example 32
Project: parabuild-ci   File: FeedTest.java   View source code 4 votes vote down vote up
protected SyndFeed getSyndFeed() throws Exception {
    SyndFeedInput in = new SyndFeedInput();
    return in.build(getFeedReader());
}
 
Example 33
Project: parabuild-ci   File: FeedAggregator.java   View source code 4 votes vote down vote up
public static void main(String[] args) {
    boolean ok = false;
    if (args.length>=2) {
        try {
            String outputType = args[0];

            SyndFeed aggrFeed = new SyndFeedImpl();
            aggrFeed.setFeedType(outputType);

            aggrFeed.setTitle("Aggregated Feed");
            aggrFeed.setDescription("Anonymous Aggregated Feed");
            aggrFeed.setAuthor("anonymous");
            aggrFeed.setLink("http://www.anonymous.com");

            List entries = new ArrayList();
            aggrFeed.setEntries(entries);

            for (int i=1;i<args.length;i++) {
                URL feedUrl = new URL(args[i]);
                SyndFeedInput input = new SyndFeedInput();

                SyndFeed feed = input.build(new XmlReader(feedUrl));

                entries.addAll(feed.getEntries());
            }

            SyndFeedOutput output = new SyndFeedOutput();
            output.output(aggrFeed,new PrintWriter(System.out));

            ok = true;
        }
        catch (Exception ex) {
            System.out.println("ERROR: "+ex.getMessage());
        }
    }

    if (!ok) {
        System.out.println();
        System.out.println("FeedAggregator aggregates different feeds into a single one.");
        System.out.println("The first parameter must be the feed type for the aggregated feed.");
        System.out.println(" [valid values are: rss_0.9, rss_0.91, rss_0.92, rss_0.93, ]");
        System.out.println(" [                  rss_0.94, rss_1.0, rss_2.0 & atom_0.3  ]");
        System.out.println("The second to last parameters are the URLs of feeds to aggregate.");
        System.out.println();
    }
}
 
Example 34
Project: parabuild-ci   File: RomeApplet.java   View source code 4 votes vote down vote up
private SyndFeed getRSSFeed(URL url) throws Exception {
	SyndFeed feed;
	SyndFeedInput input = new SyndFeedInput();
	feed = input.build(new InputStreamReader(url.openStream()));
	return feed;
}
 
Example 35
Project: feedsucker   File: BasicTests.java   View source code 4 votes vote down vote up
public void readAndPrintFeed(String feedURL) throws Exception {
        SyndFeedInput input = new SyndFeedInput();                
        SyndFeed feed = input.build(new XmlReader(new URL(feedURL)));
        System.out.println("FEED: " + feed.getDescription());
        System.out.println("type: "+feed.getFeedType()+"\n"+
                "copyright: "+feed.getCopyright()+"\n"+
                "author: "+feed.getAuthor()+"\n"+
                "uri: "+feed.getUri()+"\n"+
                "language: "+feed.getLanguage()+"\n"+
                "link: "+feed.getLink()+"\n");
        //System.out.println(feed);
        List entries = feed.getEntries();
        System.out.println("ENTRIES: ");
        for (Object o : entries) {
            SyndEntry e = (SyndEntry)o;
            System.out.println("FEED ENTRY:");
            System.out.println("title: "+e.getTitle()+"\n"+
                    "date published: "+e.getPublishedDate()+"\n"+
                    "author: "+e.getAuthor()+"\n"+                    
                    "link: "+e.getLink()+"\n"+
                    "uri: "+e.getUri());      
            System.out.println("- categories: ");
            List cat = e.getCategories();
            for (Object oc : cat) {
                SyndCategory c = (SyndCategory)oc;
                System.out.println("name: "+c.getName()+"\n"+
                        "taxonomy uri: "+c.getTaxonomyUri());
            }            
            
            SyndContent content = e.getDescription();
            System.out.println("content: "+content.getValue());       
            System.out.println("--- entry end ---");
//          List contents = e.getContents();            
//            System.out.println("- contents: ");
//            for (Object oc : contents) {
//                SyndContent c = (SyndContent)oc;
//                System.out.println("type: "+c.getType());
//                System.out.println("value:");
//                System.out.println(c.getValue());
//            }
        }
    }
 
Example 36
Project: DigitalMediaServer   File: Feed.java   View source code 4 votes vote down vote up
@SuppressWarnings("unchecked")
public void parse() throws Exception {
	SyndFeedInput input = new SyndFeedInput();
	byte b[] = downloadAndSendBinary(url);
	if (b != null) {
		SyndFeed feed = input.build(new XmlReader(new ByteArrayInputStream(b)));
		name = feed.getTitle();
		if (feed.getCategories() != null && feed.getCategories().size() > 0) {
			SyndCategory category = (SyndCategory) feed.getCategories().get(0);
			tempCategory = category.getName();
		}
		List<SyndEntry> entries = feed.getEntries();
		for (SyndEntry entry : entries) {
			tempItemTitle = entry.getTitle();
			tempItemLink = entry.getLink();
			tempFeedLink = entry.getUri();
			tempItemThumbURL = null;

			ArrayList<Element> elements = (ArrayList<Element>) entry.getForeignMarkup();
			for (Element elt : elements) {
				if ("group".equals(elt.getName()) && "media".equals(elt.getNamespacePrefix())) {
					List<Content> subElts = elt.getContent();
					for (Content subelt : subElts) {
						if (subelt instanceof Element) {
							parseElement((Element) subelt, false);
						}
					}
				}
				parseElement(elt, true);
			}
			List<SyndEnclosure> enclosures = entry.getEnclosures();
			for (SyndEnclosure enc : enclosures) {
				if (StringUtils.isNotBlank(enc.getUrl())) {
					tempItemLink = enc.getUrl();
				}
			}
			manageItem();
		}
	}
	setLastModified(System.currentTimeMillis());
}
 
Example 37
Project: Camel   File: RssConverter.java   View source code 4 votes vote down vote up
@Converter
public static SyndFeed xmlToFeed(String xml) throws FeedException {
    SyndFeedInput input = new SyndFeedInput();
    return input.build(new StringReader(xml));
}
 
Example 38
Project: apex-malhar   File: RomeSyndicationOperator.java   View source code 4 votes vote down vote up
/**
 * Thread processing of the syndication feeds
 */
@Override
public void run()
{
  try {
    while (true) {
      InputStreamReader isr = null;
      try {
        isr = new InputStreamReader(getFeedInputStream());
        SyndFeedInput feedInput = new SyndFeedInput();
        SyndFeed feed = feedInput.build(isr);
        List entries = feed.getEntries();
        List<RomeFeedEntry> nfeedItems = new ArrayList<RomeFeedEntry>();
        boolean oldEntries = false;
        for (int i = 0; i < entries.size(); ++i) {
          SyndEntry syndEntry = (SyndEntry)entries.get(i);
          RomeFeedEntry romeFeedEntry = getSerializableEntry(syndEntry);
          if (!oldEntries) {
            if (!feedItems.contains(romeFeedEntry)) {
              outputPort.emit(romeFeedEntry);
            } else if (orderedUpdate) {
              oldEntries = true;
            }
          }
          nfeedItems.add(romeFeedEntry);
        }
        feedItems = nfeedItems;
      } catch (Exception e) {
        logger.error(e.getMessage());
      } finally {
        if (isr != null) {
          try {
            isr.close();
          } catch (Exception ce) {
            logger.error(ce.getMessage());
          }
        }
      }
      Thread.sleep(interval);
    }
  } catch (InterruptedException ie) {
    logger.error("Interrupted: " + ie.getMessage());
  }
}
 
Example 39
Project: FeedExpander   File: FeedBuilderImpl.java   View source code 4 votes vote down vote up
@Override
public @Nonnull FeedBuilderImpl loadFeed() throws MalformedURLException, FeedException, IOException {
	InputStream feedContentStream = urlLoaderFactory.getUrlLoader(feedUrl).getContentAsStream(StandardCharsets.UTF_8);
	loadedFeed = new SyndFeedInput().build(new XmlReader(feedContentStream));
	return this;
}
 
Example 40
Project: FeedExpander   File: FeedBuilderTest.java   View source code 4 votes vote down vote up
private SyndFeed createSyndFeed(String expandedFeed) throws FeedException, IOException {
	return new SyndFeedInput().build(new XmlReader(new ByteArrayInputStream(expandedFeed.getBytes())));
}