com.twitter.hbc.core.endpoint.StatusesFilterEndpoint Java Examples

The following examples show how to use com.twitter.hbc.core.endpoint.StatusesFilterEndpoint. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TwitterSourceTest.java    From hazelcast-jet-contrib with Apache License 2.0 6 votes vote down vote up
@Test
public void testStream_withTermFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("BTC", "ETH"));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));

    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.", list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 
Example #2
Source File: TwitterDataSource.java    From kafka-streams with Apache License 2.0 6 votes vote down vote up
private  Client getTwitterClient(Properties props, BlockingQueue<String> messageQueue) {

        String clientName = props.getProperty("clientName");
        String consumerKey = props.getProperty("consumerKey");
        String consumerSecret = props.getProperty("consumerSecret");
        String token = props.getProperty("token");
        String tokenSecret = props.getProperty("tokenSecret");
        List<String> searchTerms = Arrays.asList(props.getProperty("searchTerms").split(","));

        Authentication authentication = new OAuth1(consumerKey,consumerSecret,token,tokenSecret);
        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        hosebirdEndpoint.trackTerms(searchTerms);

        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name(clientName)
                .hosts(hosebirdHosts)
                .authentication(authentication)
                .endpoint(hosebirdEndpoint)
                .processor(new StringDelimitedProcessor(messageQueue));

          return clientBuilder.build();

    }
 
Example #3
Source File: TwitterFireHose.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private StreamingEndpoint getEndpoint(TwitterFireHoseConfig config) {
    List<Long> followings = config.getFollowings();
    List<String> terms = config.getTrackTerms();

    if (CollectionUtils.isEmpty(followings) && CollectionUtils.isEmpty(terms)) {
        return new SampleStatusesEndpoint().createEndpoint();
    } else {
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        if (CollectionUtils.isNotEmpty(followings)) {
           hosebirdEndpoint.followings(followings);
        }

        if (CollectionUtils.isNotEmpty(terms)) {
           hosebirdEndpoint.trackTerms(terms);
        }

        return hosebirdEndpoint;
    }
}
 
Example #4
Source File: TwitterSourceMockTest.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
@Test
public void streamApiMockTest() {

    List<String> terms = new ArrayList<>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(getCredentials(),
            "http://" + server.getHostName() + ":" + server.getPort(),
            () -> new StatusesFilterEndpoint().trackTerms(terms));

    Pipeline pipeline = Pipeline.create();
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(10,
            list -> assertGreaterOrEquals("Emits at least 100 tweets in 1 min.",
                    list.size(), 100)));
    Job job = createJetMember().newJob(pipeline);

    sleepAtLeastSeconds(5);

    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 
Example #5
Source File: TwitterSourceTest.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
@Test
public void testStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.stream(credentials,
            () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withoutTimestamps()
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 
Example #6
Source File: TwitterSourceTest.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestampedStream_termFilter() {
    Pipeline pipeline = Pipeline.create();
    List<String> terms = new ArrayList<String>(Arrays.asList("San Mateo", "Brno", "London", "Istanbul"));

    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().trackTerms(terms));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(0)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 20 tweets in 1 min.",
                    list.size(), 20)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 
Example #7
Source File: TwitterSourceTest.java    From hazelcast-jet-contrib with Apache License 2.0 5 votes vote down vote up
@Test
public void testTimestampedStream_userFilter() {
    Pipeline pipeline = Pipeline.create();
    List<Long> userIds = new ArrayList<Long>(
            Arrays.asList(612473L, 759251L, 1367531L, 34713362L, 51241574L, 87818409L));
    final StreamSource<String> twitterTestStream = TwitterSources.timestampedStream(
            credentials, () -> new StatusesFilterEndpoint().followings(userIds));
    StreamStage<String> tweets = pipeline
            .readFrom(twitterTestStream)
            .withNativeTimestamps(1)
            .map(rawJson -> Json.parse(rawJson)
                                .asObject()
                                .getString("text", null));
    tweets.writeTo(AssertionSinks.assertCollectedEventually(60,
            list -> assertGreaterOrEquals("Emits at least 15 tweets in 1 min.",
                    list.size(), 15)));
    Job job = jet.newJob(pipeline);
    sleepAtLeastSeconds(5);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: "
                + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}
 
Example #8
Source File: TwitterStream.java    From AIBlueprints with MIT License 5 votes vote down vote up
public TwitterStream(SentimentDetector sentimentDetector,
                     Gson gson,
                     Properties props) {
    this.sentimentDetector = sentimentDetector;
    this.gson = gson;

    msgQueue = new LinkedBlockingQueue<String>(100000);

    Hosts hosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

    List<String> terms = Lists.newArrayList(
            props.getProperty("twitter_terms")
                    .split("\\s*,\\s*"));
    endpoint.trackTerms(terms);

    Authentication auth = new OAuth1(
            props.getProperty("twitter_consumer_key"),
            props.getProperty("twitter_consumer_secret"),
            props.getProperty("twitter_token"),
            props.getProperty("twitter_token_secret"));
    ClientBuilder builder = new ClientBuilder()
            .name("SmartCode-Client-01")
            .hosts(hosts)
            .authentication(auth)
            .endpoint(endpoint)
            .processor(new StringDelimitedProcessor(msgQueue));
    client = builder.build();
    client.connect();
}
 
Example #9
Source File: TwitterStream.java    From AIBlueprints with MIT License 5 votes vote down vote up
public TwitterStream(Gson gson, Properties props, BlockingQueue<Map<String,Object>> imageQueue) {
    this.gson = gson;
    this.imageQueue = imageQueue;

    msgQueue = new LinkedBlockingQueue<String>(100000);

    Hosts hosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

    List<String> terms = Lists.newArrayList(
            props.getProperty("twitter_terms")
                    .split("\\s*,\\s*"));
    endpoint.trackTerms(terms);

    Authentication auth = new OAuth1(
            props.getProperty("twitter_consumer_key"),
            props.getProperty("twitter_consumer_secret"),
            props.getProperty("twitter_token"),
            props.getProperty("twitter_token_secret"));
    ClientBuilder builder = new ClientBuilder()
            .name("SmartCode-Client-01")
            .hosts(hosts)
            .authentication(auth)
            .endpoint(endpoint)
            .processor(new StringDelimitedProcessor(msgQueue));
    client = builder.build();
    client.connect();
}
 
Example #10
Source File: CryptocurrencySentimentAnalysis.java    From hazelcast-jet-demos with Apache License 2.0 5 votes vote down vote up
/**
 * Builds and returns the Pipeline which represents the actual computation.
 */
private static Pipeline buildPipeline() {
    Pipeline pipeline = Pipeline.create();

    List<String> allCoinMarkers = Stream.of(CoinType.values())
                                        .flatMap(ct -> ct.markers().stream())
                                        .collect(toList());
    StreamStage<String> tweets = pipeline
            .readFrom(TwitterSources.timestampedStream(
                    loadCredentials(),
                    () -> new StatusesFilterEndpoint()
                            .trackTerms(allCoinMarkers)))
            .withNativeTimestamps(SECONDS.toMillis(1));
    StreamStageWithKey<Entry<CoinType, Double>, CoinType> tweetsWithSentiment = tweets
            .map(rawTweet -> new JSONObject(rawTweet).getString("text"))
            .flatMap(CryptocurrencySentimentAnalysis::flatMapToRelevant)
            .mapUsingService(sentimentAnalyzerContext(), (analyzer, e1) ->
                    entry(e1.getKey(), analyzer.getSentimentScore(e1.getValue())))
            .filter(e -> !e.getValue().isInfinite() && !e.getValue().isNaN())
            .groupingKey(entryKey());

    AggregateOperation1<Entry<CoinType, Double>, ?, Tuple2<Double, Long>> avgAndCount =
            allOf(averagingDouble(Entry::getValue), counting());

    tweetsWithSentiment
            .window(sliding(HALF_MINUTE.durationMillis(), 200))
            .aggregate(avgAndCount)
            .map(windowResult -> entry(tuple2(windowResult.getKey(), HALF_MINUTE), windowResult.getValue()))
            .writeTo(map(MAP_NAME_JET_RESULTS));

    tweetsWithSentiment
            .window(sliding(FIVE_MINUTES.durationMillis(), 200))
            .aggregate(avgAndCount)
            .map(windowResult -> entry(tuple2(windowResult.getKey(), FIVE_MINUTES), windowResult.getValue()))
            .writeTo(map(MAP_NAME_JET_RESULTS));

    return pipeline;
}
 
Example #11
Source File: GeoTwitterSource.java    From OSTMap with Apache License 2.0 5 votes vote down vote up
@Override
protected void initializeConnection() {
    if (LOG.isInfoEnabled()) {
        LOG.info("Initializing Twitter Streaming API connection");
    }

    this.queue = new LinkedBlockingQueue(this.queueSize);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint(false)
            .locations(Arrays.asList(
                            new Location(
                                    // europa: -32.0 34.0 40.0 75.0
                                    new Location.Coordinate(-32.0, 34.0), // south west
                                    new Location.Coordinate(40.0, 75.0))
                     //       new Location(
                     //               // north america: -168.48633, 13.23995 -50.36133, 72.76406
                     //               new Location.Coordinate(-168.48633, 13.23995), // south west
                     //               new Location.Coordinate(-50.36133, 72.76406))
                    )
            );
    endpoint.stallWarnings(false);
    OAuth1 auth = this.authenticate();

    this.initializeClient(endpoint, auth);
    if (LOG.isInfoEnabled()) {
        LOG.info("Twitter Streaming API connection established successfully");
    }

}
 
Example #12
Source File: FilterStreamExample.java    From hbc with Apache License 2.0 5 votes vote down vote up
public static void run(String consumerKey, String consumerSecret, String token, String secret) throws InterruptedException {
  BlockingQueue<String> queue = new LinkedBlockingQueue<String>(10000);
  StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
  // add some track terms
  endpoint.trackTerms(Lists.newArrayList("twitterapi", "#yolo"));

  Authentication auth = new OAuth1(consumerKey, consumerSecret, token, secret);
  // Authentication auth = new BasicAuth(username, password);

  // Create a new BasicClient. By default gzip is enabled.
  Client client = new ClientBuilder()
          .hosts(Constants.STREAM_HOST)
          .endpoint(endpoint)
          .authentication(auth)
          .processor(new StringDelimitedProcessor(queue))
          .build();

  // Establish a connection
  client.connect();

  // Do whatever needs to be done with messages
  for (int msgRead = 0; msgRead < 1000; msgRead++) {
    String msg = queue.take();
    System.out.println(msg);
  }

  client.stop();

}
 
Example #13
Source File: TwitterHandler.java    From spring4ws-demos with Apache License 2.0 5 votes vote down vote up
@PostConstruct
public void init() {
	BlockingQueue<String> queue = new LinkedBlockingQueue<>(100);
	StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

	endpoint.trackTerms(ImmutableList.of("ExtJS", "#extjs", "Sencha", "#java",
			"java8", "java9", "#websocket", "#SpringFramework", "html5", "javascript",
			"#kotlin", "kotlin"));
	endpoint.languages(ImmutableList.of("en", "de"));

	String consumerKey = this.environment.getProperty("twitter4j.oauth.consumerKey");
	String consumerSecret = this.environment
			.getProperty("twitter4j.oauth.consumerSecret");
	String accessToken = this.environment.getProperty("twitter4j.oauth.accessToken");
	String accessTokenSecret = this.environment
			.getProperty("twitter4j.oauth.accessTokenSecret");

	Authentication auth = new OAuth1(consumerKey, consumerSecret, accessToken,
			accessTokenSecret);

	this.client = new ClientBuilder().hosts(Constants.STREAM_HOST).endpoint(endpoint)
			.authentication(auth).processor(new StringDelimitedProcessor(queue))
			.build();

	this.executorService = Executors.newSingleThreadExecutor();

	TwitterStatusListener statusListener = new TwitterStatusListener(
			this.messagingTemplate, this.lastTweets);
	this.t4jClient = new Twitter4jStatusClient(this.client, queue,
			ImmutableList.of(statusListener), this.executorService);

	this.t4jClient.connect();
	this.t4jClient.process();
}
 
Example #14
Source File: TwitterStream.java    From Java-for-Data-Science with MIT License 4 votes vote down vote up
public Stream<TweetHandler> stream() {
        String myKey = "sl2WbCf4UnIr08xvHVitHJ99r";
        String mySecret = "PE6yauvXjKLuvoQNXZAJo5C8N5U5piSFb3udwkoI76paK6KyqI";
        String myToken = "1098376471-p6iWfxCLtyMvMutTb010w1D1xZ3UyJhcC2kkBjN";
        String myAccess = "2o1uGcp4b2bFynOfu2cA1uz63n5aruV0RwNsUjRpjDBZS";

        out.println("Creating Twitter Stream");
        BlockingQueue<String> statusQueue = new LinkedBlockingQueue<>(1000);
        StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
        endpoint.trackTerms(Lists.newArrayList("twitterapi", this.topic));
        endpoint.stallWarnings(false);
        Authentication twitterAuth = new OAuth1(myKey, mySecret, myToken, myAccess);

        BasicClient twitterClient = new ClientBuilder()
                .name("Twitter client")
                .hosts(Constants.STREAM_HOST)
                .endpoint(endpoint)
                .authentication(twitterAuth)
                .processor(new StringDelimitedProcessor(statusQueue))
                .build();

        twitterClient.connect();

        List<TweetHandler> list = new ArrayList();
        List<String> twitterList = new ArrayList();

        statusQueue.drainTo(twitterList);
        for(int i=0; i<numberOfTweets; i++) {
            String message;
            try {
                message = statusQueue.take();
                list.add(new TweetHandler(message));
            } catch (InterruptedException ex) {
                ex.printStackTrace();
            }
        }

//        for (int msgRead = 0; msgRead < this.numberOfTweets; msgRead++) {
//            try {
//                if (twitterClient.isDone()) {
//                  //  out.println(twitterClient.getExitEvent().getMessage());
//                    break;
//                }
//
//                String msg = statusQueue.poll(10, TimeUnit.SECONDS);
//                if (msg == null) {
//                    out.println("Waited 10 seconds - no message received");
//                } else {
//                    list.add(new TweetHandler(msg));
//                    out.println("Added message: " + msg.length());
//                }
//            } catch (InterruptedException ex) {
//                ex.printStackTrace();
//            }
//        }
        twitterClient.stop();
        out.printf("%d messages processed!\n", twitterClient.getStatsTracker().getNumMessages());

        return list.stream();
    }
 
Example #15
Source File: TwitterProducer.java    From aws-big-data-blog with Apache License 2.0 4 votes vote down vote up
public void run(String propFilePath) {
	loadFileProperties(propFilePath, DEFAULT_PROP_FILE_NAME);

	String consumerKey = System.getProperty(TWIT_CONSUMER_KEY);
	String consumerSecret = System.getProperty(TWIT_CONSUMER_SECRET);
	String token = System.getProperty(TWIT_TOKEN);
	String secret = System.getProperty(TWIT_SECRET);
	String streamName = System.getProperty(STREAM_NAME);
	String regionName = System.getProperty(REGION_NAME);
	
	while (true) {
		/**
		 * Set up your blocking queues: Be sure to size these properly based
		 * on expected TPS of your stream
		 */
		BlockingQueue<String> msgQueue = new LinkedBlockingQueue<String>(
				10000);

		/**
		 * Declare the host you want to connect to, the endpoint, and
		 * authentication (basic auth or oauth)
		 */
		StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();
		
		// Track  anything that is geo-tagged
		endpoint.addQueryParameter("locations", "-180,-90,180,90");

		// These secrets should be read from a config file
		Authentication hosebirdAuth = new OAuth1(consumerKey,
				consumerSecret, token, secret);

		// create a new basic client - by default gzip is enabled
		Client client = new ClientBuilder().hosts(Constants.STREAM_HOST)
				.endpoint(endpoint).authentication(hosebirdAuth)
				.processor(new StringDelimitedProcessor(msgQueue)).build();

		client.connect();
		

		LOG.info("Got connection to Twitter");
		
		// create producer
		ProducerClient producer = new ProducerBuilder()
				.withName("Twitter")
				.withStreamName(streamName)
				.withRegion(regionName)
				.withThreads(10)
				.build();
		
		producer.connect();
		
		LOG.info("Got connection to Kinesis");

		try {
			if (process(msgQueue, producer)) {
				break;
			}

		} catch (Exception e) {
			// if we get here, our client has broken, throw away and
			// recreate
			e.printStackTrace();
		}
		// also, if we make it here, we have had a problem, so start again
		client.stop();
	}
}
 
Example #16
Source File: HoseBirdTester.java    From kafka-streams with Apache License 2.0 3 votes vote down vote up
public static void main(String[] args)  throws InterruptedException {

        BlockingQueue<String> msgQueue = new LinkedBlockingDeque<>();



        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();
        List<String> terms = Lists.newArrayList("superman vs batman","#supermanvsbatman");
        hosebirdEndpoint.trackTerms(terms);




        Authentication hosebirdAuth  = new OAuth1("18qydWMuiUohwCtQpp1MOFCFr",
                                                  "YrYhYd09LKZLbhsKT1o4XcEPl6HiAoNykiOxYBq0dAB8t0vRCo",
                                                  "16972669-KSvyDEMc7dussPfW6a9Ru65L4eWGj637ciHLHZLyn",
                                                  "ky53NE6cbBvtNLopto7o9gVyHDejSB2kPsRhHGKEd1MrS");


        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name("bbejeck-hosebird")
                     .hosts(hosebirdHosts)
                     .authentication(hosebirdAuth)
                     .endpoint(hosebirdEndpoint)
                     .processor(new StringDelimitedProcessor(msgQueue));

        Client hosebirdClient = clientBuilder.build();
        hosebirdClient.connect();

        for (int msgRead = 0; msgRead < 100; msgRead++) {
            String msg = msgQueue.take();
            System.out.println(msg);
        }

        hosebirdClient.stop();

    }