Java Code Examples for com.twitter.hbc.core.endpoint.StreamingEndpoint

private TwitterStreamSourceContext(
        @Nonnull ILogger logger,
        @Nonnull Properties credentials,
        @Nonnull String host,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    this.logger = logger;
    checkTwitterCredentials(credentials);
    String consumerKey = credentials.getProperty("consumerKey");
    String consumerSecret = credentials.getProperty("consumerSecret");
    String token = credentials.getProperty("token");
    String tokenSecret = credentials.getProperty("tokenSecret");

    Authentication auth = new OAuth1(consumerKey, consumerSecret, token, tokenSecret);
    client = new ClientBuilder()
            .hosts(host)
            .endpoint(endpointSupplier.get())
            .authentication(auth)
            .processor(new StringDelimitedProcessor(queue))
            .build();
    client.connect();
}
 

private StreamingEndpoint getEndpoint(TwitterFireHoseConfig config) {
    List<Long> followings = config.getFollowings();
    List<String> terms = config.getTrackTerms();

    if (CollectionUtils.isEmpty(followings) && CollectionUtils.isEmpty(terms)) {
        return new SampleStatusesEndpoint().createEndpoint();
    } else {
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        if (CollectionUtils.isNotEmpty(followings)) {
           hosebirdEndpoint.followings(followings);
        }

        if (CollectionUtils.isNotEmpty(terms)) {
           hosebirdEndpoint.trackTerms(terms);
        }

        return hosebirdEndpoint;
    }
}
 

public BasicClient(String name, Hosts hosts, StreamingEndpoint endpoint, Authentication auth, boolean enableGZip, HosebirdMessageProcessor processor,
                   ReconnectionManager reconnectionManager, RateTracker rateTracker, ExecutorService executorService,
                   @Nullable BlockingQueue<Event> eventsQueue, HttpParams params, SchemeRegistry schemeRegistry) {
  Preconditions.checkNotNull(auth);
  HttpClient client;
  if (enableGZip) {
    client = new RestartableHttpClient(auth, enableGZip, params, schemeRegistry);
  } else {
    DefaultHttpClient defaultClient = new DefaultHttpClient(new PoolingClientConnectionManager(schemeRegistry), params);

    /** Set auth **/
    auth.setupConnection(defaultClient);
    client = defaultClient;
  }

  this.canRun = new AtomicBoolean(true);
  this.executorService = executorService;
  this.clientBase = new ClientBase(name, client, hosts, endpoint, auth, processor, reconnectionManager, rateTracker, eventsQueue);
}
 

ClientBase(String name, HttpClient client, Hosts hosts, StreamingEndpoint endpoint, Authentication auth,
           HosebirdMessageProcessor processor, ReconnectionManager manager, RateTracker rateTracker,
           @Nullable BlockingQueue<Event> eventsQueue) {
  this.client = Preconditions.checkNotNull(client);
  this.name = Preconditions.checkNotNull(name);

  this.endpoint = Preconditions.checkNotNull(endpoint);
  this.hosts = Preconditions.checkNotNull(hosts);
  this.auth = Preconditions.checkNotNull(auth);

  this.processor = Preconditions.checkNotNull(processor);
  this.reconnectionManager = Preconditions.checkNotNull(manager);
  this.rateTracker = Preconditions.checkNotNull(rateTracker);

  this.eventsQueue = eventsQueue;

  this.exitEvent = new AtomicReference<Event>();

  this.isRunning = new CountDownLatch(1);
  this.statsReporter = new StatsReporter();

  this.connectionEstablished = new AtomicBoolean(false);
  this.reconnect = new AtomicBoolean(false);
}
 

@Override
public StreamingEndpoint createEndpoint() {
	// this default endpoint initializer returns the sample endpoint: Returning a sample from the firehose (all tweets)
	StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
	endpoint.stallWarnings(false);
	endpoint.delimited(false);
	return endpoint;
}
 

/**
 * Equivalent of {@link #stream(Properties, SupplierEx)} with the
 * additional option to specify the Twitter {@code host}.
 *
 * @param host a Twitter host URL to connect to. These hosts are defined in
 *          {@link com.twitter.hbc.core.Constants}.
 */
@Nonnull
public static StreamSource<String> stream(
        @Nonnull Properties credentials,
        @Nonnull String host,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return SourceBuilder.stream("twitter-stream-source",
            ctx -> new TwitterStreamSourceContext(ctx.logger(), credentials, host, endpointSupplier))
                        .fillBufferFn(TwitterStreamSourceContext::fillBuffer)
                        .destroyFn(TwitterStreamSourceContext::close)
                        .build();
}
 

/**
 * Equivalent for {@link #timestampedStream(Properties, SupplierEx)} with
 * the additional option to specify the Twitter {@code host}.
 *
 * @param host a Twitter host URL to connect to. These hosts are defined in
 *          {@link com.twitter.hbc.core.Constants}.
 */
@Nonnull
public static StreamSource<String> timestampedStream(
        @Nonnull Properties credentials,
        @Nonnull String host,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return SourceBuilder.timestampedStream("twitter-timestamped-stream-source",
            ctx -> new TwitterStreamSourceContext(ctx.logger(), credentials, host, endpointSupplier))
                        .fillBufferFn(TwitterStreamSourceContext::fillTimestampedBuffer)
                        .destroyFn(TwitterStreamSourceContext::close)
                        .build();
}
 

@Override
public StreamingEndpoint createEndpoint() {
	// this default endpoint initializer returns the sample endpoint: Returning a sample from the firehose (all tweets)
	StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
	endpoint.stallWarnings(false);
	endpoint.delimited(false);
	return endpoint;
}
 

public StreamingEndpoint createEndpoint() {
    // Returns the sample endpoint: Returning a sample from the firehose (all tweets)
    StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
    endpoint.stallWarnings(false);
    endpoint.delimited(false);
    return endpoint;
}
 

@Override
public StreamingEndpoint createEndpoint() {
	// this default endpoint initializer returns the sample endpoint: Returning a sample from the firehose (all tweets)
	StatusesSampleEndpoint endpoint = new StatusesSampleEndpoint();
	endpoint.stallWarnings(false);
	endpoint.delimited(false);
	return endpoint;
}
 

@Override
public StreamingEndpoint getEndpoint() {
  return clientBase.getEndpoint();
}
 

ClientBase(String name, HttpClient client, Hosts hosts, StreamingEndpoint endpoint, Authentication auth,
           HosebirdMessageProcessor processor, ReconnectionManager manager, RateTracker rateTracker) {
  this(name, client, hosts, endpoint, auth, processor, manager, rateTracker, null);
}
 

public StreamingEndpoint getEndpoint() {
  return endpoint;
}
 

@Override
public StreamingEndpoint getEndpoint() {
  return client.getEndpoint();
}
 

/**
 * Creates a {@link StreamSource} which reads tweets using Twitter's
 * Streaming API for data ingestion to Jet pipelines. This method uses the
 * {@link com.twitter.hbc.core.Constants#STREAM_HOST} as a Twitter
 * Streaming API host.
 * <p>
 * Example usage:
 * <pre>{@code
 * Properties credentials = loadTwitterCredentials();
 * List<String> terms = new ArrayList<>(Arrays.asList("BTC", "ETH"));
 * StreamSource<String> streamSource =
 *              TwitterSources.stream(
 *                      credentials,
 *                      () -> new StatusesFilterEndpoint().trackTerms(terms)
 *              );
 * Pipeline p = Pipeline.create();
 * StreamSourceStage<String> srcStage = p.readFrom(streamSource);
 * }</pre>
 *
 * @param credentials      a Twitter OAuth1 credentials that consist of
 *                         "consumerKey", "consumerSecret", "token",
 *                         "tokenSecret" keys.
 * @param endpointSupplier a supplier function that supplies a Twitter
 *                         StreamingEndpoint to connect to
 * @return a stream source to use in {@link Pipeline#readFrom}
 */
@Nonnull
public static StreamSource<String> stream(
        @Nonnull Properties credentials,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return stream(credentials, Constants.STREAM_HOST, endpointSupplier);
}
 

/**
 * Variant of {@link #stream(Properties, SupplierEx)} which parses the
 * messages and used the {@code timestamp_ms} field as a native timestamp.
 * If you don't need this timestamp, prefer to use the other variant.
 * <p>
 * For parameter documentation {@linkplain #stream(Properties,
 * SupplierEx) see here}.
 */
@Nonnull
public static StreamSource<String> timestampedStream(
        @Nonnull Properties credentials,
        @Nonnull SupplierEx<? extends StreamingEndpoint> endpointSupplier
) {
    return timestampedStream(credentials, Constants.STREAM_HOST, endpointSupplier);
}
 

StreamingEndpoint createEndpoint(); 

StreamingEndpoint createEndpoint(); 

StreamingEndpoint createEndpoint(); 

public StreamingEndpoint getEndpoint();