com.twitter.hbc.core.HttpHosts Java Examples

private  Client getTwitterClient(Properties props, BlockingQueue<String> messageQueue) {

        String clientName = props.getProperty("clientName");
        String consumerKey = props.getProperty("consumerKey");
        String consumerSecret = props.getProperty("consumerSecret");
        String token = props.getProperty("token");
        String tokenSecret = props.getProperty("tokenSecret");
        List<String> searchTerms = Arrays.asList(props.getProperty("searchTerms").split(","));

        Authentication authentication = new OAuth1(consumerKey,consumerSecret,token,tokenSecret);
        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();

        hosebirdEndpoint.trackTerms(searchTerms);

        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name(clientName)
                .hosts(hosebirdHosts)
                .authentication(authentication)
                .endpoint(hosebirdEndpoint)
                .processor(new StringDelimitedProcessor(messageQueue));

          return clientBuilder.build();

    }
 

@Test
public void testIsScrambled() {
  int size = 50;
  List<String> hosts = new ArrayList<String>(size);
  for (int i = 0; i < size; i++) {
    hosts.add("http://thisismyawesomehost " + i + ".com");
  }
  HttpHosts httpHosts = new HttpHosts(hosts);

  boolean allSame = true;
  for (String string : hosts) {
    if (!string.equals(httpHosts.nextHost())) {
      allSame = false;
      break;
    }
  }
  assertFalse("This test failed unless you got EXTREMELY unlucky.", allSame);
}
 

@Test
public void testExceptionIfBadScheme() {
  int size = 50;
  List<String> hosts = new ArrayList<String>(size);
  for (int i = 0; i < size; i++) {
    if (i == size / 2) {
      hosts.add("thisfails.com");
    } else {
      hosts.add("https://thisismyawesomehost " + i + ".com");
    }
  }
  try {
    new HttpHosts(hosts);
    fail();
  } catch (RuntimeException e) {
    // is expected
  }
}
 

@Test
public void testProperlyHandleSuccessfulConnection() {
  ClientBase clientBase = new ClientBase("name",
          mock, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );

  mockStatusLine = mock(StatusLine.class);
  when(mockStatusLine.getStatusCode())
          .thenReturn(HttpConstants.Codes.SUCCESS);

  assertTrue(clientBase.handleConnectionResult(mockStatusLine));

  InOrder inOrder = inOrder(mockStatusLine, mockReconnectionManager);
  inOrder.verify(mockStatusLine).getStatusCode();
  inOrder.verify(mockReconnectionManager).resetCounts();

  assertFalse(clientBase.isDone());
}
 

@Test
public void testHandleIOExceptionOnConnection() throws IOException {
  ClientBase clientBase = new ClientBase("name",
          mock, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );
  when(mockConnection.connect(any(HttpUriRequest.class)))
          .thenThrow(new IOException());

  HttpUriRequest mockRequest = mock(HttpUriRequest.class);
  assertNull(clientBase.establishConnection(mockConnection, mockRequest));

  InOrder inOrder = inOrder(mockConnection, mockReconnectionManager);

  inOrder.verify(mockConnection).connect(any(HttpUriRequest.class));
  inOrder.verify(mockReconnectionManager).handleLinearBackoff();

  assertFalse(clientBase.isDone());
}
 

@Test
public void testRetryTransientAuthFailures() {
  ClientBase clientBase = new ClientBase("name",
          mock, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );

  when(mockStatusLine.getStatusCode())
          .thenReturn(401);
  when(mockStatusLine.getReasonPhrase())
          .thenReturn("reason");
  when(mockReconnectionManager.shouldReconnectOn400s())
          .thenReturn(true, true, false);

  // auth failure 3 times. We'll retry the first two times, but give up on the 3rd
  clientBase.handleConnectionResult(mockStatusLine);
  clientBase.handleConnectionResult(mockStatusLine);
  verify(mockReconnectionManager, times(2)).handleExponentialBackoff();
  assertFalse(clientBase.isDone());
  clientBase.handleConnectionResult(mockStatusLine);
  verify(mockReconnectionManager, times(2)).handleExponentialBackoff();
  assertTrue(clientBase.isDone());
}
 

@Test
public void testServiceUnavailable() {
  ClientBase clientBase = new ClientBase("name",
          mock, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );
  when(mockStatusLine.getStatusCode())
          .thenReturn(503);

  clientBase.handleConnectionResult(mockStatusLine);
  clientBase.handleConnectionResult(mockStatusLine);
  clientBase.handleConnectionResult(mockStatusLine);
  clientBase.handleConnectionResult(mockStatusLine);

  verify(mockReconnectionManager, times(4)).handleExponentialBackoff();
  assertFalse(clientBase.isDone());
}
 

@Test
public void testInterruptedExceptionDuringProcessing() throws Exception {
  ClientBase clientBase = new ClientBase("name",
          mockClient, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );

  when(mockStatusLine.getStatusCode())
          .thenReturn(200);

  doThrow(new InterruptedException()).when(mockProcessor).process();

  when(mockClient.getConnectionManager())
          .thenReturn(mockConnectionManager);

  BasicClient client = new BasicClient(clientBase, executorService);

  assertFalse(clientBase.isDone());
  client.connect();
  assertTrue(client.waitForFinish(100));
  assertTrue(client.isDone());
  verify(mockProcessor).setup(any(InputStream.class));
  verify(mockConnectionManager, atLeastOnce()).shutdown();
  assertEquals(EventType.STOPPED_BY_ERROR, client.getExitEvent().getEventType());
  assertTrue(client.getExitEvent().getUnderlyingException() instanceof InterruptedException);
}
 

@Before
public void setup() {
  client = mock(HttpClient.class);
  hosts = new HttpHosts("https://host.com");
  auth = mock(Authentication.class);
  request = mock(HttpUriRequest.class);
  mockResponse = mock(HttpResponse.class);
  mockStatusLine = mock(StatusLine.class);
  mockEntity = mock(HttpEntity.class);
}
 

public TwitterStream(SentimentDetector sentimentDetector,
                     Gson gson,
                     Properties props) {
    this.sentimentDetector = sentimentDetector;
    this.gson = gson;

    msgQueue = new LinkedBlockingQueue<String>(100000);

    Hosts hosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

    List<String> terms = Lists.newArrayList(
            props.getProperty("twitter_terms")
                    .split("\\s*,\\s*"));
    endpoint.trackTerms(terms);

    Authentication auth = new OAuth1(
            props.getProperty("twitter_consumer_key"),
            props.getProperty("twitter_consumer_secret"),
            props.getProperty("twitter_token"),
            props.getProperty("twitter_token_secret"));
    ClientBuilder builder = new ClientBuilder()
            .name("SmartCode-Client-01")
            .hosts(hosts)
            .authentication(auth)
            .endpoint(endpoint)
            .processor(new StringDelimitedProcessor(msgQueue));
    client = builder.build();
    client.connect();
}
 

@Test
public void testInfiniteIteration() {
  int size = 10;
  List<String> hosts = new ArrayList<String>(size);
  for (int i = 0; i < size; i++) {
    hosts.add("http://thisismyawesomehost " + i + ".com");
  }
  HttpHosts httpHosts = new HttpHosts(hosts);
  Set<String> hostsSet = new HashSet<String>(hosts);
  for (int i = 0; i < size * 10; i++) {
    assertTrue(hostsSet.contains(httpHosts.nextHost()));
  }
}
 

@Test
public void testContainsAll() {
  int size = 30;
  List<String> hosts = new ArrayList<String>(size);
  for (int i = 0; i < size; i++) {
    hosts.add("http://thisismyawesomehost " + i + ".com");
  }
  HttpHosts httpHosts = new HttpHosts(hosts);
  Set<String> hostsSet = new HashSet<String>(hosts);
  for (int i = 0; i < size; i++) {
    assertTrue(hostsSet.remove(httpHosts.nextHost()));
  }
  assertTrue(hostsSet.isEmpty());
}
 

@Test
public void testValidHttpMethod() {
  new ClientBuilder()
          .hosts(new HttpHosts(Constants.STREAM_HOST))
          .endpoint(StatusesSampleEndpoint.PATH, "gEt")
          .processor(new NullProcessor())
          .authentication(new BasicAuth("username", "password"))
          .build();

}
 

@Test
public void testInvalidHttpMethod() {
  try {
    new ClientBuilder()
            .hosts(new HttpHosts(Constants.STREAM_HOST))
            .endpoint(StatusesSampleEndpoint.PATH, "FAIL!")
            .processor(new NullProcessor())
            .authentication(new BasicAuth("username", "password"))
            .build();
    fail();
  } catch (Exception e) {
    // expected
  }
}
 

@Test
public void testBuilderSuccess() {
  new ClientBuilder()
          .hosts(new HttpHosts(Constants.STREAM_HOST))
          .endpoint(new StatusesSampleEndpoint())
          .processor(new NullProcessor())
          .authentication(new BasicAuth("username", "password"))
          .build();

}
 

@Test
public void testIOExceptionDuringProcessing() throws Exception {
  ClientBase clientBase = new ClientBase("name",
          mockClient, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );
  BasicClient client = new BasicClient(clientBase, executorService);
  final CountDownLatch latch = new CountDownLatch(1);
  when(mockStatusLine.getStatusCode())
          .thenReturn(200);

  doNothing().when(mockProcessor).setup(any(InputStream.class));
  doThrow(new IOException()).
          doThrow(new IOException()).
          doThrow(new IOException()).
          doAnswer(new Answer() {
            @Override
            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
              latch.countDown();
              return null;
            }
          }).when(mockProcessor).process();

  client.connect();
  latch.await();
  assertFalse(clientBase.isDone());
  verify(mockProcessor, times(4)).setup(any(InputStream.class));
  // throw 3 exceptions, 4th one keeps going
  verify(mockProcessor, atLeast(4)).process();

  client.stop();
  verify(mockConnectionManager, atLeastOnce()).shutdown();
  assertTrue(client.isDone());
  assertEquals(EventType.STOPPED_BY_USER, clientBase.getExitEvent().getEventType());
}
 

@Test
public void testUnknownEndpointFails() {
  ClientBase clientBase = new ClientBase("name",
          mock, new HttpHosts("http://hi"), new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );
  when(mockStatusLine.getStatusCode())
          .thenReturn(404);
  when(mockStatusLine.getReasonPhrase())
          .thenReturn("reason");
  clientBase.handleConnectionResult(mockStatusLine);
  assertTrue(clientBase.isDone());
}
 

public TwitterStream(Gson gson, Properties props, BlockingQueue<Map<String,Object>> imageQueue) {
    this.gson = gson;
    this.imageQueue = imageQueue;

    msgQueue = new LinkedBlockingQueue<String>(100000);

    Hosts hosts = new HttpHosts(Constants.STREAM_HOST);
    StatusesFilterEndpoint endpoint = new StatusesFilterEndpoint();

    List<String> terms = Lists.newArrayList(
            props.getProperty("twitter_terms")
                    .split("\\s*,\\s*"));
    endpoint.trackTerms(terms);

    Authentication auth = new OAuth1(
            props.getProperty("twitter_consumer_key"),
            props.getProperty("twitter_consumer_secret"),
            props.getProperty("twitter_token"),
            props.getProperty("twitter_token_secret"));
    ClientBuilder builder = new ClientBuilder()
            .name("SmartCode-Client-01")
            .hosts(hosts)
            .authentication(auth)
            .endpoint(endpoint)
            .processor(new StringDelimitedProcessor(msgQueue));
    client = builder.build();
    client.connect();
}
 

@Test
public void testConnectionRetries() throws Exception {
  HttpHosts mockHttpHosts = mock(HttpHosts.class);
  ClientBase clientBase = new ClientBase("name",
          mockClient, mockHttpHosts, new RawEndpoint("/endpoint", HttpConstants.HTTP_GET), mockAuth,
          mockProcessor, mockReconnectionManager, mockRateTracker
  );

  BasicClient client = new BasicClient(clientBase, executorService);
  final CountDownLatch latch = new CountDownLatch(1);
  when(mockHttpHosts.nextHost())
          .thenReturn("http://somehost.com");
  when(mockClient.execute(any(HttpUriRequest.class)))
          .thenReturn(mockResponse)
          .thenReturn(mockResponse)
          .thenThrow(new IOException())
          .thenReturn(mockResponse);
  when(mockStatusLine.getStatusCode())
          .thenReturn(HttpConstants.Codes.UNAUTHORIZED)
          .thenReturn(HttpConstants.Codes.SERVICE_UNAVAILABLE)
          .thenReturn(HttpConstants.Codes.SUCCESS);

  // turn off the client when we start processing
  doAnswer(new Answer() {
    @Override
    public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
      latch.countDown();
      return null;
    }
  }).when(mockProcessor).process();

  // for 401 Unauthorized
  when(mockReconnectionManager.shouldReconnectOn400s()).thenReturn(true);

  /** for shutdown **/
  when(mockClient.getConnectionManager())
          .thenReturn(mockConnectionManager);

  assertFalse(clientBase.isDone());
  client.connect();
  latch.await();
  client.stop();
  assertTrue(client.isDone());

  // exponential backoff twice: once for 401 once for 503
  verify(mockReconnectionManager, times(2)).handleExponentialBackoff();
  // for thrown IOException
  verify(mockReconnectionManager).handleLinearBackoff();
  // for successful connection
  verify(mockReconnectionManager).resetCounts();

  // finally start setting up processor/processing for the last attempt that goes through
  verify(mockProcessor, atLeastOnce()).setup(any(InputStream.class));
  verify(mockProcessor, atLeastOnce()).process();

  assertEquals(EventType.STOPPED_BY_USER, clientBase.getExitEvent().getEventType());
  verify(mockConnectionManager, atLeastOnce()).shutdown();
}
 

/**
 * @param host Http host in the form of <scheme>://<host>
 */
public ClientBuilder hosts(String host) {
  this.hosts = new HttpHosts(Preconditions.checkNotNull(host));
  return this;
}
 

/**
 * Construct a sitestream controller using the default host (sitestream.twitter.com) and a
 * DefaultHttpClient.
 */
public SitestreamController(Authentication auth) {
  this.client = new DefaultHttpClient(new PoolingClientConnectionManager());
  this.hosts = new HttpHosts(Constants.SITESTREAM_HOST);
  this.auth = Preconditions.checkNotNull(auth);
}
 

/**
 * Construct a sitestream controller using the default host (sitestream.twitter.com)
 */
public SitestreamController(HttpClient client, Authentication auth) {
  this.client = Preconditions.checkNotNull(client);
  this.hosts = new HttpHosts(Constants.SITESTREAM_HOST);
  this.auth = Preconditions.checkNotNull(auth);
}
 

public static void main(String[] args)  throws InterruptedException {

        BlockingQueue<String> msgQueue = new LinkedBlockingDeque<>();



        Hosts hosebirdHosts = new HttpHosts(Constants.STREAM_HOST);
        StatusesFilterEndpoint hosebirdEndpoint = new StatusesFilterEndpoint();
        List<String> terms = Lists.newArrayList("superman vs batman","#supermanvsbatman");
        hosebirdEndpoint.trackTerms(terms);




        Authentication hosebirdAuth  = new OAuth1("18qydWMuiUohwCtQpp1MOFCFr",
                                                  "YrYhYd09LKZLbhsKT1o4XcEPl6HiAoNykiOxYBq0dAB8t0vRCo",
                                                  "16972669-KSvyDEMc7dussPfW6a9Ru65L4eWGj637ciHLHZLyn",
                                                  "ky53NE6cbBvtNLopto7o9gVyHDejSB2kPsRhHGKEd1MrS");


        ClientBuilder clientBuilder = new ClientBuilder();
        clientBuilder.name("bbejeck-hosebird")
                     .hosts(hosebirdHosts)
                     .authentication(hosebirdAuth)
                     .endpoint(hosebirdEndpoint)
                     .processor(new StringDelimitedProcessor(msgQueue));

        Client hosebirdClient = clientBuilder.build();
        hosebirdClient.connect();

        for (int msgRead = 0; msgRead < 100; msgRead++) {
            String msg = msgQueue.take();
            System.out.println(msg);
        }

        hosebirdClient.stop();

    }