org.apache.flume.lifecycle.LifecycleController Java Examples

@Test
public void testLifecycle() throws InterruptedException,
    InstantiationException, IllegalAccessException {
  setUp();
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

private void bind() throws InterruptedException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("host", "0.0.0.0");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (FlumeException e) {
      // Assume port in use, try another one
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
          source.getLifecycleState());
}
 

@Before
public void setUp() throws Exception {

    baseDir = Files.createTempDir();

    configFile = new File(baseDir, TESTFILE.getName());
    Files.copy(TESTFILE, configFile);

    eventBus = new EventBus("test");
    provider =
            new PollingPropertiesFileConfigurationProvider("host1",
                    configFile, eventBus, 1);
    provider.start();
    LifecycleController.waitForOneOf(provider, LifecycleState.START_OR_ERROR);
}
 

@After
public void teardown() throws Exception {
    removeAppenders(avroLogger);
    eventSource.stop();
    Assert.assertTrue("Reached stop or error", LifecycleController
            .waitForOneOf(eventSource, LifecycleState.STOP_OR_ERROR));
    Assert.assertEquals("Server is stopped", LifecycleState.STOP,
            eventSource.getLifecycleState());
}
 

@Before
public void setUp() throws Exception {
    eventSource = new AvroSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    avroLogger = (Logger) LogManager.getLogger("avrologger");
    /*
     * Clear out all other appenders associated with this logger to ensure
     * we're only hitting the Avro appender.
     */
    removeAppenders(avroLogger);
    final Context context = new Context();
    testPort = String.valueOf(AvailablePortFinder.getNextAvailable());
    context.put("port", testPort);
    context.put("bind", "0.0.0.0");
    Configurables.configure(eventSource, context);

    final List<Channel> channels = new ArrayList<>();
    channels.add(channel);

    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(channels);

    eventSource.setChannelProcessor(new ChannelProcessor(cs));

    eventSource.start();

    Assert.assertTrue("Reached start or error", LifecycleController
            .waitForOneOf(eventSource, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
            eventSource.getLifecycleState());
}
 

@Test
public void testLifecycle() throws InterruptedException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("host", "0.0.0.0");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      // Assume port in use, try another one
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

private void stop() throws InterruptedException {
  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Before
public void setUp() throws Exception {

  baseDir = Files.createTempDir();

  configFile = new File(baseDir, TESTFILE.getName());
  Files.copy(TESTFILE, configFile);

  eventBus = new EventBus("test");
  provider =
      new PollingPropertiesFileConfigurationProvider("host1",
          configFile, eventBus, 1);
  provider.start();
  LifecycleController.waitForOneOf(provider, LifecycleState.START_OR_ERROR);
}
 

@Test
public void testLifecycle() throws IOException, InterruptedException {
  Context context = new Context();
  File f1 = new File(tmpDir.getAbsolutePath() + "/file1");

  Files.write("file1line1\nfile1line2\nfile1line3\nfile1line4\n" +
              "file1line5\nfile1line6\nfile1line7\nfile1line8\n",
              f1, Charsets.UTF_8);

  context.put(SpoolDirectorySourceConfigurationConstants.SPOOL_DIRECTORY,
      tmpDir.getAbsolutePath());

  Configurables.configure(source, context);

  for (int i = 0; i < 10; i++) {
    source.start();

    Assert
        .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
            source, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
        source.getLifecycleState());

    source.stop();
    Assert.assertTrue("Reached stop or error",
        LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
    Assert.assertEquals("Server is stopped", LifecycleState.STOP,
        source.getLifecycleState());
  }
}
 

@Test
public void testLifecycle() throws InterruptedException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Override
protected void doSetUp() throws Exception {
    eb = new EventBus("test");
    es = new EventSync();
    es.reset();
    eb.register(es);
    cp = new PollingZooKeeperConfigurationProvider(AGENT_NAME, "localhost:"
            + zkServer.getPort(), null, eb);
    cp.start();
    LifecycleController.waitForOneOf(cp, LifecycleState.START_OR_ERROR);
}
 

private void doRequest(boolean serverEnableCompression, boolean clientEnableCompression, int compressionLevel) throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();
      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("threads", "50");
      if (serverEnableCompression) {
        context.put("compression-type", "deflate");
      } else {
        context.put("compression-type", "none");
      }

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client;
  if (clientEnableCompression) {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort), new CompressionChannelFactory(6)));
  } else {
    client = SpecificRequestor.getClient(
        AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
            selectedPort)));
  }

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Test
public void testSslRequest() throws InterruptedException, IOException {
  boolean bound = false;

  for (int i = 0; i < 10 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("bind", "0.0.0.0");
      context.put("ssl", "true");
      context.put("keystore", "src/test/resources/server.p12");
      context.put("keystore-password", "password");
      context.put("keystore-type", "PKCS12");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      /*
       * NB: This assume we're using the Netty server under the hood and the
       * failure is to bind. Yucky.
       */
      Thread.sleep(100);
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  AvroSourceProtocol client = SpecificRequestor.getClient(
      AvroSourceProtocol.class, new NettyTransceiver(new InetSocketAddress(
      selectedPort), new SSLChannelFactory()));

  AvroFlumeEvent avroEvent = new AvroFlumeEvent();

  avroEvent.setHeaders(new HashMap<CharSequence, CharSequence>());
  avroEvent.setBody(ByteBuffer.wrap("Hello avro ssl".getBytes()));

  Status status = client.append(avroEvent);

  Assert.assertEquals(Status.OK, status);

  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "Hello avro ssl",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  logger.debug("Round trip event:{}", event);

  source.stop();
  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}
 

@Test
public void testSslSinkWithNonTrustedCert() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  boolean failed = false;
  try {
    for (int i = 0; i < 5; i++) {
      sink.process();
      failed = true;
    }
  } catch (EventDeliveryException ex) {
    logger.info("Correctly failed to send event", ex);
  }


  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();

  if (failed) {
    Assert.fail("SSL-enabled sink successfully connected to a server with an untrusted certificate when it should have failed");
  }
}
 

@Test
public void testSslSinkWithNonSslServer() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  boolean failed = false;
  try {
    for (int i = 0; i < 5; i++) {
      sink.process();
      failed = true;
    }
  } catch (EventDeliveryException ex) {
    logger.info("Correctly failed to send event", ex);
  }


  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();

  if (failed) {
    Assert.fail("SSL-enabled sink successfully connected to a non-SSL-enabled server, that's wrong.");
  }
}
 

@Test
public void testSslProcessWithTrustStore() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("truststore", "src/test/resources/truststore.jks");
  context.put("truststore-password", "password");
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

@Test
public void testFailedConnect() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {

  setUp();
  Event event = EventBuilder.withBody("test event 1",
      Charset.forName("UTF8"));
  Server server = createServer(new MockAvroServer());

  server.start();
  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Thread.sleep(500L); // let socket startup
  server.close();
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
        threwException);
  }

  server = createServer(new MockAvroServer());
  server.start();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));
  server.close();
}
 

@Test
public void testRequest() throws InterruptedException, IOException {
  boolean bound = false;
  int i;

  for (i = 0; i < 100 && !bound; i++) {
    try {
      Context context = new Context();

      context.put("port", String.valueOf(selectedPort = 41414 + i));
      context.put("host", "0.0.0.0");

      Configurables.configure(source, context);

      source.start();
      bound = true;
    } catch (ChannelException e) {
      // Assume port in use, try another one
    }
  }

  Assert
      .assertTrue("Reached start or error", LifecycleController.waitForOneOf(
          source, LifecycleState.START_OR_ERROR));
  Assert.assertEquals("Server is started", LifecycleState.START,
      source.getLifecycleState());

  // setup a requester, to send a flume OG event
  URL url = new URL("http", "0.0.0.0", selectedPort, "/");
  Transceiver http = new HttpTransceiver(url);
  FlumeOGEventAvroServer client = SpecificRequestor.getClient(
      FlumeOGEventAvroServer.class, http);

  AvroFlumeOGEvent avroEvent =  AvroFlumeOGEvent.newBuilder().setHost("foo").
      setPriority(Priority.INFO).setNanos(0).setTimestamp(1).
      setFields(new HashMap<CharSequence, ByteBuffer> ()).
      setBody(ByteBuffer.wrap("foo".getBytes())).build();

  client.append(avroEvent);

  // check if the even has arrived in the channel through OG avro source
  Transaction transaction = channel.getTransaction();
  transaction.begin();

  Event event = channel.take();
  Assert.assertNotNull(event);
  Assert.assertEquals("Channel contained our event", "foo",
      new String(event.getBody()));
  transaction.commit();
  transaction.close();

  source.stop();

  Assert.assertTrue("Reached stop or error",
      LifecycleController.waitForOneOf(source, LifecycleState.STOP_OR_ERROR));
  Assert.assertEquals("Server is stopped", LifecycleState.STOP,
      source.getLifecycleState());
}