Java Code Examples for org.apache.flume.Sink#Status

The following examples show how to use org.apache.flume.Sink#Status . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StringSourceTests.java    From pulsar with Apache License 2.0 5 votes vote down vote up 
@Test
public void TestOpenAndReadSource() throws Exception {
    Map<String, Object> conf = Maps.newHashMap();
    StringSource stringSource = new StringSource();
    conf.put("name", "a1");
    conf.put("confFile", "./src/test/resources/flume/sink.conf");
    conf.put("noReloadConf", false);
    conf.put("zkConnString", "");
    conf.put("zkBasePath", "");
    Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
    stringSource.open(conf, mockSourceContext);
    Thread.sleep(3 * 1000);
    sink.start();
    Transaction transaction = channel.getTransaction();

    transaction.begin();
    for (int i = 0; i < 10; i++) {
        channel.put(event);
    }
    transaction.commit();
    transaction.close();

    for (int i = 0; i < 5; i++) {
        Sink.Status status = sink.process();
        Assert.assertEquals(Sink.Status.READY, status);
    }

    Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
    stringSource.close();
}
Example 2
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 5 votes vote down vote up 
@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
Example 3
Source File: TestThriftSink.java    From mt-flume with Apache License 2.0 5 votes vote down vote up 
@Test
public void testProcess() throws Exception {

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port);

  channel.start();
  sink.start();

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();
  for (int i = 0; i < 6; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertEquals(11, src.flumeEvents.size());
  Assert.assertEquals(6, src.batchCount);
  Assert.assertEquals(0, src.individualCount);

}
Example 4
Source File: TestLoadBalancingSinkProcessor.java    From mt-flume with Apache License 2.0 5 votes vote down vote up 
@Test
public void testCustomSelector() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  // This selector will result in all events going to s2
  Context ctx = getContext(FixedOrderSelector.class.getCanonicalName());
  ctx.put("selector." + FixedOrderSelector.SET_ME, "foo");
  LoadBalancingSinkProcessor lbsp = getProcessor(sinks, ctx);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}
Example 5
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFailedConnect() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {

  setUp();
  Event event = EventBuilder.withBody("test event 1",
      Charset.forName("UTF8"));
  Server server = createServer(new MockAvroServer());

  server.start();
  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Thread.sleep(500L); // let socket startup
  server.close();
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
        threwException);
  }

  server = createServer(new MockAvroServer());
  server.start();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));
  server.close();
}
Example 6
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
Example 7
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testSslProcessWithTrustStore() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("truststore", "src/test/resources/truststore.jks");
  context.put("truststore-password", "password");
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
Example 8
Source File: TestThriftSink.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFailedConnect() throws Exception {

  Event event = EventBuilder.withBody("test event 1",
    Charset.forName("UTF8"));

  sink.start();

  Thread.sleep(500L); // let socket startup
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
      threwException);
  }

  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port);

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());
  sink.stop();
}
Example 9
Source File: TestLoadBalancingSinkProcessor.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRandomOneActiveSink() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);


  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("random", sinks, false);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}
Example 10
Source File: TestLoadBalancingSinkProcessor.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRandomBackoff() throws Exception {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("random", sinks, true);

  // TODO: there is a remote possibility that s0 or s2
  // never get hit by the random assignment
  // and thus not backoffed, causing the test to fail
  for(int i=0; i < 50; i++) {
    // a well behaved runner would always check the return.
    lbsp.process();
  }
  Assert.assertEquals(50, s2.getEvents().size());
  s2.setFail(true);
  s1.setFail(false); // s1 should still be backed off
  try {
    lbsp.process();
    // nothing should be able to process right now
    Assert.fail("Expected EventDeliveryException");
  } catch (EventDeliveryException e) {
    // this is expected
  }
  Thread.sleep(2100); // wait for s1 to no longer be backed off
  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals(50, s1.getEvents().size());
  Assert.assertEquals(50, s2.getEvents().size());
  Assert.assertEquals(0, s3.getEvents().size());
}
Example 11
Source File: TestLoadBalancingSinkProcessor.java    From mt-flume with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRoundRobinOneActiveSink() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);


  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  // s3 always fails
  s3.setFail(true);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin", sinks, false);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}