org.apache.flume.event.EventBuilder Java Examples

The following examples show how to use org.apache.flume.event.EventBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestMemoryChannel.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testPutTake() throws InterruptedException, EventDeliveryException {
  Event event = EventBuilder.withBody("test event".getBytes());
  Context context = new Context();

  Configurables.configure(channel, context);

  Transaction transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  channel.put(event);
  transaction.commit();
  transaction.close();

  transaction = channel.getTransaction();
  Assert.assertNotNull(transaction);

  transaction.begin();
  Event event2 = channel.take();
  Assert.assertEquals(event, event2);
  transaction.commit();
}
 
Example #2
Source File: FlumeEventAvroEventDeserializer.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Override
public Event readEvent() throws IOException {
  if (fileReader.hasNext()) {
    record = fileReader.next(record);

    HashMap<String, String> newHeaders = new HashMap<String, String>();
    HashMap<Utf8, Utf8> headers = (HashMap<Utf8, Utf8>)record.get("headers");
    newHeaders.put("timestamp", Long.valueOf((new Date().getTime())).toString());
    newHeaders.put("category", headers.get(new Utf8("category")).toString());

    ByteBuffer body = (ByteBuffer)record.get("body");
    int length = body.limit() - body.position();
    byte[] bodyBytes = new byte[length];
    System.arraycopy(body.array(), body.position(), bodyBytes, 0, length);

    return EventBuilder.withBody(bodyBytes, newHeaders);
  }
  return null;
}
 
Example #3
Source File: KafkaSinkTest.java    From flume-ng-kafka-sink with Apache License 2.0 6 votes vote down vote up
private Sink.Status prepareAndSend(Context context, String msg) throws EventDeliveryException {
    Sink kafkaSink = new KafkaSink();
    Configurables.configure(kafkaSink, context);
    Channel memoryChannel = new MemoryChannel();
    Configurables.configure(memoryChannel, context);
    kafkaSink.setChannel(memoryChannel);
    kafkaSink.start();

    Transaction tx = memoryChannel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody(msg.getBytes());
    memoryChannel.put(event);
    tx.commit();
    tx.close();

    return kafkaSink.process();
}
 
Example #4
Source File: TestStaticInterceptor.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testPreserve() throws ClassNotFoundException,
    InstantiationException, IllegalAccessException {
  Interceptor.Builder builder = InterceptorBuilderFactory.newInstance(
      InterceptorType.STATIC.toString());
  Context ctx = new Context();
  ctx.put(Constants.PRESERVE, "true");
  ctx.put(Constants.VALUE, "replacement value");

  builder.configure(ctx);
  Interceptor interceptor = builder.build();

  Event event = EventBuilder.withBody("test", Charsets.UTF_8);
  event.getHeaders().put(Constants.KEY, "incumbent value");

  Assert.assertNotNull(event.getHeaders().get(Constants.KEY));

  event = interceptor.intercept(event);
  String val = event.getHeaders().get(Constants.KEY);

  Assert.assertNotNull(val);
  Assert.assertEquals("incumbent value", val);
}
 
Example #5
Source File: Consumer.java    From rabbitmq-flume-plugin with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private Event parseMessage(Envelope envelope, AMQP.BasicProperties props, byte[] body) {
    // Create the event passing in the body
    Event event = EventBuilder.withBody(body);

    // Get the headers from properties, exchange, and routing-key
    Map<String, String> headers = buildHeaders(props);

    String exchange = envelope.getExchange();
    if (exchange != null && !exchange.isEmpty()) {
        headers.put("exchange", exchange);
    }

    String routingKey = envelope.getRoutingKey();
    if (routingKey != null && !routingKey.isEmpty()) {
        headers.put("routing-key", routingKey);
    }

    event.setHeaders(headers);
    return event;
}
 
Example #6
Source File: TestNettyAvroRpcClient.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * First connect the client, then close the client, then send a request.
 * @throws FlumeException
 * @throws EventDeliveryException
 */
@Test(expected=EventDeliveryException.class)
public void testClientClosedRequest() throws FlumeException,
    EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcTestUtils.getStockLocalClient(server.getPort());
    client.close();
    Assert.assertFalse("Client should not be active", client.isActive());
    System.out.println("Yaya! I am not active after client close!");
    client.append(EventBuilder.withBody("hello", Charset.forName("UTF8")));
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #7
Source File: TestMorphlineInterceptor.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
/** morphline says route to southpole if it's an avro file, otherwise route to northpole */
public void testIfDetectMimeTypeRouteToSouthPole() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM, RESOURCES_DIR + "/test-morphlines/ifDetectMimeType.conf");
  context.put(MorphlineHandlerImpl.MORPHLINE_VARIABLE_PARAM + ".MY.MIME_TYPE", "avro/binary");

  Event input = EventBuilder.withBody(Files.toByteArray(new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.avro")));
  Event actual = build(context).intercept(input);

  Map<String, String> expected = new HashMap();
  expected.put(Fields.ATTACHMENT_MIME_TYPE, "avro/binary");
  expected.put("flume.selector.header", "goToSouthPole");
  Event expectedEvent = EventBuilder.withBody(input.getBody(), expected);
  assertEqualsEvent(expectedEvent, actual);
}
 
Example #8
Source File: TestNettyAvroRpcClient.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * First connect the client, then shut down the server, then send a request.
 * @throws FlumeException
 * @throws EventDeliveryException
 * @throws InterruptedException
 */
@Test(expected=EventDeliveryException.class)
public void testServerDisconnect() throws FlumeException,
    EventDeliveryException, InterruptedException {
  NettyAvroRpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcTestUtils.getStockLocalClient(server.getPort());
    server.close();
    Thread.sleep(1000L); // wait a second for the close to occur
    try {
      server.join();
    } catch (InterruptedException ex) {
      logger.warn("Thread interrupted during join()", ex);
      Thread.currentThread().interrupt();
    }
    try {
      client.append(EventBuilder.withBody("hello", Charset.forName("UTF8")));
    } finally {
      Assert.assertFalse("Client should not be active", client.isActive());
    }
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #9
Source File: TestAsyncHBaseSink.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test (expected = EventDeliveryException.class)
public void testTimeOut() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(),
    true);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  Assert.fail();
}
 
Example #10
Source File: ThriftSource.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Override
public Status appendBatch(List<ThriftFlumeEvent> events) throws TException {
  sourceCounter.incrementAppendBatchReceivedCount();
  sourceCounter.addToEventReceivedCount(events.size());

  List<Event> flumeEvents = Lists.newArrayList();
  for(ThriftFlumeEvent event : events) {
    flumeEvents.add(EventBuilder.withBody(event.getBody(),
      event.getHeaders()));
  }

  try {
    getChannelProcessor().processEventBatch(flumeEvents);
  } catch (ChannelException ex) {
    logger.warn("Thrift source %s could not append events to the " +
      "channel.", getName());
    return Status.FAILED;
  }

  sourceCounter.incrementAppendBatchAcceptedCount();
  sourceCounter.addToEventAcceptedCount(events.size());
  return Status.OK;
}
 
Example #11
Source File: AvroSource.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Override
public Status append(AvroFlumeEvent avroEvent) {
  logger.debug("Avro source {}: Received avro event: {}", getName(),
      avroEvent);
  sourceCounter.incrementAppendReceivedCount();
  sourceCounter.incrementEventReceivedCount();

  Event event = EventBuilder.withBody(avroEvent.getBody().array(),
      toStringMap(avroEvent.getHeaders()));

  try {
    getChannelProcessor().processEvent(event);
  } catch (ChannelException ex) {
    logger.warn("Avro source " + getName() + ": Unable to process event. " +
        "Exception follows.", ex);
    return Status.FAILED;
  }

  sourceCounter.incrementAppendAcceptedCount();
  sourceCounter.incrementEventAcceptedCount();

  return Status.OK;
}
 
Example #12
Source File: TestCassandraSink.java    From ingestion with Apache License 2.0 6 votes vote down vote up
@Test
public void processOneEventWithBatchSizeOne() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  ctx.put("batchSize", "1");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verify(table).save(ImmutableList.of(event));
  verifyNoMoreInteractions(table);
  verify(tx).begin();
  verify(tx).commit();
  verify(tx).close();
}
 
Example #13
Source File: TestBucketWriter.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testEventCountingRoller() throws IOException, InterruptedException {
  int maxEvents = 100;
  MockHDFSWriter hdfsWriter = new MockHDFSWriter();
  BucketWriter bucketWriter = new BucketWriter(0, 0, maxEvents, 0, ctx,
      "/tmp", "file", "", ".tmp", null, null, SequenceFile.CompressionType.NONE,
      hdfsWriter, timedRollerPool, null,
      new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()), 0,
      null, null, 30000, Executors.newSingleThreadExecutor());

  Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
  for (int i = 0; i < 1000; i++) {
    bucketWriter.append(e);
  }

  logger.info("Number of events written: {}", hdfsWriter.getEventsWritten());
  logger.info("Number of bytes written: {}", hdfsWriter.getBytesWritten());
  logger.info("Number of files opened: {}", hdfsWriter.getFilesOpened());

  Assert.assertEquals("events written", 1000, hdfsWriter.getEventsWritten());
  Assert.assertEquals("bytes written", 3000, hdfsWriter.getBytesWritten());
  Assert.assertEquals("files opened", 10, hdfsWriter.getFilesOpened());
}
 
Example #14
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testThreeParamBatchAppend() throws FlumeException,
    EventDeliveryException {
  int batchSize = 7;
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort(),
        batchSize);

    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < batchSize; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #15
Source File: RpcTestUtils.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method for testing simple (single) with compression level 6 appends on handlers
 * @param handler
 * @throws FlumeException
 * @throws EventDeliveryException
 */
public static void handlerSimpleAppendTest(AvroSourceProtocol handler, boolean enableServerCompression, boolean enableClientCompression, int compressionLevel)
    throws FlumeException, EventDeliveryException {
  NettyAvroRpcClient client = null;
  Server server = startServer(handler, 0, enableServerCompression);
  try {
    Properties starterProp = new Properties();
    if (enableClientCompression) {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "deflate");
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_LEVEL, "" + compressionLevel);
    } else {
      starterProp.setProperty(RpcClientConfigurationConstants.CONFIG_COMPRESSION_TYPE, "none");
    }
    client = getStockLocalClient(server.getPort(), starterProp);
    boolean isActive = client.isActive();
    Assert.assertTrue("Client should be active", isActive);
    client.append(EventBuilder.withBody("wheee!!!", Charset.forName("UTF8")));
  } finally {
    stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #16
Source File: TestMorphlineInterceptor.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testNoOperation() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM, RESOURCES_DIR + "/test-morphlines/noOperation.conf");
  Event input = EventBuilder.withBody("foo", Charsets.UTF_8);
  input.getHeaders().put("name", "nadja");
  MorphlineInterceptor interceptor = build(context);
  Event actual = interceptor.intercept(input);
  interceptor.close();
  Event expected = EventBuilder.withBody("foo".getBytes("UTF-8"), ImmutableMap.of("name", "nadja"));
  assertEqualsEvent(expected, actual);
  
  List<Event> actualList = build(context).intercept(Collections.singletonList(input));
  List<Event> expectedList = Collections.singletonList(expected);
  assertEqualsEventList(expectedList, actualList);
}
 
Example #17
Source File: TestBucketWriter.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testInUseSuffix() throws IOException, InterruptedException {
  final int ROLL_INTERVAL = 1000; // seconds. Make sure it doesn't change in course of test
  final String SUFFIX = "WELCOME_TO_THE_HELLMOUNTH";

  MockHDFSWriter hdfsWriter = new MockHDFSWriter();
  HDFSTextSerializer serializer = new HDFSTextSerializer();
  BucketWriter bucketWriter = new BucketWriter(ROLL_INTERVAL, 0, 0, 0, ctx,
      "/tmp", "file", "", SUFFIX, null, null, SequenceFile.CompressionType.NONE, hdfsWriter,
      timedRollerPool, null,
      new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()),
      0, null, null, 30000, Executors.newSingleThreadExecutor());

  Event e = EventBuilder.withBody("foo", Charsets.UTF_8);
  bucketWriter.append(e);

  Assert.assertTrue("Incorrect in use suffix", hdfsWriter.getOpenedFilePath().contains(SUFFIX));
}
 
Example #18
Source File: TestRpcClientFactory.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testTwoParamBatchAppendOverflow() throws FlumeException,
    EventDeliveryException {
  RpcClient client = null;
  Server server = RpcTestUtils.startServer(new OKAvroHandler());
  try {
    client = RpcClientFactory.getDefaultInstance(localhost, server.getPort());
    int batchSize = client.getBatchSize();
    int moreThanBatch = batchSize + 1;
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < moreThanBatch; i++) {
      events.add(EventBuilder.withBody("evt: " + i, Charset.forName("UTF8")));
    }
    client.appendBatch(events);
  } finally {
    RpcTestUtils.stopServer(server);
    if (client != null) client.close();
  }
}
 
Example #19
Source File: TestRegexExtractorInterceptor.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldExtractAddHeadersForAllMatchGroups() throws Exception {
  Context context = new Context();
  context.put(RegexExtractorInterceptor.REGEX, "(\\d):(\\d):(\\d)");
  context.put(RegexExtractorInterceptor.SERIALIZERS, "s1 s2 s3");
  context.put(RegexExtractorInterceptor.SERIALIZERS + ".s1.name", "Num1");
  context.put(RegexExtractorInterceptor.SERIALIZERS + ".s2.name", "Num2");
  context.put(RegexExtractorInterceptor.SERIALIZERS + ".s3.name", "Num3");

  fixtureBuilder.configure(context);
  Interceptor fixture = fixtureBuilder.build();

  Event event = EventBuilder.withBody("1:2:3.4foobar5", Charsets.UTF_8);

  Event expected = EventBuilder.withBody("1:2:3.4foobar5", Charsets.UTF_8);
  expected.getHeaders().put("Num1", "1");
  expected.getHeaders().put("Num2", "2");
  expected.getHeaders().put("Num3", "3");

  Event actual = fixture.intercept(event);

  Assert.assertArrayEquals(expected.getBody(), actual.getBody());
  Assert.assertEquals(expected.getHeaders(), actual.getHeaders());
}
 
Example #20
Source File: TestStaticInterceptor.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test
public void testReplace() throws ClassNotFoundException,
    InstantiationException, IllegalAccessException {
  Interceptor.Builder builder = InterceptorBuilderFactory.newInstance(
      InterceptorType.STATIC.toString());
  Context ctx = new Context();
  ctx.put(Constants.PRESERVE, "false");
  ctx.put(Constants.VALUE, "replacement value");

  builder.configure(ctx);
  Interceptor interceptor = builder.build();

  Event event = EventBuilder.withBody("test", Charsets.UTF_8);
  event.getHeaders().put(Constants.KEY, "incumbent value");

  Assert.assertNotNull(event.getHeaders().get(Constants.KEY));

  event = interceptor.intercept(event);
  String val = event.getHeaders().get(Constants.KEY);

  Assert.assertNotNull(val);
  Assert.assertEquals("replacement value", val);
}
 
Example #21
Source File: AvroLegacySource.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Override
public Void append( AvroFlumeOGEvent evt ) throws AvroRemoteException {
  counterGroup.incrementAndGet("rpc.received");
  Map<String, String> headers = new HashMap<String, String>();

  // extract Flume OG event headers
  headers.put(HOST, evt.getHost().toString());
  headers.put(TIMESTAMP, evt.getTimestamp().toString());
  headers.put(PRIORITY, evt.getPriority().toString());
  headers.put(NANOS, evt.getNanos().toString());
  for (Entry<CharSequence, ByteBuffer> entry : evt.getFields().entrySet()) {
    headers.put(entry.getKey().toString(), entry.getValue().toString());
  }
  headers.put(OG_EVENT, "yes");

  Event event = EventBuilder.withBody(evt.getBody().array(), headers);
  try {
    getChannelProcessor().processEvent(event);
    counterGroup.incrementAndGet("rpc.events");
  } catch (ChannelException ex) {
    return null;
  }

  counterGroup.incrementAndGet("rpc.successful");
  return null;
}
 
Example #22
Source File: FlumeSinkExample.java    From bahir-flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    //FlumeSink send data
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    FlumeEventBuilder<String> flumeEventBuilder = new FlumeEventBuilder<String>() {
        @Override
        public Event createFlumeEvent(String value, RuntimeContext ctx) {
            return EventBuilder.withBody(value, Charset.defaultCharset());
        }
    };

    FlumeSink<String> flumeSink = new FlumeSink<>(clientType, hostname, port, flumeEventBuilder, 1, 1, 1);

    // Note: parallelisms and FlumeSink batchSize
    // if every parallelism not enough batchSize, this parallelism not word FlumeThriftService output
    DataStreamSink<String> dataStream = env.fromElements("one", "two", "three", "four", "five")
            .addSink(flumeSink);

    env.execute();
}
 
Example #23
Source File: XmlXpathDeserializer.java    From ingestion with Apache License 2.0 6 votes vote down vote up
@Override
public Event readEvent() throws IOException {
    ensureOpen();
    if (!currentIt.hasNext()) {
        return null;
    } else {
        final String node = currentIt.next();
        if (outputBody) {
          return EventBuilder.withBody(node, Charsets.UTF_8);
        } else {
          final Event event = EventBuilder.withBody(body, Charsets.UTF_8);
          event.getHeaders().put(outputHeader, node);
          return event;
        }
    }
}
 
Example #24
Source File: TestEmbeddedAgent.java    From mt-flume with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 30000L)
public void testPutAll() throws Exception {
  List<Event> events = Lists.newArrayList();
  events.add(EventBuilder.withBody(body, headers));
  agent.configure(properties);
  agent.start();
  agent.putAll(events);

  Event event;
  while((event = eventCollector.poll()) == null) {
    Thread.sleep(500L);
  }
  Assert.assertNotNull(event);
  Assert.assertArrayEquals(body, event.getBody());
  Assert.assertEquals(headers, event.getHeaders());
}
 
Example #25
Source File: TestMemoryChannel.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testCapacityBufferEmptyingAfterTakeCommit() {
  Context context = new Context();
  Map<String, String> parms = new HashMap<String, String>();
  parms.put("capacity", "3");
  parms.put("transactionCapacity", "3");
  context.putAll(parms);
  Configurables.configure(channel,  context);

  Transaction tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.take();
  channel.take();
  tx.commit();
  tx.close();

  tx = channel.getTransaction();
  tx.begin();
  channel.put(EventBuilder.withBody("test".getBytes()));
  channel.put(EventBuilder.withBody("test".getBytes()));
  tx.commit();
  tx.close();
}
 
Example #26
Source File: TestCassandraSink.java    From ingestion with Apache License 2.0 5 votes vote down vote up
@Ignore @Test
public void processDriverException() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  doThrow(DriverException.class).when(table).save(anyListOf(Event.class));
  boolean hasThrown = false;
  try {
    sink.process();
  } catch (EventDeliveryException ex) {
    hasThrown = true;
    if (!(ex.getCause() instanceof DriverException)) {
      fail("Did not throw inner DriverException: " + ex);
    }
  }
  verify(tx).begin();
  verify(tx).rollback();
  verify(tx).close();
  verifyNoMoreInteractions(tx);
  if (!hasThrown) {
    fail("Did not throw exception");
  }
}
 
Example #27
Source File: TestAvroSink.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 
Example #28
Source File: TestThriftRpcClient.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
/**
 * Insert events 0..count-1
 *
 * @param client
 * @param count
 * @throws Exception
 */
public static void insertEvents(RpcClient client,
                                int count) throws Exception {
  for (int i = 0; i < count; i++) {
    Map<String, String> header = new HashMap<String, String>();
    header.put(SEQ, String.valueOf(i));
    client.append(EventBuilder.withBody(String.valueOf(i).getBytes(),
      header));
  }
}
 
Example #29
Source File: ScribeSource.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
public ResultCode Log(List<LogEntry> list) throws TException {
  if (list != null) {
    sourceCounter.addToEventReceivedCount(list.size());

    try {
      List<Event> events = new ArrayList<Event>(list.size());

      for (LogEntry entry : list) {
        Map<String, String> headers = new HashMap<String, String>(1, 1);
        headers.put(SCRIBE_CATEGORY, entry.getCategory());

        Event event = EventBuilder.withBody(entry.getMessage().getBytes(), headers);
        events.add(event);
      }

      if (events.size() > 0) {
        getChannelProcessor().processEventBatch(events);
      }

      sourceCounter.addToEventAcceptedCount(list.size());
      return ResultCode.OK;
    } catch (Exception e) {
      LOG.warn("Scribe source handling failure", e);
    }
  }

  return ResultCode.TRY_LATER;
}
 
Example #30
Source File: TestFlumeEventAvroEventSerializer.java    From mt-flume with Apache License 2.0 5 votes vote down vote up
public void createAvroFile(File file, String codec)
    throws FileNotFoundException, IOException {

  if(file.exists()){
    FileUtils.forceDelete(file);
  }

  // serialize a few events using the reflection-based avro serializer
  OutputStream out = new FileOutputStream(file);

  Context ctx = new Context();
  if (codec != null) {
    ctx.put("compressionCodec", codec);
  }

  EventSerializer.Builder builder =
      new FlumeEventAvroEventSerializer.Builder();
  EventSerializer serializer = builder.build(ctx, out);

  serializer.afterCreate();
  serializer.write(EventBuilder.withBody("yo man!", Charsets.UTF_8));
  serializer.write(EventBuilder.withBody("2nd event!", Charsets.UTF_8));
  serializer.write(EventBuilder.withBody("last one!", Charsets.UTF_8));
  serializer.flush();
  serializer.beforeClose();
  out.flush();
  out.close();
}