org.apache.flume.Channel Java Examples

/**
 * Given a list of channel names as space delimited string,
 * returns list of channels.
 * @return List of {@linkplain Channel}s represented by the names.
 */
protected List<Channel> getChannelListFromNames(String channels,
        Map<String, Channel> channelNameMap) {
  List<Channel> configuredChannels = new ArrayList<Channel>();
  if(channels == null || channels.isEmpty()) {
    return configuredChannels;
  }
  String[] chNames = channels.split(" ");
  for (String name : chNames) {
    Channel ch = channelNameMap.get(name);
    if (ch != null) {
      configuredChannels.add(ch);
    } else {
      throw new FlumeException("Selector channel not found: "
              + name);
    }
  }
  return configuredChannels;
}
 

@Test
public void testReusableChannel() throws Exception {
  String agentName = "agent1";
  Map<String, String> properties = getPropertiesForChannel(agentName,
      RecyclableChannel.class.getName());
  MemoryConfigurationProvider provider =
      new MemoryConfigurationProvider(agentName, properties);

  MaterializedConfiguration config1 = provider.getConfiguration();
  Channel channel1 = config1.getChannels().values().iterator().next();
  Assert.assertTrue(channel1 instanceof RecyclableChannel);

  MaterializedConfiguration config2 = provider.getConfiguration();
  Channel channel2 = config2.getChannels().values().iterator().next();
  Assert.assertTrue(channel2 instanceof RecyclableChannel);

  Assert.assertSame(channel1, channel2);
}
 

@Before
public void setUp() {
  source = new SpoolDirectorySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
}
 

@Ignore @Test
public void shouldIndexOneEvent() throws Exception {
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  Transaction tx = channel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody("event #1 or 1".getBytes());
  channel.put(event);
  tx.commit();
  tx.close();

  fixture.process();
  fixture.stop();
  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

  assertMatchAllQuery(1, event);
  assertBodyQuery(1, event);
}
 

@Override
public List<Channel> getRequiredChannels(Event event) {
  String headerValue = event.getHeaders().get(headerName);
  if (headerValue == null || headerValue.trim().length() == 0) {
    return defaultChannels;
  }

  List<Channel> channels = channelMapping.get(headerValue);

  //This header value does not point to anything
  //Return default channel(s) here.
  if (channels == null) {
    channels = defaultChannels;
  }

  return channels;
}
 

/**
 * Lack of exception test.
 */
@Test
public void testAppend() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();
  Configurables.configure(channel, context);
  Configurables.configure(sink, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = EventBuilder.withBody(("Test " + i).getBytes());

    channel.put(event);
    sink.process();
  }

  sink.stop();
}
 

@Test
public void testOptionalChannels() throws Exception {
  Context context = new Context();
  context.put(ReplicatingChannelSelector.CONFIG_OPTIONAL, "ch1");
  Configurables.configure(selector, context);
  List<Channel> channels = selector.getRequiredChannels(new MockEvent());
  Assert.assertNotNull(channels);
  Assert.assertEquals(3, channels.size());
  Assert.assertEquals("ch2", channels.get(0).getName());
  Assert.assertEquals("ch3", channels.get(1).getName());
  Assert.assertEquals("ch4", channels.get(2).getName());

  List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
  Assert.assertEquals(1, optCh.size());
  Assert.assertEquals("ch1", optCh.get(0).getName());

}
 

/**
 * <p>
 * A convenience method for multiple-event <code>take</code> transactions.
 * </p>
 * @return a list of at most <code>max</code> events
 * @see #transact(Channel,Callable)
 */
public static List<Event> take(final Channel channel, final int max)
    throws ChannelException {
  return transact(channel, new Callable<List<Event>>() {
      @Override
      public List<Event> call() {
        List<Event> events = new ArrayList<Event>(max);
        while (events.size() < max) {
          Event event = channel.take();
          if (event == null) {
            break;
          }
          events.add(event);
        }
        return events;
      }
    });
}
 

@Test
public void testReusableChannel() throws Exception {
    String agentName = "agent1";
    Map<String, String> properties = getPropertiesForChannel(agentName,
            RecyclableChannel.class.getName());
    MemoryConfigurationProvider provider =
            new MemoryConfigurationProvider(agentName, properties);

    MaterializedConfiguration config1 = provider.getConfiguration();
    Channel channel1 = config1.getChannels().values().iterator().next();
    assertTrue(channel1 instanceof RecyclableChannel);

    MaterializedConfiguration config2 = provider.getConfiguration();
    Channel channel2 = config2.getChannels().values().iterator().next();
    assertTrue(channel2 instanceof RecyclableChannel);

    assertSame(channel1, channel2);
}
 

@Test
public void testUnspecifiedChannel() throws Exception {
    String agentName = "agent1";
    Map<String, String> properties = getPropertiesForChannel(agentName,
            UnspecifiedChannel.class.getName());
    MemoryConfigurationProvider provider =
            new MemoryConfigurationProvider(agentName, properties);

    MaterializedConfiguration config1 = provider.getConfiguration();
    Channel channel1 = config1.getChannels().values().iterator().next();
    assertTrue(channel1 instanceof UnspecifiedChannel);

    MaterializedConfiguration config2 = provider.getConfiguration();
    Channel channel2 = config2.getChannels().values().iterator().next();
    assertTrue(channel2 instanceof UnspecifiedChannel);

    assertSame(channel1, channel2);
}
 

@Test
public void testMultipleOptionalChannelsSameChannelTwice() throws Exception {
  Context context = new Context();
  context.put(ReplicatingChannelSelector.CONFIG_OPTIONAL, "ch1 ch4 ch1");
  Configurables.configure(selector, context);
  List<Channel> channels = selector.getRequiredChannels(new MockEvent());
  Assert.assertNotNull(channels);
  Assert.assertEquals(2, channels.size());
  Assert.assertEquals("ch2", channels.get(0).getName());
  Assert.assertEquals("ch3", channels.get(1).getName());

  List<Channel> optCh = selector.getOptionalChannels(new MockEvent());
  Assert.assertEquals(2, optCh.size());
  Assert.assertEquals("ch1", optCh.get(0).getName());
  Assert.assertEquals("ch4", optCh.get(1).getName());
}
 

@Test
public void stop() {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Session session = mock(Session.class);
  final Cluster cluster = mock(Cluster.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.setChannel(channel);
  sink.session = session;
  sink.cluster = cluster;
  sink.stop();
  verify(session).isClosed();
  verify(session).close();
  verifyNoMoreInteractions(session);
  verify(cluster).isClosed();
  verify(cluster).close();
  verifyNoMoreInteractions(cluster);
}
 

@Before
public void setUp() throws Exception {
  port = NetworkUtils.getRandomPort();
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<>();
  channels.add(ch);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();
}
 

public static Set<String> consumeChannel(Channel channel,
  boolean checkForCorruption) throws Exception {
  Set<String> result = Sets.newHashSet();
  int[] batchSizes = new int[] {
      1000, 100, 10, 1
  };
  for (int i = 0; i < batchSizes.length; i++) {
    while(true) {
      Set<String> batch = takeEvents(channel, batchSizes[i], checkForCorruption);
      if(batch.isEmpty()) {
        break;
      }
      result.addAll(batch);
    }
  }
  return result;
}
 

@Override
public Status process() throws EventDeliveryException {
    Channel channel = getChannel();
    Transaction tx = channel.getTransaction();
    try {
        tx.begin();
        Event event = channel.take();
        if (event == null) {
            tx.commit();
            return Status.READY;

        }

        String data = null;
        if(writeBody){
            data = new String(event.getBody());
        } else {
            data = mapper.writeValueAsString(event.getHeaders());
        }

        producer.send(new KeyedMessage<String, String>(topic, data));
        tx.commit();
        return Status.READY;
    } catch (Exception e) {
        try {
            tx.rollback();
            return Status.BACKOFF;
        } catch (Exception e2) {
            log.error("Rollback Exception:{}", e2);
        }
        log.error("KafkaSink Exception:{}", e);
        return Status.BACKOFF;
    } finally {
        tx.close();
    }
}
 

@Test
public void testThreeEvents() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 3; i++){
    for(int j = 0; j < 3; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
  testUtility.deleteTable(tableName.getBytes());
}
 

@Test
public void testOneEventWithDefaults() throws Exception {
  Map<String,String> ctxMap = new HashMap<String,String>();
  ctxMap.put("table", tableName);
  ctxMap.put("columnFamily", columnFamily);
  ctxMap.put("serializer",
          "org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer");
  ctxMap.put("keep-alive", "0");
  ctxMap.put("timeout", "10000");
  Context tmpctx = new Context();
  tmpctx.putAll(ctxMap);

  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, tmpctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, tmpctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
          Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

private Channel initChannel() {
    //Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");

    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}
 

Channel bindAndStartChannel(ElasticSearchSink fixture) {
  // Configure the channel
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());

  // Wire them together
  fixture.setChannel(channel);
  fixture.start();
  return channel;
}
 

public static Set<String> takeWithoutCommit(Channel channel, Transaction tx,
        int number) {
  Set<String> events = Sets.newHashSet();
  tx.begin();
  for (int i = 0; i < number; i++) {
    Event e = channel.take();
    if (e == null) {
      break;
    }
    events.add(new String(e.getBody()));
  }
  return events;
}
 

@Override
public void configure(Context context) {
  String optionalList = context.getString(CONFIG_OPTIONAL);
  requiredChannels = new ArrayList<Channel>(getAllChannels());
  Map<String, Channel> channelNameMap = getChannelNameMap();
  if(optionalList != null && !optionalList.isEmpty()) {
    for(String optional : optionalList.split("\\s+")) {
      Channel optionalChannel = channelNameMap.get(optional);
      requiredChannels.remove(optionalChannel);
      if (!optionalChannels.contains(optionalChannel)) {
        optionalChannels.add(optionalChannel);
      }
    }
  }
}
 

/**
 * Stops the channels.
 */
private void stopChannels() {
    for (String channelName : channelsRef.keySet()) {
        Channel channel = channelsRef.get(channelName);
        LifecycleState state = channel.getLifecycleState();
        System.out.println("Stopping " + channelName + " (lyfecycle state=" + state.toString() + ")");
        supervisorRef.unsupervise(channel);
    } // for
}
 

@Before
public void setUp() {
  source = new AvroSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@Ignore @Test
public void processIllegalArgumentException() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  doThrow(IllegalArgumentException.class).when(table).save(anyListOf(Event.class));
  boolean hasThrown = false;
  try {
    sink.process();
  } catch (EventDeliveryException ex) {
    hasThrown = true;
    if (!(ex.getCause() instanceof IllegalArgumentException)) {
      fail("Did not throw inner IllegalArgumentException: " + ex);
    }
  }
  verify(tx).begin();
  verify(tx).rollback();
  verify(tx).close();
  verifyNoMoreInteractions(tx);
  if (!hasThrown) {
    fail("Did not throw exception");
  }
}
 

@Test
public void testCreateTable () throws Exception {
    
    String ddl = "CREATE TABLE flume_test " +
            "  (flume_time timestamp not null, col1 varchar , col2 varchar" +
            "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";

    final String fullTableName = "FLUME_TEST";
    sinkContext = new Context ();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER,EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION,"^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES,"col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR,DefaultKeyGenerator.TIMESTAMP.name());

    
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    Assert.assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
    try {
        boolean exists = admin.tableExists(fullTableName);
        Assert.assertTrue(exists);
    }finally {
        admin.close();
    }
}
 

@Test
public void testSinkLifecycle () {
    
    String ddl = "CREATE TABLE flume_test " +
            "  (flume_time timestamp not null, col1 varchar , col2 varchar" +
            "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";
    
    sinkContext = new Context ();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, "flume_test");
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, getUrl());
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER,EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION,"^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES,"col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR,DefaultKeyGenerator.TIMESTAMP.name());

    
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    Assert.assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    Assert.assertEquals(LifecycleState.START, sink.getLifecycleState());
    sink.stop();
    Assert.assertEquals(LifecycleState.STOP, sink.getLifecycleState());
}
 

@Test
public void testCreateTable () throws Exception {
    
    String ddl = "CREATE TABLE flume_test " +
            "  (flume_time timestamp not null, col1 varchar , col2 varchar" +
            "  CONSTRAINT pk PRIMARY KEY (flume_time))\n";

    final String fullTableName = "FLUME_TEST";
    sinkContext = new Context ();
    sinkContext.put(FlumeConstants.CONFIG_TABLE, fullTableName);
    sinkContext.put(FlumeConstants.CONFIG_JDBC_URL, TestUtil.PHOENIX_JDBC_URL);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER,EventSerializers.REGEX.name());
    sinkContext.put(FlumeConstants.CONFIG_TABLE_DDL, ddl);
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_REGULAR_EXPRESSION,"^([^\t]+)\t([^\t]+)$");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_COLUMN_NAMES,"col1,col2");
    sinkContext.put(FlumeConstants.CONFIG_SERIALIZER_PREFIX + FlumeConstants.CONFIG_ROWKEY_TYPE_GENERATOR,DefaultKeyGenerator.TIMESTAMP.name());

    
    sink = new PhoenixSink();
    Configurables.configure(sink, sinkContext);
    Assert.assertEquals(LifecycleState.IDLE, sink.getLifecycleState());
  
    final Channel channel = this.initChannel();
    sink.setChannel(channel);
    
    sink.start();
    HBaseAdmin admin = driver.getConnectionQueryServices(TestUtil.PHOENIX_JDBC_URL, TestUtil.TEST_PROPERTIES).getAdmin();
    try {
        boolean exists = admin.tableExists(fullTableName);
        Assert.assertTrue(exists);
    }finally {
        admin.close();
    }
}
 

@Test
public void testEmptyChannelResultsInStatusBackoff()
    throws InterruptedException, LifecycleException, EventDeliveryException {
  LOG.debug("Starting...");
  Context context = new Context();
  Channel channel = new MemoryChannel();
  context.put("hdfs.path", testPath);
  context.put("keep-alive", "0");
  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
  sink.setChannel(channel);
  sink.start();
  Assert.assertEquals(Status.BACKOFF, sink.process());
  sink.stop();
}
 

private List<Event> takeEventsFromChannel(Channel channel) {
    List<Event> events = new ArrayList<Event>();
    for (int i = 0; i < this.batchsize; i++) {
        this.sinkCounter.incrementEventDrainAttemptCount();
        events.add(channel.take());
    }
    events.removeAll(Collections.singleton(null));
    return events;
}
 

/**
 * Sets up tests by creating a unique instance of the tested class, and by defining the behaviour of the mocked
 * classes.
 *  
 * @throws Exception
 */
@Before
public void setUp() throws Exception {
    // set up the instance of the tested class
    channelSelector = new RoundRobinChannelSelector();
    Channel channel1 = new MemoryChannel();
    channel1.setName("ch1");
    Channel channel2 = new MemoryChannel();
    channel2.setName("ch2");
    Channel channel3 = new MemoryChannel();
    channel3.setName("ch3");
    Channel channel4 = new MemoryChannel();
    channel4.setName("ch4");
    Channel channel5 = new MemoryChannel();
    channel5.setName("ch5");
    Channel channel6 = new MemoryChannel();
    channel6.setName("ch6");
    ArrayList<Channel> allChannels = new ArrayList<Channel>();
    allChannels.add(channel1);
    allChannels.add(channel2);
    allChannels.add(channel3);
    allChannels.add(channel4);
    allChannels.add(channel5);
    allChannels.add(channel6);
    channelSelector.setChannels(allChannels);
    
    // set up other instances
    context = new Context();
    context.put("storages", "3");
    context.put("storages.storage1", "ch1");
    context.put("storages.storage2", "ch2,ch3");
    context.put("storages.storage3", "ch4,ch5,ch6");
}