org.apache.flume.channel.MemoryChannel Java Examples

@Before
public void setUp() {

    source = new IRCSource();
    channel = new MemoryChannel();

    context.put("host", "localhost");
    context.put("port", "6667");
    context.put("nick", "admin");
    context.put("irc-channels", "memory,jdbc");
    context.put("channels", "memory");
    context.put("password", "1234");
    context.put("replyPing", "false");
    context.put("user", "root");
    context.put("name", "admin");
    context.put("replyPing", "false");


    source.configure(context);
}
 

@Before
public void setUp() throws Exception {
  port = NetworkUtils.getRandomPort();
  source = new ThriftSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<>();
  channels.add(ch);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();
}
 

@Before
public void setup() {
    //        Context channelContext = new Context();
    //        channelContext.put("checkpointDir","data/check");
    //        channelContext.put("dataDirs","data/data");
    //        channelContext.put("capacity","1000");
    //        channelContext.put("transactionCapacity","100");
    //        channelContext.put("checkpointInterval","300");
    //        channel = new FileChannel();
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "5000");
    channel = new MemoryChannel();
    channel.setName("junitChannel");
    Configurables.configure(channel, channelContext);
    channel.start();

    druidSink = new DruidSink();
    druidSink.setChannel(channel);
    druidSink.configure(getMockContext());
    druidSink.start();
}
 

private Sink.Status prepareAndSend(Context context, String msg) throws EventDeliveryException {
    Sink kafkaSink = new KafkaSink();
    Configurables.configure(kafkaSink, context);
    Channel memoryChannel = new MemoryChannel();
    Configurables.configure(memoryChannel, context);
    kafkaSink.setChannel(memoryChannel);
    kafkaSink.start();

    Transaction tx = memoryChannel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody(msg.getBytes());
    memoryChannel.put(event);
    tx.commit();
    tx.close();

    return kafkaSink.process();
}
 

@Before
public void setUp() throws Exception {
  port = NetworkUtils.getRandomPort();
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<>();
  channels.add(ch);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();
}
 

@BeforeMethod
public void setUp() throws Exception {
    if (sink != null) {
        throw new RuntimeException("double setup");
    }
    Context context = new Context();
    context.put("hostname", "127.0.0.1");
    context.put("port", "44445");
    context.put("batch-size", String.valueOf(2));
    context.put("connect-timeout", String.valueOf(2000L));
    context.put("request-timeout", String.valueOf(3000L));
    sink = new AvroSink();
    channel = new MemoryChannel();
    sink.setChannel(channel);
    Configurables.configure(sink, context);
    Configurables.configure(channel, context);

    mockSourceContext = mock(SourceContext.class);
}
 

private void createSparkAvroSink() {
	sparkAvroChannel = new MemoryChannel();
	Map<String, String> channelParamters = new HashMap<>();
	channelParamters.put("capacity", "100000");
	channelParamters.put("transactionCapacity", "1000");
	Context channelContext = new Context(channelParamters);
	Configurables.configure(sparkAvroChannel, channelContext);
	String channelName = "SparkAvroMemoryChannel-" + UUID.randomUUID();
	sparkAvroChannel.setName(channelName);

	sparkAvroSink = new AvroSink();
	sparkAvroSink.setName("SparkAvroSink-" + UUID.randomUUID());
	Map<String, String> paramters = new HashMap<>();
	paramters.put("type", "avro");
	paramters.put("hostname", "localhost");
	paramters.put("port", "41111");
	paramters.put("batch-size", "100");
	Context sinkContext = new Context(paramters);
	sparkAvroSink.configure(sinkContext);
	Configurables.configure(sparkAvroSink, sinkContext);
	sparkAvroSink.setChannel(sparkAvroChannel);

	sparkAvroChannel.start();
	sparkAvroSink.start();
}
 

/**
 * [CygnusSink.start] -------- The sink starts properly.
 */
@Test
public void testStart() {
    System.out.println(getTestTraceHead("[NGSISink.start]") + "-------- The sink starts properly");
    NGSISinkImpl sink = new NGSISinkImpl();
    sink.configure(createContext(null, null, null, null, null, null, null, null, null, null, null));
    sink.setChannel(new MemoryChannel());
    sink.start();
    LifecycleState state = sink.getLifecycleState();
    
    try {
        assertEquals(LifecycleState.START, state);
        System.out.println(getTestTraceHead("[NGSISink.start]")
                + "-  OK  - The sink started properly, the lifecycle state is '" + state.toString() + "'");
    } catch (AssertionError e) {
        System.out.println(getTestTraceHead("[NGSISink.start]")
                + "- FAIL - The sink did not start properly, the lifecycle state is '" + state.toString() + "'");
    } // try catch
}
 

public void setUp(String compressionType, int compressionLevel) {
  if (sink != null) { throw new RuntimeException("double setup");}
  sink = new AvroSink();
  channel = new MemoryChannel();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  if (compressionType.equals("deflate")) {
    context.put("compression-type", compressionType);
    context.put("compression-level", Integer.toString(compressionLevel));
  }

  sink.setChannel(channel);

  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
}
 

@Before
public void setUp() throws Exception {
  sink = new ThriftSink();
  channel = new MemoryChannel();
  hostname = "0.0.0.0";
  port = random.nextInt(50000) + 1024;
  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("request-timeout", String.valueOf(2000L));

  sink.setChannel(channel);

  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
}
 

@Before
public void setUp() throws Exception {
  URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
  Files.copy(Resources.newInputStreamSupplier(schemaUrl),
      new File("/tmp/myrecord.avsc"));

  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
}
 

@Before
public void setUp() {
  source = new SyslogUDPSource(); //SyslogTcpSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  Context context = new Context();
  context.put("port", String.valueOf(TEST_SYSLOG_PORT));
  source.configure(context);
}
 

@Before
public void initiate() throws Exception{
  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  File TESTFILE = new File(
      TestLog4jAppender.class.getClassLoader()
          .getResource("flume-log4jtest.properties").getFile());
  FileReader reader = new FileReader(TESTFILE);
  props = new Properties();
  props.load(reader);
  reader.close();
}
 

@Before
public void setUp() {
  source = new SpoolDirectorySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
}
 

@Before
public void setUp() throws Exception {
  URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
  Files.copy(Resources.newInputStreamSupplier(schemaUrl),
      new File("/tmp/myrecord.avsc"));

  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
}
 

@Test
public void testLifecycle() throws InterruptedException, LifecycleException {
  LOG.debug("Starting...");
  Context context = new Context();

  context.put("hdfs.path", testPath);
  /*
   * context.put("hdfs.rollInterval", String.class);
   * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",
   * String.class);
   */
  Configurables.configure(sink, context);

  sink.setChannel(new MemoryChannel());

  sink.start();
  sink.stop();
}
 

@Test (expected = EventDeliveryException.class)
public void testTimeOut() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(),
    true);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  Assert.fail();
}
 

@Before
public void setUp() {
    source = new SNMPSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);

    source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

private void _do() throws TTransportException, IOException, InterruptedException {
  final Context context = new Context();
  final InetSocketAddress contactPoint = CassandraTestHelper.getCassandraContactPoint();
  context.put("tables", "keyspaceTestCassandraSinkIT.tableTestCassandraSinkIT");
  context.put("hosts", contactPoint.getAddress().getHostAddress());
  context.put("batchSize", "1");
  context.put("consistency", "QUORUM");

  final File cqlFile = File.createTempFile("flumeTest", "cql");
  cqlFile.deleteOnExit();

  IOUtils.write(
      "CREATE KEYSPACE IF NOT EXISTS keyspaceTestCassandraSinkIT WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };\n\n"
          + "CREATE TABLE IF NOT EXISTS keyspaceTestCassandraSinkIT.tableTestCassandraSinkIT ("
          + "id uuid, bool_field boolean, int_field int, PRIMARY KEY (int_field)"
          + ");\n\n",
      new FileOutputStream(cqlFile));

  context.put("cqlFile", cqlFile.getAbsolutePath());
  sink = new CassandraSink();
  sink.configure(context);

  Context channelContext = new Context();
  channelContext.put("capacity", "10000");
  channelContext.put("transactionCapacity", "200");
  channel = new MemoryChannel();
  channel.setName("junitChannel");
  Configurables.configure(channel, channelContext);
  sink.setChannel(channel);

  sink.start();
  sink.stop();
}
 

@Before
public void prepareMongo() throws Exception {
    fongo = new Fongo("mongo test server");

    Context mongoContext = new Context();
    mongoContext.put("batchSize", "1");
    mongoContext.put("mappingFile", "/mapping_definition_1.json");
    mongoContext.put("mongoUri", "INJECTED");

    mongoSink = new MongoSink();

    injectFongo(mongoSink);
    Configurables.configure(mongoSink, mongoContext);

    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");

    channel = new MemoryChannel();
    channel.setName("junitChannel");
    Configurables.configure(channel, channelContext);

    mongoSink.setChannel(channel);

    channel.start();
    mongoSink.start();
}
 

@Test
public void testThreeEvents() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  HBaseSink sink = new HBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 3; i++){
    for(int j = 0; j < 3; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
  testUtility.deleteTable(tableName.getBytes());
}
 

public static void main(String[] args) throws Exception {
    //Flume Source
    ThriftSource source = new ThriftSource();
    Channel ch = new MemoryChannel();
    Configurables.configure(ch, new Context());

    Context context = new Context();
    context.put("port", String.valueOf(port));
    context.put("bind", hostname);
    Configurables.configure(source, context);

    List<Channel> channels = new ArrayList<>();
    channels.add(ch);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
    System.out.println("ThriftSource service start.");

    while (true) {
        Transaction transaction = ch.getTransaction();
        transaction.begin();
        Event event = ch.take();
        if (null != event) {
            System.out.println(event);
            System.out.println(new String(event.getBody()).trim());
        }
        transaction.commit();
        transaction.close();
    }

}
 

@Test(expected = FlumeException.class)
public void testMissingTable() throws Exception {
  deleteTable = false;
  ctx.put("batchSize", "2");
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  sink.process();
  Assert.assertFalse(sink.isConfNull());
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 2);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 2; i++){
    for(int j = 0; j < 2; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(2, found);
  out = results[2];
  Assert.assertArrayEquals(Longs.toByteArray(2), out);
  sink.process();
  sink.stop();
}
 

private Channel initChannel() {
    //Channel configuration
    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");

    Channel channel = new MemoryChannel();
    channel.setName("memorychannel");
    Configurables.configure(channel, channelContext);
    return channel;
}
 

@Test
public void testThreeEvents() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for(int i = 0; i < 3; i++){
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for(int i = 0; i < 3; i++){
    for(int j = 0; j < 3; j++){
      if(Arrays.equals(results[j],Bytes.toBytes(valBase + "-" + i))){
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
}
 

@Test
public void testOneEvent() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
      Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

@Test
public void testOneEventWithDefaults() throws Exception {
  Map<String,String> ctxMap = new HashMap<String,String>();
  ctxMap.put("table", tableName);
  ctxMap.put("columnFamily", columnFamily);
  ctxMap.put("serializer",
          "org.apache.flume.sink.hbase.SimpleAsyncHbaseEventSerializer");
  ctxMap.put("keep-alive", "0");
  ctxMap.put("timeout", "10000");
  Context tmpctx = new Context();
  tmpctx.putAll(ctxMap);

  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, tmpctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, tmpctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
          Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  sink.stop();
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

Channel bindAndStartChannel(ElasticSearchSink fixture) {
  // Configure the channel
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());

  // Wire them together
  fixture.setChannel(channel);
  fixture.start();
  return channel;
}
 

@Before
public void initiate() throws Exception{
  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
  File TESTFILE = new File(
      TestLog4jAppender.class.getClassLoader()
          .getResource("flume-log4jtest.properties").getFile());
  FileReader reader = new FileReader(TESTFILE);
  props = new Properties();
  props.load(reader);
  reader.close();
}
 

@Before
public void initiate() throws InterruptedException{
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
}