org.apache.flume.channel.ChannelProcessor Java Examples

@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 

@Before
public void setUp() throws Exception {
  URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
  Files.copy(Resources.newInputStreamSupplier(schemaUrl),
      new File("/tmp/myrecord.avsc"));

  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
}
 

@Before
public void setUp() throws Exception {
  port = NetworkUtils.getRandomPort();
  source = new ThriftSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<>();
  channels.add(ch);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();
}
 

@Before
public void setUp() throws Exception {
  port = NetworkUtils.getRandomPort();
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<>();
  channels.add(ch);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);
  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    try {
        source = SOURCE_FACTORY.create(
                context.getProperty(SOURCE_NAME).getValue(),
                context.getProperty(SOURCE_TYPE).getValue());

        String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
        String agentName = context.getProperty(AGENT_NAME).getValue();
        String sourceName = context.getProperty(SOURCE_NAME).getValue();
        Configurables.configure(source,
            getFlumeSourceContext(flumeConfig, agentName, sourceName));

        if (source instanceof PollableSource) {
            source.setChannelProcessor(new ChannelProcessor(
                new NifiChannelSelector(pollableSourceChannel)));
            source.start();
        }
    } catch (Throwable th) {
        getLogger().error("Error creating source", th);
        throw Throwables.propagate(th);
    }
}
 

@Override
public void start() {
  PollableSource source = (PollableSource) getSource();
  ChannelProcessor cp = source.getChannelProcessor();
  cp.initialize();
  source.start();

  runner = new PollingRunner();

  runner.source = source;
  runner.counterGroup = counterGroup;
  runner.shouldStop = shouldStop;

  runnerThread = new Thread(runner);
  runnerThread.setName(getClass().getSimpleName() + "-" + 
      source.getClass().getSimpleName() + "-" + source.getName());
  runnerThread.start();

  lifecycleState = LifecycleState.START;
}
 

@Override
public void stop() {

  runner.shouldStop.set(true);

  try {
    runnerThread.interrupt();
    runnerThread.join();
  } catch (InterruptedException e) {
    logger
    .warn(
        "Interrupted while waiting for polling runner to stop. Please report this.",
        e);
    Thread.currentThread().interrupt();
  }

  Source source = getSource();
  source.stop();
  ChannelProcessor cp = source.getChannelProcessor();
  cp.close();

  lifecycleState = LifecycleState.STOP;
}
 

@Before
public void setUp() {
  source = new SyslogUDPSource(); //SyslogTcpSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  Context context = new Context();
  context.put("port", String.valueOf(TEST_SYSLOG_PORT));
  source.configure(context);
}
 

@Before
public void setUp() {
  logger.info("Running setup");

  channel = new MemoryChannel();
  source = new NetcatSource();

  Context context = new Context();

  Configurables.configure(channel, context);
  List<Channel> channels = Lists.newArrayList(channel);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@Before
public void setUp() throws Exception {
  URL schemaUrl = getClass().getClassLoader().getResource("myrecord.avsc");
  Files.copy(Resources.newInputStreamSupplier(schemaUrl),
      new File("/tmp/myrecord.avsc"));

  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
}
 

@OnScheduled
public void onScheduled(final ProcessContext context) {
    try {
        source = SOURCE_FACTORY.create(
                context.getProperty(SOURCE_NAME).getValue(),
                context.getProperty(SOURCE_TYPE).getValue());

        String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
        String agentName = context.getProperty(AGENT_NAME).getValue();
        String sourceName = context.getProperty(SOURCE_NAME).getValue();
        Configurables.configure(source,
            getFlumeSourceContext(flumeConfig, agentName, sourceName));

        if (source instanceof PollableSource) {
            source.setChannelProcessor(new ChannelProcessor(
                new NifiChannelSelector(pollableSourceChannel)));
            source.start();
        }
    } catch (Throwable th) {
        getLogger().error("Error creating source", th);
        throw Throwables.propagate(th);
    }
}
 

@Before
public void setUp() {
  source = new SpoolDirectorySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
    if (source instanceof PollableSource) {
        super.onTrigger(context, sessionFactory);
    } else if (source instanceof EventDrivenSource) {
        ProcessSessionFactory old = sessionFactoryRef.getAndSet(sessionFactory);
        if (old != sessionFactory) {
            if (runnerRef.get() != null) {
                stopped();
                sessionFactoryRef.set(sessionFactory);
            }

            runnerRef.set(new EventDrivenSourceRunner());
            eventDrivenSourceChannelRef.set(new NifiSessionFactoryChannel(sessionFactoryRef.get(), SUCCESS));
            eventDrivenSourceChannelRef.get().start();
            source.setChannelProcessor(new ChannelProcessor(
                new NifiChannelSelector(eventDrivenSourceChannelRef.get())));
            runnerRef.get().setSource(source);
            runnerRef.get().start();
        }
    }
}
 

@SuppressWarnings("unchecked")
@Before
public void setup() throws Exception {
	mockIt = mock(ConsumerIterator.class);
	mockMessageAndMetadata = mock(MessageAndMetadata.class);
	mockChannelProcessor = mock(ChannelProcessor.class);
	mockBuffer = mock(ByteBuffer.class);
	mockMessage = mock(Message.class);
	mockKafkaSource = new KafkaSource();
	
	when(mockMessage.payload()).thenReturn(mockBuffer);
	when(mockMessageAndMetadata.message()).thenReturn(mockMessage);
	
	Field field = AbstractSource.class.getDeclaredField("channelProcessor");
	field.setAccessible(true);
	field.set(mockKafkaSource, mockChannelProcessor);

	field = KafkaSource.class.getDeclaredField("it");
	field.setAccessible(true);
	field.set(mockKafkaSource, mockIt);
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSessionFactory sessionFactory) throws ProcessException {
    if (source instanceof PollableSource) {
        super.onTrigger(context, sessionFactory);
    } else if (source instanceof EventDrivenSource) {
        ProcessSessionFactory old = sessionFactoryRef.getAndSet(sessionFactory);
        if (old != sessionFactory) {
            if (runnerRef.get() != null) {
                stopped();
                sessionFactoryRef.set(sessionFactory);
            }

            runnerRef.set(new EventDrivenSourceRunner());
            eventDrivenSourceChannelRef.set(new NifiSessionFactoryChannel(sessionFactoryRef.get(), SUCCESS));
            eventDrivenSourceChannelRef.get().start();
            source.setChannelProcessor(new ChannelProcessor(
                new NifiChannelSelector(eventDrivenSourceChannelRef.get())));
            runnerRef.get().setSource(source);
            runnerRef.get().start();
        }
    }
}
 

@Before
public void initiate() throws Exception{
  int port = 25430;
  source = new AvroSource();
  ch = new MemoryChannel();
  Configurables.configure(ch, new Context());

  Context context = new Context();
  context.put("port", String.valueOf(port));
  context.put("bind", "localhost");
  Configurables.configure(source, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(ch);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();
  File TESTFILE = new File(
      TestLog4jAppender.class.getClassLoader()
          .getResource("flume-log4jtest.properties").getFile());
  FileReader reader = new FileReader(TESTFILE);
  props = new Properties();
  props.load(reader);
  reader.close();
}
 

public ExecRunnable(String command, String eventTerminator, String lineTerminator, ChannelProcessor channelProcessor, CounterGroup counterGroup, boolean restart, long restartThrottle, boolean logStderr, int bufferCount, Charset charset) {
	this.command = command;
          this.eventTerminator = eventTerminator;
	this.lineTerminator = lineTerminator;
	this.channelProcessor = channelProcessor;
	this.counterGroup = counterGroup;
	this.restartThrottle = restartThrottle;
	this.bufferCount = bufferCount;
	this.restart = restart;
	this.logStderr = logStderr;
	this.charset = charset;
}
 

@Before
public void setUp() {
    source = new RedisSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    List<Channel> channels = new ArrayList<Channel>();
    channels.add(channel);

    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);

    source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

public static void main(String[] args) throws Exception {
    //Flume Source
    ThriftSource source = new ThriftSource();
    Channel ch = new MemoryChannel();
    Configurables.configure(ch, new Context());

    Context context = new Context();
    context.put("port", String.valueOf(port));
    context.put("bind", hostname);
    Configurables.configure(source, context);

    List<Channel> channels = new ArrayList<>();
    channels.add(ch);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));
    source.start();
    System.out.println("ThriftSource service start.");

    while (true) {
        Transaction transaction = ch.getTransaction();
        transaction.begin();
        Event event = ch.take();
        if (null != event) {
            System.out.println(event);
            System.out.println(new String(event.getBody()).trim());
        }
        transaction.commit();
        transaction.close();
    }

}
 

@Override
public void start()
{
  super.start();
  emitTimer = new Timer();

  final ChannelProcessor channel = getChannelProcessor();
  final int cacheSize = cache.size();
  emitTimer.scheduleAtFixedRate(new TimerTask()
  {
    @Override
    public void run()
    {
      int lastIndex = startIndex + rate;
      if (lastIndex > cacheSize) {
        lastIndex -= cacheSize;
        processBatch(channel, cache.subList(startIndex, cacheSize));
        startIndex = 0;
        while (lastIndex > cacheSize) {
          processBatch(channel, cache);
          lastIndex -= cacheSize;
        }
        processBatch(channel, cache.subList(0, lastIndex));
      } else {
        processBatch(channel, cache.subList(startIndex, lastIndex));
      }
      startIndex = lastIndex;
    }

  }, 0, 1000);
}
 

@Before
public void setUp() throws Exception {
    eventSource = new AvroSource();
    channel = new MemoryChannel();

    Configurables.configure(channel, new Context());

    avroLogger = (Logger) LogManager.getLogger("avrologger");
    /*
     * Clear out all other appenders associated with this logger to ensure
     * we're only hitting the Avro appender.
     */
    removeAppenders(avroLogger);
    final Context context = new Context();
    testPort = String.valueOf(AvailablePortFinder.getNextAvailable());
    context.put("port", testPort);
    context.put("bind", "0.0.0.0");
    Configurables.configure(eventSource, context);

    final List<Channel> channels = new ArrayList<>();
    channels.add(channel);

    final ChannelSelector cs = new ReplicatingChannelSelector();
    cs.setChannels(channels);

    eventSource.setChannelProcessor(new ChannelProcessor(cs));

    eventSource.start();

    Assert.assertTrue("Reached start or error", LifecycleController
            .waitForOneOf(eventSource, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Server is started", LifecycleState.START,
            eventSource.getLifecycleState());
}
 

public CountingSourceRunner(PollableSource source, int until, Channel channel) {
  this.source = source;
  this.until = until;
  if(channel != null) {
    ReplicatingChannelSelector selector = new ReplicatingChannelSelector();
    List<Channel> channels = Lists.newArrayList();
    channels.add(channel);
    selector.setChannels(channels);
    this.source.setChannelProcessor(new ChannelProcessor(selector));
  }
}
 

@SuppressWarnings("unchecked")
@Override
void afterSetup() throws Exception {
  baseDir = Files.createTempDir();
  passwordFile = new File(baseDir, "password");
  Assert.assertTrue(passwordFile.createNewFile());
  initialConext = mock(InitialContext.class);
  channelProcessor = mock(ChannelProcessor.class);
  events = Lists.newArrayList();
  doAnswer(new Answer<Void>() {
    @Override
    public Void answer(InvocationOnMock invocation) throws Throwable {
      events.addAll((List<Event>)invocation.getArguments()[0]);
      return null;
    }
  }).when(channelProcessor).processEventBatch(any(List.class));
  consumerFactory = mock(JMSMessageConsumerFactory.class);
  consumer = spy(create());
  when(consumerFactory.create(any(ConnectionFactory.class), anyString(),
      any(JMSDestinationType.class), anyString(), anyInt(), anyLong(),
      any(JMSMessageConverter.class), any(Optional.class),
      any(Optional.class))).thenReturn(consumer);
  when(initialConext.lookup(anyString())).thenReturn(connectionFactory);
  contextFactory = mock(InitialContextFactory.class);
  when(contextFactory.create(any(Properties.class))).thenReturn(initialConext);
  source = new JMSSource(consumerFactory, contextFactory);
  source.setName("JMSSource-" + UUID.randomUUID());
  source.setChannelProcessor(channelProcessor);
  context = new Context();
  context.put(JMSSourceConfiguration.BATCH_SIZE, String.valueOf(batchSize));
  context.put(JMSSourceConfiguration.DESTINATION_NAME, "INBOUND");
  context.put(JMSSourceConfiguration.DESTINATION_TYPE,
      JMSSourceConfiguration.DESTINATION_TYPE_QUEUE);
  context.put(JMSSourceConfiguration.PROVIDER_URL, "dummy:1414");
  context.put(JMSSourceConfiguration.INITIAL_CONTEXT_FACTORY, "ldap://dummy:389");
}
 

@Test
public void testLifecycle() throws InterruptedException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();
    Event event = channel.take();

    Assert.assertArrayEquals(String.valueOf(i).getBytes(),
        new String(event.getBody()).getBytes());
  }
  source.stop();
}
 

@Test
public void testBatchProcessWithLifeCycle() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  int batchSize = 10;

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");
  context.put("batchSize", Integer.toString(batchSize));

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();

    for (long j = batchSize; j > 0; j--) {
      Event event = channel.take();
      String expectedVal = String.valueOf(((i+1)*batchSize)-j);
      String resultedVal = new String(event.getBody());
      Assert.assertTrue("Expected " + expectedVal + " is not equals to " +
          resultedVal, expectedVal.equals(resultedVal));
    }
  }

  source.stop();
}
 

@Test
public void testProcess() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  for (long i = 0; i < 100; i++) {
    source.process();
    Event event = channel.take();

    Assert.assertArrayEquals(String.valueOf(i).getBytes(),
        new String(event.getBody()).getBytes());
  }
}
 

@Before
public void setUp() {
  sources = new ArrayList<>(NUM_HOSTS);
  chs = new ArrayList<>(NUM_HOSTS);

  for(int i = 0; i < NUM_HOSTS; i++) {
    AvroSource source = new AvroSource();
    Channel channel = new MemoryChannel();
    Configurables.configure(channel, new Context());

    Context context = new Context();
    context.put("port", String.valueOf(ports.get(i)));
    context.put("bind", "localhost");
    Configurables.configure(source, context);

    List<Channel> channels = new ArrayList<>();
    channels.add(channel);
    ChannelSelector rcs = new ReplicatingChannelSelector();
    rcs.setChannels(channels);
    source.setChannelProcessor(new ChannelProcessor(rcs));

    sources.add(source);
    chs.add(channel);

    source.start();
  }

}
 

@Before
public void setUp() {
  context.put("keep-alive", "1");
  context.put("capacity", "1000");
  context.put("transactionCapacity", "1000");
  Configurables.configure(channel, context);
  rcs.setChannels(Lists.newArrayList(channel));

  source = new ExecSource();
  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

@BeforeClass
public static void setUpClass() throws Exception {
  selectedPort = findFreePort();

  source = new HTTPSource();
  channel = new MemoryChannel();

  Context ctx = new Context();
  ctx.put("capacity", "100");
  Configurables.configure(channel, ctx);

  List<Channel> channels = new ArrayList<Channel>(1);
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  channel.start();
  Context context = new Context();

  context.put("port", String.valueOf(selectedPort));
  context.put("host", "0.0.0.0");

  Configurables.configure(source, context);
  source.start();
}
 

private void configureSource() {
  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}