org.apache.flume.instrumentation.SourceCounter Java Examples

The following examples show how to use org.apache.flume.instrumentation.SourceCounter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
@Override
public void configure(Context context) {
    // Only the queue name does not have a default value
    Configurables.ensureRequiredNonNull(context, QUEUE_KEY);

    // Assign all of the configured values
    hostname = context.getString(HOST_KEY, ConnectionFactory.DEFAULT_HOST);
    port = context.getInteger(PORT_KEY, ConnectionFactory.DEFAULT_AMQP_PORT);
    enableSSL = context.getBoolean(SSL_KEY, false);
    virtualHost = context.getString(VHOST_KEY, ConnectionFactory.DEFAULT_VHOST);
    username = context.getString(USER_KEY, ConnectionFactory.DEFAULT_USER);
    password = context.getString(PASSWORD_KEY, ConnectionFactory.DEFAULT_PASS);
    queue = context.getString(QUEUE_KEY, null);
    exchange = context.getString(EXCHANGE_KEY, null);
    autoAck = context.getBoolean(AUTOACK_KEY, false);
    requeuing = context.getBoolean(REQUEUING, false);
    prefetchCount = context.getInteger(PREFETCH_COUNT_KEY, 0);
    timeout = context.getInteger(TIMEOUT_KEY, -1);
    consumerThreads = context.getInteger(THREAD_COUNT_KEY, 1);

    // Ensure that Flume can connect to RabbitMQ
    testRabbitMQConnection();

    // Create and configure the counters
    sourceCounter = new SourceCounter(getName());
    counterGroup = new CounterGroup();
    counterGroup.setName(getName());
}
 
Example #2
@Override
public void configure(Context context) {
    // Only the queue name does not have a default value
    Configurables.ensureRequiredNonNull(context, QUEUE_KEY);

    // Assign all of the configured values
    hostname = context.getString(HOST_KEY, ConnectionFactory.DEFAULT_HOST);
    port = context.getInteger(PORT_KEY, ConnectionFactory.DEFAULT_AMQP_PORT);
    enableSSL = context.getBoolean(SSL_KEY, false);
    virtualHost = context.getString(VHOST_KEY, ConnectionFactory.DEFAULT_VHOST);
    username = context.getString(USER_KEY, ConnectionFactory.DEFAULT_USER);
    password = context.getString(PASSWORD_KEY, ConnectionFactory.DEFAULT_PASS);
    queue = context.getString(QUEUE_KEY, null);
    exchange = context.getString(EXCHANGE_KEY, null);
    autoAck = context.getBoolean(AUTOACK_KEY, false);
    requeuing = context.getBoolean(REQUEUING, false);
    prefetchCount = context.getInteger(PREFETCH_COUNT_KEY, 0);
    timeout = context.getInteger(TIMEOUT_KEY, -1);
    consumerThreads = context.getInteger(THREAD_COUNT_KEY, 1);

    // Ensure that Flume can connect to RabbitMQ
    testRabbitMQConnection();

    // Create and configure the counters
    sourceCounter = new SourceCounter(getName());
    counterGroup = new CounterGroup();
    counterGroup.setName(getName());
}
 
Example #3
Source Project: mt-flume   Author: javachen   File: ExecSource.java    License: Apache License 2.0 5 votes vote down vote up
public ExecRunnable(String shell, String command, ChannelProcessor channelProcessor,
    SourceCounter sourceCounter, boolean restart, long restartThrottle,
    boolean logStderr, int bufferCount, long batchTimeout, Charset charset) {
  this.command = command;
  this.channelProcessor = channelProcessor;
  this.sourceCounter = sourceCounter;
  this.restartThrottle = restartThrottle;
  this.bufferCount = bufferCount;
  this.batchTimeout = batchTimeout;
  this.restart = restart;
  this.logStderr = logStderr;
  this.charset = charset;
  this.shell = shell;
}
 
Example #4
Source Project: mt-flume   Author: javachen   File: MultiportSyslogTCPSource.java    License: Apache License 2.0 5 votes vote down vote up
public MultiportSyslogHandler(int maxEventSize, int batchSize,
    ChannelProcessor cp, SourceCounter ctr, String portHeader,
    ThreadSafeDecoder defaultDecoder,
    ConcurrentMap<Integer, ThreadSafeDecoder> portCharsets) {
  channelProcessor = cp;
  sourceCounter = ctr;
  this.maxEventSize = maxEventSize;
  this.batchSize = batchSize;
  this.portHeader = portHeader;
  this.defaultDecoder = defaultDecoder;
  this.portCharsets = portCharsets;
  syslogParser = new SyslogParser();
  lineSplitter = new LineSplitter(maxEventSize);
}
 
Example #5
Source Project: ns4_gear_watchdog   Author: newsettle   File: TaildirSource.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
protected SourceCounter getSourceCounter() {
    return sourceCounter;
}
 
Example #6
Source Project: flume-customized   Author: yanghua   File: SpoolFileSource.java    License: Apache License 2.0 4 votes vote down vote up
public SpoolFileRunnable(ReliableSpoolFileEventReader reader, SourceCounter sourceCounter) {
    this.reader = reader;
    this.sourceCounter = sourceCounter;
}
 
Example #7
Source Project: pulsar   Author: apache   File: SourceOfFlume.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void doStart() {
    log.info("start source of flume ...");
    this.counter = new SourceCounter("flume-source");
    this.counter.start();
}
 
Example #8
Source Project: flume-ng-extends-source   Author: ningg   File: SpoolDirectoryTailFileSource.java    License: MIT License 4 votes vote down vote up
public SpoolDirectoryTailFileRunnable(ReliableSpoolDirectoryTailFileEventReader reader, SourceCounter sourceCounter) {
	this.reader = reader;
	this.sourceCounter = sourceCounter;
}
 
Example #9
Source Project: rabbitmq-flume-plugin   Author: gmr   File: Consumer.java    License: BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
public Consumer setSourceCounter(SourceCounter sourceCounter) {
    this.sourceCounter = sourceCounter;
    return this;
}
 
Example #10
Source Project: rabbitmq-flume-plugin   Author: gmr   File: Consumer.java    License: BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
public Consumer setSourceCounter(SourceCounter sourceCounter) {
    this.sourceCounter = sourceCounter;
    return this;
}
 
Example #11
Source Project: mt-flume   Author: javachen   File: SpoolDirectorySource.java    License: Apache License 2.0 4 votes vote down vote up
public SpoolDirectoryRunnable(ReliableSpoolingFileEventReader reader,
    SourceCounter sourceCounter) {
  this.reader = reader;
  this.sourceCounter = sourceCounter;
}
 
Example #12
Source Project: mt-flume   Author: javachen   File: TestMultiportSyslogTCPSource.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test that different charsets are parsed by different ports correctly.
 */
@Test
public void testPortCharsetHandling() throws UnknownHostException, Exception {

  ///////////////////////////////////////////////////////
  // port setup

  InetAddress localAddr = InetAddress.getLocalHost();
  DefaultIoSessionDataStructureFactory dsFactory =
      new DefaultIoSessionDataStructureFactory();


  // one faker on port 10001
  int port1 = 10001;
  NioSession session1 = mock(NioSession.class);
  session1.setAttributeMap(dsFactory.getAttributeMap(session1));
  SocketAddress sockAddr1 = new InetSocketAddress(localAddr, port1);
  when(session1.getLocalAddress()).thenReturn(sockAddr1);

  // another faker on port 10002
  int port2 = 10002;
  NioSession session2 = mock(NioSession.class);
  session2.setAttributeMap(dsFactory.getAttributeMap(session2));
  SocketAddress sockAddr2 = new InetSocketAddress(localAddr, port2);
  when(session2.getLocalAddress()).thenReturn(sockAddr2);

  // set up expected charsets per port
  ConcurrentMap<Integer, ThreadSafeDecoder> portCharsets =
      new ConcurrentHashMap<Integer, ThreadSafeDecoder>();
  portCharsets.put(port1, new ThreadSafeDecoder(Charsets.ISO_8859_1));
  portCharsets.put(port2, new ThreadSafeDecoder(Charsets.UTF_8));

  ///////////////////////////////////////////////////////
  // channel / source setup

  // set up channel to receive events
  MemoryChannel chan = new MemoryChannel();
  chan.configure(new Context());
  chan.start();
  ReplicatingChannelSelector sel = new ReplicatingChannelSelector();
  sel.setChannels(Lists.<Channel>newArrayList(chan));
  ChannelProcessor chanProc = new ChannelProcessor(sel);

  // defaults to UTF-8
  MultiportSyslogHandler handler = new MultiportSyslogHandler(
      1000, 10, chanProc, new SourceCounter("test"), "port",
      new ThreadSafeDecoder(Charsets.UTF_8), portCharsets);

  // initialize buffers
  handler.sessionCreated(session1);
  handler.sessionCreated(session2);

  ///////////////////////////////////////////////////////
  // event setup

  // Create events of varying charsets.
  String header = "<10>2012-08-17T02:14:00-07:00 192.168.1.110 ";

  // These chars encode under ISO-8859-1 as illegal bytes under UTF-8.
  String dangerousChars = "þÿÀÁ";

  ///////////////////////////////////////////////////////
  // encode and send them through the message handler
  String msg;
  IoBuffer buf;
  Event evt;

  // valid ISO-8859-1 on the right (ISO-8859-1) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.ISO_8859_1));
  handler.messageReceived(session1, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertNull(evt.getHeaders().get(SyslogUtils.EVENT_STATUS));

  // valid ISO-8859-1 on the wrong (UTF-8) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.ISO_8859_1));
  handler.messageReceived(session2, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertEquals("Expected invalid event due to character encoding",
      SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(),
      evt.getHeaders().get(SyslogUtils.EVENT_STATUS));

  // valid UTF-8 on the right (UTF-8) port
  msg = header + dangerousChars + "\n";
  buf = IoBuffer.wrap(msg.getBytes(Charsets.UTF_8));
  handler.messageReceived(session2, buf);
  evt = takeEvent(chan);
  Assert.assertNotNull("Event vanished!", evt);
  Assert.assertNull(evt.getHeaders().get(SyslogUtils.EVENT_STATUS));
}
 
Example #13
Source Project: ingestion   Author: Stratio   File: SNMPSource.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
protected SourceCounter getSourceCounter() {
    return sourceCounter;
}
 
Example #14
Source Project: ingestion   Author: Stratio   File: IRCSource.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
protected SourceCounter getSourceCounter() {
    return sourceCounter;
}