Java Code Examples for org.apache.flume.Event#getHeaders()

/** {@inheritDoc} */
@Override
public Event intercept(Event event) {
  List<T> objects = getObjectsToCount(event);
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("Identified %d objects to count from event: %s", objects.size(),
      objects));
  }

  if (!objects.isEmpty()) {
    Map<String, String> headers = event.getHeaders();
    for (T obj : objects) {
      long total = counters.incrementCount(obj);
      headers.put(String.valueOf(obj), String.valueOf(total));
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format("Added counter to event header: %s=%d", obj.toString(), total));
      }
    }
  }
  return event;
}
 

private Map<String, Object> parseEvent(Event event) {
    final Map<String, String> headers = event.getHeaders();
    Map<String, Object> parsedEvent = null;
    if (MapUtils.isNotEmpty(headers)) {
        parsedEvent = new HashMap<String, Object>();
        for (String header : headers.keySet()) {
            if (timestampField.equalsIgnoreCase(header)) {
                parsedEvent.put(header, Long.valueOf(headers.get(header)));
            } else {
                parsedEvent.put(header, headers.get(header));
            }

        }
    }
    return parsedEvent;
}
 

/**
 * Test bad event format 2: The first char is not <
 */

@Test
public void testExtractBadEvent2() {
  String badData1 = "hi guys! <10> bad bad data\n";
  SyslogUtils util = new SyslogUtils(false);
  ChannelBuffer buff = ChannelBuffers.buffer(100);
  buff.writeBytes(badData1.getBytes());
  Event e = util.extractEvent(buff);
  if(e == null){
    throw new NullPointerException("Event is null");
  }
  Map<String, String> headers = e.getHeaders();
  Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_FACILITY));
  Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_SEVERITY));
  Assert.assertEquals(SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(),
      headers.get(SyslogUtils.EVENT_STATUS));
  Assert.assertEquals(badData1.trim(), new String(e.getBody()).trim());

}
 

private void publishMessage(Event event) throws EventDeliveryException {
    String rk;

    Map<String, String> headers = event.getHeaders();

    // Use a headers supplied routing key if it exists
    if (headers.containsKey(ROUTING_KEY)) {
        rk = headers.get(ROUTING_KEY);
    } else {
        rk = routingKey;
    }

    try {
        rmqChannel.basicPublish(exchange, rk, mandatory, createProperties(headers), event.getBody());
    } catch (IOException ex) {
        logger.error("Error publishing event message: {}", ex.toString());
        closeRabbitMQConnection();
        throw new EventDeliveryException(ex.toString());
    }

    if (publisherConfirms) waitForConfirmation();
}
 

/**
 * Test bad event format 1: Priority is not numeric
 */
@Test
public void testExtractBadEvent1() {
  String badData1 = "<10F> bad bad data\n";
  SyslogUtils util = new SyslogUtils(false);
  ChannelBuffer buff = ChannelBuffers.buffer(100);
  buff.writeBytes(badData1.getBytes());
  Event e = util.extractEvent(buff);
  if(e == null){
    throw new NullPointerException("Event is null");
  }
  Map<String, String> headers = e.getHeaders();
  Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_FACILITY));
  Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_SEVERITY));
  Assert.assertEquals(SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(),
      headers.get(SyslogUtils.EVENT_STATUS));
  Assert.assertEquals(badData1.trim(), new String(e.getBody()).trim());

}
 

@Test
public void testGoodEventBadEvent(){
  String badData1 = "hi guys! <10F> bad bad data\n";
  String priority = "<10>";
  String goodData1 = "Good good good data\n";
  SyslogUtils util = new SyslogUtils(false);
  ChannelBuffer buff = ChannelBuffers.buffer(100);
  buff.writeBytes((priority+goodData1).getBytes());
  buff.writeBytes(badData1.getBytes());

  Event e2 = util.extractEvent(buff);
  if(e2 == null){
    throw new NullPointerException("Event is null");
  }
  Map<String, String> headers2 = e2.getHeaders();
  Assert.assertEquals("1", headers2.get(SyslogUtils.SYSLOG_FACILITY));
  Assert.assertEquals("2", headers2.get(SyslogUtils.SYSLOG_SEVERITY));
  Assert.assertEquals(null,
      headers2.get(SyslogUtils.EVENT_STATUS));
  Assert.assertEquals(goodData1.trim(), new String(e2.getBody()).trim());

  Event e = util.extractEvent(buff);

  if(e == null){
    throw new NullPointerException("Event is null");
  }
  Map<String, String> headers = e.getHeaders();
  Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_FACILITY));
  Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_SEVERITY));
  Assert.assertEquals(SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(),
      headers.get(SyslogUtils.EVENT_STATUS));
  Assert.assertEquals(badData1.trim(), new String(e.getBody()).trim());

}
 

@Override
public Event intercept(Event event) {
  Map<String, String> headers = event.getHeaders();
  if (headers.containsKey("Bad-Words")) {
    headers.remove("Bad-Words");
  }
  return event;
}
 

public static String dumpEvent(Event event, int maxBytes) {
  StringBuilder buffer = new StringBuilder();
  if (event == null || event.getBody() == null) {
    buffer.append("null");
  } else if (event.getBody().length == 0) {
    // do nothing... in this case, HexDump.dump() will throw an exception
  } else {
    byte[] body = event.getBody();
    byte[] data = Arrays.copyOf(body, Math.min(body.length, maxBytes));
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    try {
      HexDump.dump(data, 0, out, 0);
      String hexDump = new String(out.toByteArray());
      // remove offset since it's not relevant for such a small dataset
      if(hexDump.startsWith(HEXDUMP_OFFSET)) {
        hexDump = hexDump.substring(HEXDUMP_OFFSET.length());
      }
      buffer.append(hexDump);
    } catch (Exception e) {
     if(LOGGER.isInfoEnabled()) {
       LOGGER.info("Exception while dumping event", e);
     }
      buffer.append("...Exception while dumping: ").append(e.getMessage());
    }
    String result = buffer.toString();
    if(result.endsWith(EOL) && buffer.length() > EOL.length()) {
      buffer.delete(buffer.length() - EOL.length(), buffer.length()).toString();
    }
  }
  return "{ headers:" + event.getHeaders() + " body:" + buffer + " }";
}
 

private void appendHeaders(XContentBuilder builder, Event event)
    throws IOException {
  Map<String, String> headers = event.getHeaders();
  for (String key : headers.keySet()) {
    ContentBuilderUtil.appendField(builder, key,
        headers.get(key).getBytes(charset));
  }
}
 

@Override
public Event intercept(Event event) {
  Map<String, String> headers = event.getHeaders();
  if (preserveExisting && headers.containsKey(headerName)) {
    // we must preserve the existing id
  } else if (isMatch(event)) {
    headers.put(headerName, generateUUID());
  }
  return event;
}
 

@Override
public void initialize(Event event, byte[] columnFamily) {
	this.headers = event.getHeaders();
	this.payload = event.getBody();
	this.depositHeaders = false;
	setSearchQueryInstruction();
	// hard coding column family impl here for now.
}
 

private List<ColumnNameValue> getColumnNameValueListFromEvent(Event event) {
    List<ColumnNameValue> columnNameValues = new ArrayList<ColumnNameValue>();
    Map<String, String> headers = event.getHeaders();
    for (StreamField field : streamFields) {
        String fieldContent = headers.get(field.getName());
        columnNameValues.add(new ColumnNameValue(field.getName(), fieldContent));
    }
    return columnNameValues;
}
 

@Test
public void testAvroReflect() throws IOException {
  loadProperties("flume-log4jtest-avro-reflect.properties");
  PropertyConfigurator.configure(props);
  Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class);
  String msg = "This is log message number " + String.valueOf(0);

  AppEvent appEvent = new AppEvent();
  appEvent.setMessage(msg);

  logger.info(appEvent);

  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  Assert.assertNotNull(event);

  Schema schema = ReflectData.get().getSchema(appEvent.getClass());

  ReflectDatumReader<AppEvent> reader = new ReflectDatumReader<AppEvent>(AppEvent.class);
  BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
  AppEvent recordFromEvent = reader.read(null, decoder);
  Assert.assertEquals(msg, recordFromEvent.getMessage());

  Map<String, String> hdrs = event.getHeaders();

  Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));

  Assert.assertNull("Schema URL should not be set",
      hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString()));
  Assert.assertEquals("Schema string should be set", schema.toString(),
      hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString()));

  transaction.commit();
  transaction.close();

}
 

@Test
public void testGoodEventGoodEvent() {

  String priority = "<10>";
  String goodData1 = "Good good good data\n";
  SyslogUtils util = new SyslogUtils(false);
  ChannelBuffer buff = ChannelBuffers.buffer(100);
  buff.writeBytes((priority+goodData1).getBytes());
  String priority2 = "<20>";
  String goodData2 = "Good really good data\n";
  buff.writeBytes((priority2+goodData2).getBytes());
  Event e = util.extractEvent(buff);
  if(e == null){
    throw new NullPointerException("Event is null");
  }
  Map<String, String> headers = e.getHeaders();
  Assert.assertEquals("1", headers.get(SyslogUtils.SYSLOG_FACILITY));
  Assert.assertEquals("2", headers.get(SyslogUtils.SYSLOG_SEVERITY));
  Assert.assertEquals(null,
      headers.get(SyslogUtils.EVENT_STATUS));
  Assert.assertEquals(goodData1.trim(), new String(e.getBody()).trim());


  Event e2 = util.extractEvent(buff);
  if(e2 == null){
    throw new NullPointerException("Event is null");
  }
  Map<String, String> headers2 = e2.getHeaders();
  Assert.assertEquals("2", headers2.get(SyslogUtils.SYSLOG_FACILITY));
  Assert.assertEquals("4", headers2.get(SyslogUtils.SYSLOG_SEVERITY));
  Assert.assertEquals(null,
      headers.get(SyslogUtils.EVENT_STATUS));
  Assert.assertEquals(goodData2.trim(), new String(e2.getBody()).trim());

}
 

private void appendHeaders(XContentBuilder builder, Event event)
    throws IOException {
  Map<String, String> headers = event.getHeaders();
  for (Map.Entry<String, String> entry : headers.entrySet()) {
    ContentBuilderUtil.appendField(builder, entry.getKey(),
            entry.getValue().getBytes(charset));
  }
}
 

@Test
public void testAvroReflect() throws IOException {
  loadProperties("flume-log4jtest-avro-reflect.properties");
  PropertyConfigurator.configure(props);
  Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class);
  String msg = "This is log message number " + String.valueOf(0);

  AppEvent appEvent = new AppEvent();
  appEvent.setMessage(msg);

  logger.info(appEvent);

  Transaction transaction = ch.getTransaction();
  transaction.begin();
  Event event = ch.take();
  Assert.assertNotNull(event);

  Schema schema = ReflectData.get().getSchema(appEvent.getClass());

  ReflectDatumReader<AppEvent> reader = new ReflectDatumReader<AppEvent>(AppEvent.class);
  BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null);
  AppEvent recordFromEvent = reader.read(null, decoder);
  Assert.assertEquals(msg, recordFromEvent.getMessage());

  Map<String, String> hdrs = event.getHeaders();

  Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString()));

  Assert.assertNull("Schema URL should not be set",
      hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString()));
  Assert.assertEquals("Schema string should be set", schema.toString(),
      hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString()));

  transaction.commit();
  transaction.close();

}
 

@Override
public Status process() throws EventDeliveryException {

    Status status = Status.READY;

    Channel channel = getChannel();
    Transaction tx = channel.getTransaction();
    try {
        tx.begin();
        
        List<KeyedMessage<String, String>> datas = new ArrayList<KeyedMessage<String, String>>();

        int txnEventCount = 0;
        for (txnEventCount = 0; txnEventCount < batchNumMessages; txnEventCount++) {
            Event event = channel.take();
            if (event == null) {
            	break;
            }         
            Map<String, String> headers = event.getHeaders();
            if(headers == null){
              logger.warn("headers are Null");
              continue;
            }
            
            String topic = headers.get("category");
            if(topic == null){
              logger.warn("headers do not contain entry of category");
              continue;
            }
            topic = topicPrefix + "." + topic;
            
            KeyedMessage<String, String> m = new KeyedMessage<String, String>(topic, new String(event.getBody()));
            datas.add(m);
        }

        producer.send(datas);  

        tx.commit();
    } catch (Exception e) {
        logger.error("can't process events, drop it!", e);
        tx.rollback();
        throw new EventDeliveryException(e);
    } finally {
    	tx.close();
    }
    return status;
}
 

@Override
public Event intercept(Event event) {
    if (invalidConfiguration) {
        return event;
    } // if

    LOGGER.debug("[nmi] Event intercepted, id=" + event.hashCode());

    // Casting to NGSIEvent
    NGSIEvent ngsiEvent = (NGSIEvent) event;

    // Get the original headers
    Map<String, String> headers = event.getHeaders();

    // Get some original header values
    String originalService = headers.get(CommonConstants.HEADER_FIWARE_SERVICE);
    String originalServicePath = headers.get(CommonConstants.HEADER_FIWARE_SERVICE_PATH);

    // Create the mapped NotifyContextRequest
    ImmutableTriple<String, String, ContextElement> map = doMap(originalService, originalServicePath,
            ngsiEvent.getOriginalCE());
    LOGGER.debug("[nmi] Mapped ContextElement: " + map.getRight().toString());

    // Add the mapped ContextElement to the NGSIEvent
    ngsiEvent.setMappedCE(map.getRight());

    // Add the bytes version of the mapped ContextElement to event's body
    byte[] originalCEBytes = ngsiEvent.getBody();
    byte[] mappedCEBytes = map.getRight().toString().getBytes();
    byte[] newBody = new byte[originalCEBytes.length + mappedCEBytes.length];
    System.arraycopy(originalCEBytes, 0, newBody, 0, originalCEBytes.length);
    System.arraycopy(mappedCEBytes, 0, newBody, originalCEBytes.length, mappedCEBytes.length);
    ngsiEvent.setBody(newBody);
    LOGGER.debug("[nmi] New body: " + new String(newBody));

    // Add the mapped service and service path to the headers
    headers.put(NGSIConstants.FLUME_HEADER_MAPPED_SERVICE, map.getLeft());
    LOGGER.debug("[nmi] Header added to NGSI event (" + NGSIConstants.FLUME_HEADER_MAPPED_SERVICE + ": "
            + map.getLeft() + ")");
    headers.put(NGSIConstants.FLUME_HEADER_MAPPED_SERVICE_PATH, map.getMiddle());
    LOGGER.debug("[nmi] Header added to NGSI event (" + NGSIConstants.FLUME_HEADER_MAPPED_SERVICE_PATH + ": "
            + map.getMiddle() + ")");

    // Return the intercepted event
    LOGGER.debug("[nmi] Event put in the channel, id=" + ngsiEvent.hashCode());
    return ngsiEvent;
}
 

@Override
public void append(Event event) throws EventDeliveryException {
  // Thrift IPC client is not thread safe, so don't allow state changes or
  // client.append* calls unless the lock is acquired.
  ClientWrapper client = null;
  boolean destroyedClient = false;
  try {
    if (!isActive()) {
      throw new EventDeliveryException("Client was closed due to error. " +
        "Please create a new client");
    }
    client = connectionManager.checkout();
    final ThriftFlumeEvent thriftEvent = new ThriftFlumeEvent(event
      .getHeaders(), ByteBuffer.wrap(event.getBody()));
    doAppend(client, thriftEvent).get(requestTimeout, TimeUnit.MILLISECONDS);
  } catch (Throwable e) {
    if (e instanceof ExecutionException) {
      Throwable cause = e.getCause();
      if (cause instanceof EventDeliveryException) {
        throw (EventDeliveryException) cause;
      } else if (cause instanceof TimeoutException) {
        throw new EventDeliveryException("Append call timeout", cause);
      }
    }
    destroyedClient = true;
    // If destroy throws, we still don't want to reuse the client, so mark it
    // as destroyed before we actually do.
    if (client != null) {
      connectionManager.destroy(client);
    }
    if (e instanceof Error) {
      throw (Error) e;
    } else if (e instanceof RuntimeException) {
      throw (RuntimeException) e;
    }
    throw new EventDeliveryException("Failed to send event. ", e);
  } finally {
    if (client != null && !destroyedClient) {
      connectionManager.checkIn(client);
    }
  }
}
 

/**
 * A no-op for this simple, special-case implementation
 * @param event
 * @return
 */
@Override
protected Container convert(Event event) {
    return new Container(event.getHeaders(), new String(event.getBody(), Charsets.UTF_8));
}