Java Code Examples for org.apache.flume.Event#getHeaders()
The following examples show how to use
org.apache.flume.Event#getHeaders() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RollingCountInterceptor.java From flume-interceptor-analytics with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override public Event intercept(Event event) { List<T> objects = getObjectsToCount(event); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Identified %d objects to count from event: %s", objects.size(), objects)); } if (!objects.isEmpty()) { Map<String, String> headers = event.getHeaders(); for (T obj : objects) { long total = counters.incrementCount(obj); headers.put(String.valueOf(obj), String.valueOf(total)); if (LOG.isDebugEnabled()) { LOG.debug(String.format("Added counter to event header: %s=%d", obj.toString(), total)); } } } return event; }
Example 2
Source File: EventParser.java From ingestion with Apache License 2.0 | 6 votes |
private Map<String, Object> parseEvent(Event event) { final Map<String, String> headers = event.getHeaders(); Map<String, Object> parsedEvent = null; if (MapUtils.isNotEmpty(headers)) { parsedEvent = new HashMap<String, Object>(); for (String header : headers.keySet()) { if (timestampField.equalsIgnoreCase(header)) { parsedEvent.put(header, Long.valueOf(headers.get(header))); } else { parsedEvent.put(header, headers.get(header)); } } } return parsedEvent; }
Example 3
Source File: TestSyslogUtils.java From mt-flume with Apache License 2.0 | 6 votes |
/** * Test bad event format 2: The first char is not < */ @Test public void testExtractBadEvent2() { String badData1 = "hi guys! <10> bad bad data\n"; SyslogUtils util = new SyslogUtils(false); ChannelBuffer buff = ChannelBuffers.buffer(100); buff.writeBytes(badData1.getBytes()); Event e = util.extractEvent(buff); if(e == null){ throw new NullPointerException("Event is null"); } Map<String, String> headers = e.getHeaders(); Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_FACILITY)); Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_SEVERITY)); Assert.assertEquals(SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(), headers.get(SyslogUtils.EVENT_STATUS)); Assert.assertEquals(badData1.trim(), new String(e.getBody()).trim()); }
Example 4
Source File: RabbitMQSink.java From rabbitmq-flume-plugin with BSD 3-Clause "New" or "Revised" License | 6 votes |
private void publishMessage(Event event) throws EventDeliveryException { String rk; Map<String, String> headers = event.getHeaders(); // Use a headers supplied routing key if it exists if (headers.containsKey(ROUTING_KEY)) { rk = headers.get(ROUTING_KEY); } else { rk = routingKey; } try { rmqChannel.basicPublish(exchange, rk, mandatory, createProperties(headers), event.getBody()); } catch (IOException ex) { logger.error("Error publishing event message: {}", ex.toString()); closeRabbitMQConnection(); throw new EventDeliveryException(ex.toString()); } if (publisherConfirms) waitForConfirmation(); }
Example 5
Source File: TestSyslogUtils.java From mt-flume with Apache License 2.0 | 6 votes |
/** * Test bad event format 1: Priority is not numeric */ @Test public void testExtractBadEvent1() { String badData1 = "<10F> bad bad data\n"; SyslogUtils util = new SyslogUtils(false); ChannelBuffer buff = ChannelBuffers.buffer(100); buff.writeBytes(badData1.getBytes()); Event e = util.extractEvent(buff); if(e == null){ throw new NullPointerException("Event is null"); } Map<String, String> headers = e.getHeaders(); Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_FACILITY)); Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_SEVERITY)); Assert.assertEquals(SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(), headers.get(SyslogUtils.EVENT_STATUS)); Assert.assertEquals(badData1.trim(), new String(e.getBody()).trim()); }
Example 6
Source File: TestSyslogUtils.java From mt-flume with Apache License 2.0 | 5 votes |
@Test public void testGoodEventBadEvent(){ String badData1 = "hi guys! <10F> bad bad data\n"; String priority = "<10>"; String goodData1 = "Good good good data\n"; SyslogUtils util = new SyslogUtils(false); ChannelBuffer buff = ChannelBuffers.buffer(100); buff.writeBytes((priority+goodData1).getBytes()); buff.writeBytes(badData1.getBytes()); Event e2 = util.extractEvent(buff); if(e2 == null){ throw new NullPointerException("Event is null"); } Map<String, String> headers2 = e2.getHeaders(); Assert.assertEquals("1", headers2.get(SyslogUtils.SYSLOG_FACILITY)); Assert.assertEquals("2", headers2.get(SyslogUtils.SYSLOG_SEVERITY)); Assert.assertEquals(null, headers2.get(SyslogUtils.EVENT_STATUS)); Assert.assertEquals(goodData1.trim(), new String(e2.getBody()).trim()); Event e = util.extractEvent(buff); if(e == null){ throw new NullPointerException("Event is null"); } Map<String, String> headers = e.getHeaders(); Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_FACILITY)); Assert.assertEquals("0", headers.get(SyslogUtils.SYSLOG_SEVERITY)); Assert.assertEquals(SyslogUtils.SyslogStatus.INVALID.getSyslogStatus(), headers.get(SyslogUtils.EVENT_STATUS)); Assert.assertEquals(badData1.trim(), new String(e.getBody()).trim()); }
Example 7
Source File: CensoringInterceptor.java From mt-flume with Apache License 2.0 | 5 votes |
@Override public Event intercept(Event event) { Map<String, String> headers = event.getHeaders(); if (headers.containsKey("Bad-Words")) { headers.remove("Bad-Words"); } return event; }
Example 8
Source File: EventHelper.java From mt-flume with Apache License 2.0 | 5 votes |
public static String dumpEvent(Event event, int maxBytes) { StringBuilder buffer = new StringBuilder(); if (event == null || event.getBody() == null) { buffer.append("null"); } else if (event.getBody().length == 0) { // do nothing... in this case, HexDump.dump() will throw an exception } else { byte[] body = event.getBody(); byte[] data = Arrays.copyOf(body, Math.min(body.length, maxBytes)); ByteArrayOutputStream out = new ByteArrayOutputStream(); try { HexDump.dump(data, 0, out, 0); String hexDump = new String(out.toByteArray()); // remove offset since it's not relevant for such a small dataset if(hexDump.startsWith(HEXDUMP_OFFSET)) { hexDump = hexDump.substring(HEXDUMP_OFFSET.length()); } buffer.append(hexDump); } catch (Exception e) { if(LOGGER.isInfoEnabled()) { LOGGER.info("Exception while dumping event", e); } buffer.append("...Exception while dumping: ").append(e.getMessage()); } String result = buffer.toString(); if(result.endsWith(EOL) && buffer.length() > EOL.length()) { buffer.delete(buffer.length() - EOL.length(), buffer.length()).toString(); } } return "{ headers:" + event.getHeaders() + " body:" + buffer + " }"; }
Example 9
Source File: ElasticSearchDynamicSerializer.java From ingestion with Apache License 2.0 | 5 votes |
private void appendHeaders(XContentBuilder builder, Event event) throws IOException { Map<String, String> headers = event.getHeaders(); for (String key : headers.keySet()) { ContentBuilderUtil.appendField(builder, key, headers.get(key).getBytes(charset)); } }
Example 10
Source File: UUIDInterceptor.java From mt-flume with Apache License 2.0 | 5 votes |
@Override public Event intercept(Event event) { Map<String, String> headers = event.getHeaders(); if (preserveExisting && headers.containsKey(headerName)) { // we must preserve the existing id } else if (isMatch(event)) { headers.put(headerName, generateUUID()); } return event; }
Example 11
Source File: HbaseJsonEventSerializer.java From searchanalytics-bigdata with MIT License | 5 votes |
@Override public void initialize(Event event, byte[] columnFamily) { this.headers = event.getHeaders(); this.payload = event.getBody(); this.depositHeaders = false; setSearchQueryInstruction(); // hard coding column family impl here for now. }
Example 12
Source File: StratioDecisionSink.java From ingestion with Apache License 2.0 | 5 votes |
private List<ColumnNameValue> getColumnNameValueListFromEvent(Event event) { List<ColumnNameValue> columnNameValues = new ArrayList<ColumnNameValue>(); Map<String, String> headers = event.getHeaders(); for (StreamField field : streamFields) { String fieldContent = headers.get(field.getName()); columnNameValues.add(new ColumnNameValue(field.getName(), fieldContent)); } return columnNameValues; }
Example 13
Source File: TestLog4jAppenderWithAvro.java From kite with Apache License 2.0 | 5 votes |
@Test public void testAvroReflect() throws IOException { loadProperties("flume-log4jtest-avro-reflect.properties"); PropertyConfigurator.configure(props); Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class); String msg = "This is log message number " + String.valueOf(0); AppEvent appEvent = new AppEvent(); appEvent.setMessage(msg); logger.info(appEvent); Transaction transaction = ch.getTransaction(); transaction.begin(); Event event = ch.take(); Assert.assertNotNull(event); Schema schema = ReflectData.get().getSchema(appEvent.getClass()); ReflectDatumReader<AppEvent> reader = new ReflectDatumReader<AppEvent>(AppEvent.class); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null); AppEvent recordFromEvent = reader.read(null, decoder); Assert.assertEquals(msg, recordFromEvent.getMessage()); Map<String, String> hdrs = event.getHeaders(); Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString())); Assert.assertNull("Schema URL should not be set", hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString())); Assert.assertEquals("Schema string should be set", schema.toString(), hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString())); transaction.commit(); transaction.close(); }
Example 14
Source File: TestSyslogUtils.java From mt-flume with Apache License 2.0 | 5 votes |
@Test public void testGoodEventGoodEvent() { String priority = "<10>"; String goodData1 = "Good good good data\n"; SyslogUtils util = new SyslogUtils(false); ChannelBuffer buff = ChannelBuffers.buffer(100); buff.writeBytes((priority+goodData1).getBytes()); String priority2 = "<20>"; String goodData2 = "Good really good data\n"; buff.writeBytes((priority2+goodData2).getBytes()); Event e = util.extractEvent(buff); if(e == null){ throw new NullPointerException("Event is null"); } Map<String, String> headers = e.getHeaders(); Assert.assertEquals("1", headers.get(SyslogUtils.SYSLOG_FACILITY)); Assert.assertEquals("2", headers.get(SyslogUtils.SYSLOG_SEVERITY)); Assert.assertEquals(null, headers.get(SyslogUtils.EVENT_STATUS)); Assert.assertEquals(goodData1.trim(), new String(e.getBody()).trim()); Event e2 = util.extractEvent(buff); if(e2 == null){ throw new NullPointerException("Event is null"); } Map<String, String> headers2 = e2.getHeaders(); Assert.assertEquals("2", headers2.get(SyslogUtils.SYSLOG_FACILITY)); Assert.assertEquals("4", headers2.get(SyslogUtils.SYSLOG_SEVERITY)); Assert.assertEquals(null, headers.get(SyslogUtils.EVENT_STATUS)); Assert.assertEquals(goodData2.trim(), new String(e2.getBody()).trim()); }
Example 15
Source File: ElasticSearchDynamicSerializer.java From ElasticsearchSink2 with Apache License 2.0 | 5 votes |
private void appendHeaders(XContentBuilder builder, Event event) throws IOException { Map<String, String> headers = event.getHeaders(); for (Map.Entry<String, String> entry : headers.entrySet()) { ContentBuilderUtil.appendField(builder, entry.getKey(), entry.getValue().getBytes(charset)); } }
Example 16
Source File: TestLog4jAppenderWithAvro.java From mt-flume with Apache License 2.0 | 5 votes |
@Test public void testAvroReflect() throws IOException { loadProperties("flume-log4jtest-avro-reflect.properties"); PropertyConfigurator.configure(props); Logger logger = LogManager.getLogger(TestLog4jAppenderWithAvro.class); String msg = "This is log message number " + String.valueOf(0); AppEvent appEvent = new AppEvent(); appEvent.setMessage(msg); logger.info(appEvent); Transaction transaction = ch.getTransaction(); transaction.begin(); Event event = ch.take(); Assert.assertNotNull(event); Schema schema = ReflectData.get().getSchema(appEvent.getClass()); ReflectDatumReader<AppEvent> reader = new ReflectDatumReader<AppEvent>(AppEvent.class); BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(event.getBody(), null); AppEvent recordFromEvent = reader.read(null, decoder); Assert.assertEquals(msg, recordFromEvent.getMessage()); Map<String, String> hdrs = event.getHeaders(); Assert.assertNull(hdrs.get(Log4jAvroHeaders.MESSAGE_ENCODING.toString())); Assert.assertNull("Schema URL should not be set", hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_URL.toString())); Assert.assertEquals("Schema string should be set", schema.toString(), hdrs.get(Log4jAvroHeaders.AVRO_SCHEMA_LITERAL.toString())); transaction.commit(); transaction.close(); }
Example 17
Source File: KafkaSink.java From mt-flume with Apache License 2.0 | 4 votes |
@Override public Status process() throws EventDeliveryException { Status status = Status.READY; Channel channel = getChannel(); Transaction tx = channel.getTransaction(); try { tx.begin(); List<KeyedMessage<String, String>> datas = new ArrayList<KeyedMessage<String, String>>(); int txnEventCount = 0; for (txnEventCount = 0; txnEventCount < batchNumMessages; txnEventCount++) { Event event = channel.take(); if (event == null) { break; } Map<String, String> headers = event.getHeaders(); if(headers == null){ logger.warn("headers are Null"); continue; } String topic = headers.get("category"); if(topic == null){ logger.warn("headers do not contain entry of category"); continue; } topic = topicPrefix + "." + topic; KeyedMessage<String, String> m = new KeyedMessage<String, String>(topic, new String(event.getBody())); datas.add(m); } producer.send(datas); tx.commit(); } catch (Exception e) { logger.error("can't process events, drop it!", e); tx.rollback(); throw new EventDeliveryException(e); } finally { tx.close(); } return status; }
Example 18
Source File: NGSINameMappingsInterceptor.java From fiware-cygnus with GNU Affero General Public License v3.0 | 4 votes |
@Override public Event intercept(Event event) { if (invalidConfiguration) { return event; } // if LOGGER.debug("[nmi] Event intercepted, id=" + event.hashCode()); // Casting to NGSIEvent NGSIEvent ngsiEvent = (NGSIEvent) event; // Get the original headers Map<String, String> headers = event.getHeaders(); // Get some original header values String originalService = headers.get(CommonConstants.HEADER_FIWARE_SERVICE); String originalServicePath = headers.get(CommonConstants.HEADER_FIWARE_SERVICE_PATH); // Create the mapped NotifyContextRequest ImmutableTriple<String, String, ContextElement> map = doMap(originalService, originalServicePath, ngsiEvent.getOriginalCE()); LOGGER.debug("[nmi] Mapped ContextElement: " + map.getRight().toString()); // Add the mapped ContextElement to the NGSIEvent ngsiEvent.setMappedCE(map.getRight()); // Add the bytes version of the mapped ContextElement to event's body byte[] originalCEBytes = ngsiEvent.getBody(); byte[] mappedCEBytes = map.getRight().toString().getBytes(); byte[] newBody = new byte[originalCEBytes.length + mappedCEBytes.length]; System.arraycopy(originalCEBytes, 0, newBody, 0, originalCEBytes.length); System.arraycopy(mappedCEBytes, 0, newBody, originalCEBytes.length, mappedCEBytes.length); ngsiEvent.setBody(newBody); LOGGER.debug("[nmi] New body: " + new String(newBody)); // Add the mapped service and service path to the headers headers.put(NGSIConstants.FLUME_HEADER_MAPPED_SERVICE, map.getLeft()); LOGGER.debug("[nmi] Header added to NGSI event (" + NGSIConstants.FLUME_HEADER_MAPPED_SERVICE + ": " + map.getLeft() + ")"); headers.put(NGSIConstants.FLUME_HEADER_MAPPED_SERVICE_PATH, map.getMiddle()); LOGGER.debug("[nmi] Header added to NGSI event (" + NGSIConstants.FLUME_HEADER_MAPPED_SERVICE_PATH + ": " + map.getMiddle() + ")"); // Return the intercepted event LOGGER.debug("[nmi] Event put in the channel, id=" + ngsiEvent.hashCode()); return ngsiEvent; }
Example 19
Source File: ThriftRpcClient.java From mt-flume with Apache License 2.0 | 4 votes |
@Override public void append(Event event) throws EventDeliveryException { // Thrift IPC client is not thread safe, so don't allow state changes or // client.append* calls unless the lock is acquired. ClientWrapper client = null; boolean destroyedClient = false; try { if (!isActive()) { throw new EventDeliveryException("Client was closed due to error. " + "Please create a new client"); } client = connectionManager.checkout(); final ThriftFlumeEvent thriftEvent = new ThriftFlumeEvent(event .getHeaders(), ByteBuffer.wrap(event.getBody())); doAppend(client, thriftEvent).get(requestTimeout, TimeUnit.MILLISECONDS); } catch (Throwable e) { if (e instanceof ExecutionException) { Throwable cause = e.getCause(); if (cause instanceof EventDeliveryException) { throw (EventDeliveryException) cause; } else if (cause instanceof TimeoutException) { throw new EventDeliveryException("Append call timeout", cause); } } destroyedClient = true; // If destroy throws, we still don't want to reuse the client, so mark it // as destroyed before we actually do. if (client != null) { connectionManager.destroy(client); } if (e instanceof Error) { throw (Error) e; } else if (e instanceof RuntimeException) { throw (RuntimeException) e; } throw new EventDeliveryException("Failed to send event. ", e); } finally { if (client != null && !destroyedClient) { connectionManager.checkIn(client); } } }
Example 20
Source File: FlumeEventStringAvroEventSerializer.java From flume-plugins with MIT License | 2 votes |
/** * A no-op for this simple, special-case implementation * @param event * @return */ @Override protected Container convert(Event event) { return new Container(event.getHeaders(), new String(event.getBody(), Charsets.UTF_8)); }