com.amazonaws.services.lambda.runtime.events.KinesisEvent.KinesisEventRecord Java Examples

public void handler(KinesisEvent event, Context context) throws HandlerException {
  if (!initialized) {
    init(context);
  }

  KinesisHandlerConfig handlerConfig = (KinesisHandlerConfig) this.config.getHandlerConfig();
  this.recordIterator = new KinesisEventIterator(new LambdaContext(context),
          event.getRecords(),
          handlerConfig.getAddShardIdToPartitions(),
          handlerConfig.getDecompress(),
          handlerConfig.getBufferSize());

  /*
   * Get processors based on the source stream ARN
   */
  KinesisEventRecord firstRecord = event.getRecords().get(0);
  this.source = SourceUtils.getSource(firstRecord.getEventSourceARN(), sources);

  super.process(context);
}
 

@Test
public void testAggregatedRecord() throws Exception {
	// create a new KinesisEvent.Record from the aggregated data
	KinesisEvent.Record r = new KinesisEvent.Record();
	r.setPartitionKey(aggregated.getPartitionKey());
	r.setApproximateArrivalTimestamp(new Date(System.currentTimeMillis()));
	r.setData(ByteBuffer.wrap(aggregated.toRecordBytes()));
	r.setKinesisSchemaVersion("1.0");
	KinesisEventRecord ker = new KinesisEventRecord();
	ker.setKinesis(r);

	// deaggregate the record
	List<KinesisClientRecord> userRecords = deaggregator.deaggregate(Arrays.asList(ker));

	assertEquals("Deaggregated Count Matches", aggregated.getNumUserRecords(), userRecords.size());
	verifyOneToOneMapping(userRecords);
}
 

public KinesisEventIterator(LambdaContext context,
                            List<KinesisEventRecord> records,
                            Boolean addShardidToPartitions,
                            Boolean decompress,
                            int bufferSize) {
  this.iterator = records.iterator();
  this.context = context;
  this.decompress = decompress;
  this.byteArrayOutputStream = new ByteArrayOutputStream();
  this.bufferSize = bufferSize;

  /*
   * All events in a batch will come from the same shard. So we only need to query this once.
   * 
   * eventid = shardId:sequenceId
   */
  if (addShardidToPartitions) {
    this.shardId = records.get(0).getEventID().split(":")[0];
  }
}
 

@Override
public InternalEvent next() {
  KinesisEventRecord nextRecord = this.iterator.next();
  if (decompress) {
    ByteBuffer gzip = nextRecord.getKinesis().getData();
    ByteBufferInputStream byteBufferInputStream = new ByteBufferInputStream(Collections.singletonList(gzip));
    try (GZIPInputStream gzipInputStream = new GZIPInputStream(byteBufferInputStream)) {
      IOUtils.copy(gzipInputStream, byteArrayOutputStream, bufferSize);
      nextRecord.getKinesis().setData(ByteBuffer.wrap(byteArrayOutputStream.toByteArray()));
    } catch (IOException e) {
      throw new KinesisIteratorException("Kinesis iterator was not able to expand the data gzip successfully.", e);
    } finally {
      byteArrayOutputStream.reset(); //clears output so it can be used again later
    }
  }
  return new KinesisInternalEvent(nextRecord, this.context, this.shardId);
}
 

private KinesisWrapper(final InternalEvent internal) {
  KinesisEventRecord eventRecord = ((KinesisInternalEvent) internal).getRecord();
  Record record = eventRecord.getKinesis();

  this.partitionKey = record.getPartitionKey();
  this.sequenceNumber = record.getSequenceNumber();
  this.eventSource = eventRecord.getEventSource();
  this.sourceArn = eventRecord.getEventSourceARN();
  this.functionName = internal.getCtx().getContext().getFunctionName();
  this.functionVersion = internal.getCtx().getContext().getFunctionVersion();
  this.processingTime = System.currentTimeMillis();
  this.arrivalTime = record.getApproximateArrivalTimestamp().getTime();
  this.timestamp = internal.getEventTime();
  this.processingDelay = processingTime - timestamp;

  if (internal.getEventObj() != null) {
    this.payload = internal.getEventObj().getPayload();
  } else {
    this.payload = null;
  }
}
 

@Override
public Void handleRequest(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();

	// extract the records from the event
	List<KinesisEventRecord> records = event.getRecords();

	logger.log(String.format("Recieved %s Raw Records", records.size()));

	try {
		// now deaggregate the message contents
		List<KinesisClientRecord> deaggregated = new RecordDeaggregator<KinesisEventRecord>().deaggregate(records);
		logger.log(String.format("Received %s Deaggregated User Records", deaggregated.size()));

		deaggregated.stream().forEachOrdered(rec -> {
			logger.log(rec.partitionKey());
		});
	} catch (Exception e) {
		logger.log(e.getMessage());
	}

	return null;
}
 

@SuppressWarnings("unchecked")
private List<Record> convertType(List<T> inputRecords) {
	List<Record> records = null;

	if (inputRecords.size() > 0 && inputRecords.get(0) instanceof KinesisEventRecord) {
		records = convertToKinesis((List<KinesisEventRecord>) inputRecords);
	} else if (inputRecords.size() > 0 && inputRecords.get(0) instanceof Record) {
		records = (List<Record>) inputRecords;
	} else {
		if (inputRecords.size() == 0) {
			return new ArrayList<Record>();
		} else {
			throw new InvalidArgumentException("Input Types must be Kinesis Event or Model Records");
		}
	}

	return records;
}
 

@Override
public Void handleRequest(KinesisEvent event, Context context) {
	LambdaLogger logger = context.getLogger();

	// extract the records from the event
	List<KinesisEventRecord> records = event.getRecords();

	logger.log(String.format("Recieved %s Raw Records", records.size()));

	// now deaggregate the message contents
	List<UserRecord> deaggregated = new RecordDeaggregator<KinesisEventRecord>().deaggregate(records);
	logger.log(String.format("Received %s Deaggregated User Records", deaggregated.size()));
	
	deaggregated.stream().forEachOrdered(rec -> {
		logger.log(rec.getPartitionKey());
	});

	return null;
}
 

@SuppressWarnings("unchecked")
private List<KinesisClientRecord> convertType(List<T> inputRecords) throws Exception {
	List<KinesisClientRecord> records = null;

	if (inputRecords.size() > 0 && inputRecords.get(0) instanceof KinesisEventRecord) {
		records = convertToKinesis((List<KinesisEventRecord>) inputRecords);
	} else if (inputRecords.size() > 0 && inputRecords.get(0) instanceof Record) {
		records = new ArrayList<>();
		for (Record rec : (List<Record>) inputRecords) {
			records.add(KinesisClientRecord.fromRecord((Record) rec));
		}
	} else {
		if (inputRecords.size() == 0) {
			return new ArrayList<KinesisClientRecord>();
		} else {
			throw new Exception("Input Types must be Kinesis Event or Model Records");
		}
	}

	return records;
}
 

@Test
public void testAggregatedRecord() {
	// create a new KinesisEvent.Record from the aggregated data
	KinesisEvent.Record r = new KinesisEvent.Record();
	r.setPartitionKey(aggregated.getPartitionKey());
	r.setApproximateArrivalTimestamp(new Date(System.currentTimeMillis()));
	r.setData(ByteBuffer.wrap(aggregated.toRecordBytes()));
	r.setKinesisSchemaVersion("1.0");
	KinesisEventRecord ker = new KinesisEventRecord();
	ker.setKinesis(r);

	// deaggregate the record
	List<UserRecord> userRecords = deaggregator.deaggregate(Arrays.asList(ker));

	assertEquals("Deaggregated Count Matches", aggregated.getNumUserRecords(), userRecords.size());
	verifyOneToOneMapping(userRecords);
}
 

@Test
public void testEmpty() throws Exception {
	// invoke deaggregation on the static records, returning a List of UserRecord
	List<KinesisClientRecord> records = deaggregator.deaggregate(new ArrayList<KinesisEventRecord>());

	assertEquals("Processed Record Count Correct", records.size(), 0);
	verifyOneToOneMapping(records);
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
	aggregator = new RecordAggregator();

	recordList = new LinkedList<>();

	// create 10 random records for testing
	for (int i = 0; i < c; i++) {
		// create trackable id
		String id = UUID.randomUUID().toString();

		// create a kinesis model record
		byte[] data = RandomStringUtils.randomAlphabetic(20).getBytes();

		KinesisEvent.Record r = new KinesisEvent.Record();
		r.withPartitionKey(id).withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
				.withData(ByteBuffer.wrap(data));
		KinesisEventRecord ker = new KinesisEventRecord();
		ker.setKinesis(r);
		recordList.add(ker);

		// add the record to the check set
		checkset.put(id, ker);

		// add the record to the aggregated AggRecord // create an aggregated set of
		aggregator.addUserRecord(id, data);
	}

	// get the aggregated data
	aggregated = aggregator.clearAndGet();
	assertEquals("Aggregated Record Count Correct", aggregated.getNumUserRecords(), c);
}
 

private void verifyOneToOneMapping(List<KinesisClientRecord> userRecords) {
	userRecords.stream().forEachOrdered(userRecord -> {
		// get the original checkset record by ID
		KinesisEventRecord toCheck = checkset.get(userRecord.partitionKey());

		// confirm that toCheck is not null
		assertNotNull("Found Original CheckSet Record", toCheck);

		// confirm that the data is the same
		assertTrue("Data Correct", userRecord.data().compareTo(toCheck.getKinesis().getData()) == 0);
	});
}
 

public static List<software.amazon.awssdk.services.kinesis.model.Record> convertToKinesis(
		List<KinesisEventRecord> inputRecords) {
	List<software.amazon.awssdk.services.kinesis.model.Record> response = new ArrayList<>();

	inputRecords.stream().forEachOrdered(record -> {
		response.add(convertOne(record));
	});

	return response;

}
 

public static software.amazon.awssdk.services.kinesis.model.Record convertOne(KinesisEventRecord record) {
	KinesisEvent.Record r = record.getKinesis();
	software.amazon.awssdk.services.kinesis.model.Record out = software.amazon.awssdk.services.kinesis.model.Record
			.builder().partitionKey(r.getPartitionKey()).encryptionType(r.getEncryptionType())
			.approximateArrivalTimestamp(r.getApproximateArrivalTimestamp().toInstant())
			.sequenceNumber(r.getSequenceNumber()).data(SdkBytes.fromByteBuffer(r.getData())).build();

	return out;
}
 

private Record convertOne(KinesisEventRecord record) {
	KinesisEvent.Record r = record.getKinesis();
	Record out = Record.builder().partitionKey(r.getPartitionKey()).encryptionType(r.getEncryptionType())
			.approximateArrivalTimestamp(r.getApproximateArrivalTimestamp().toInstant())
			.sequenceNumber(r.getSequenceNumber()).data(SdkBytes.fromByteBuffer(r.getData())).build();

	return out;

}
 

@Test
public void testEmpty() {
	// invoke deaggregation on the static records, returning a List of UserRecord
	List<UserRecord> records = deaggregator.deaggregate(new ArrayList<KinesisEventRecord>());

	assertEquals("Processed Record Count Correct", records.size(), 0);
	verifyOneToOneMapping(records);
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
	aggregator = new RecordAggregator();

	recordList = new LinkedList<>();

	// create 10 random records for testing
	for (int i = 0; i < c; i++) {
		// create trackable id
		String id = UUID.randomUUID().toString();

		// create a kinesis model record
		byte[] data = RandomStringUtils.randomAlphabetic(20).getBytes();

		KinesisEvent.Record r = new KinesisEvent.Record();
		r.withPartitionKey(id).withApproximateArrivalTimestamp(new Date(System.currentTimeMillis()))
				.withData(ByteBuffer.wrap(data));
		KinesisEventRecord ker = new KinesisEventRecord();
		ker.setKinesis(r);
		recordList.add(ker);

		// add the record to the check set
		checkset.put(id, ker);

		// add the record to the aggregated AggRecord // create an aggregated set of
		aggregator.addUserRecord(id, data);
	}

	// get the aggregated data
	aggregated = aggregator.clearAndGet();
	assertEquals("Aggregated Record Count Correct", aggregated.getNumUserRecords(), c);
}
 

private void verifyOneToOneMapping(List<UserRecord> userRecords) {
	userRecords.stream().forEachOrdered(userRecord -> {
		// get the original checkset record by ID
		KinesisEventRecord toCheck = checkset.get(userRecord.getPartitionKey());

		// confirm that toCheck is not null
		assertNotNull("Found Original CheckSet Record", toCheck);

		// confirm that the data is the same
		assertEquals("Data Correct", userRecord.getData().toString(), toCheck.getKinesis().getData().toString());
	});
}
 

private com.amazonaws.services.kinesis.model.Record convertOne(KinesisEventRecord record) {
	KinesisEvent.Record r = record.getKinesis();
	com.amazonaws.services.kinesis.model.Record out = new com.amazonaws.services.kinesis.model.Record()
			.withPartitionKey(r.getPartitionKey()).withEncryptionType(r.getEncryptionType())
			.withApproximateArrivalTimestamp(r.getApproximateArrivalTimestamp())
			.withSequenceNumber(r.getSequenceNumber()).withData(r.getData());

	return out;

}
 

public static KinesisEvent createEvent(Class clazz, String resource, String eventSourceArn) throws IOException {
  /*
   * Create a kinesis record from a sample JSON file
   */
  String json =
      IOUtils.toString(new InputStreamReader(clazz.getResourceAsStream(resource), "UTF-8"));

  Date approximateArrivalTimestamp = new Date();
  approximateArrivalTimestamp.setTime(1478737790000l);

  Record rec = new Record();
  rec.withPartitionKey("1").withSequenceNumber("2").withData(ByteBuffer.wrap(json.getBytes()))
      .withApproximateArrivalTimestamp(approximateArrivalTimestamp);

  /*
   * Create a KinesisEventRecord and add single Record
   */
  KinesisEventRecord krecord = new KinesisEventRecord();
  krecord.setKinesis(rec);
  krecord.setEventSourceARN(eventSourceArn);
  krecord.setEventID("shardId-000000000000:1234");

  /*
   * Add single KinesisEventRecord to a KinesisEvent
   */
  KinesisEvent kevent = new KinesisEvent();
  List<KinesisEventRecord> events = new ArrayList<KinesisEventRecord>(1);
  events.add(krecord);
  kevent.setRecords(events);

  return kevent;
}
 

public KinesisInternalEvent(KinesisEventRecord record, LambdaContext context, String shardId) {
  super(new String(record.getKinesis().getData().array()), context,
      record.getKinesis().getApproximateArrivalTimestamp().getTime());

  super.addMetadata("eventSource", record.getEventSource());
  super.addMetadata("eventSourceArn", record.getEventSourceARN());
  super.addMetadata("eventID", record.getEventID());
  super.addMetadata("awsRegion", record.getAwsRegion());
  super.addMetadata("partitionKey", record.getKinesis().getPartitionKey());
  super.addMetadata("sequenceNumber", record.getKinesis().getSequenceNumber());

  this.record = record;
  this.shardId = shardId;
}
 

public KinesisEventRecord getRecord() {
  return record;
}
 

@Override
protected List<KinesisEventRecord> convertEvent(KinesisEvent event) {
	// TODO: maybe convert to List<Message>
	return event.getRecords();
}
 

private List<KinesisClientRecord> convertToKinesis(List<KinesisEventRecord> inputRecords) {
	List<KinesisClientRecord> response = new ArrayList<>();

	inputRecords.stream().forEachOrdered(record -> {
		response.add(KinesisClientRecord.fromRecord(convertOne(record)));
	});

	return response;

}
 

private List<com.amazonaws.services.kinesis.model.Record> convertToKinesis(List<KinesisEventRecord> inputRecords) {
	List<com.amazonaws.services.kinesis.model.Record> response = new ArrayList<>();

	inputRecords.stream().forEachOrdered(record -> {
		response.add(convertOne(record));
	});

	return response;

}