com.amazonaws.services.kinesis.model.PutRecordsRequestEntry Java Examples

private void createJsonMessages(String streamName, int count, int idStart)
{
    String jsonFormat = "{\"id\" : %d, \"name\" : \"%s\"}";
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        long id = idStart + i;
        String name = UUID.randomUUID().toString();
        String jsonVal = String.format(jsonFormat, id, name);

        // ? with StandardCharsets.UTF_8
        putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonVal.getBytes(UTF_8)));
        putRecordsRequestEntry.setPartitionKey(Long.toString(id));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 

@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest)
        throws AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        List<PutRecordsResultEntry> resultList = new ArrayList<>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 

private void createMessages(String streamName, int count)
        throws Exception
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest);
}
 

@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) throws AmazonServiceException, AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        ArrayList<PutRecordsResultEntry> resultList = new ArrayList<PutRecordsResultEntry>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 

private void createJsonMessages(String streamName, int count, int idStart)
        throws Exception
{
    String jsonFormat = "{\"id\" : %d, \"name\" : \"%s\"}";
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        long id = idStart + i;
        String name = UUID.randomUUID().toString();
        String jsonVal = String.format(jsonFormat, id, name);

        // ? with StandardCharsets.UTF_8
        putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonVal.getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(id));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 

private void createDummyMessages(String streamName, int count)
        throws Exception
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 

private void generateRecords()
{
  // Create dummy message
  int recordNo = 1;
  while (recordNo <= sendCount) {
    String dataStr = "Record_" + recordNo;
    PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
    putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
    putRecordsEntry.setPartitionKey(dataStr);
    putRecordsRequestEntryList.add(putRecordsEntry);
    if ( (putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount )) {
      PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
      putRecordsRequest.setStreamName(streamName);
      putRecordsRequest.setRecords(putRecordsRequestEntryList);
      client.putRecords(putRecordsRequest);
      putRecordsRequestEntryList.clear();
    }
    recordNo++;
  }
}
 

protected void addEntry(PutRecordsRequestEntry entry) {
    int newDataSize = dataSize + entry.getData().remaining() +
            entry.getPartitionKey().length();
    if (newDataSize <= 5 * 1024 * 1024 && entries.size() < 500) {
        dataSize = newDataSize;
        entries.add(entry);
    } else {
        flush();
        dataSize = 0;
        addEntry(entry);
    }
}
 

@Override
protected void runOnce() throws Exception {
    ClickEvent event = inputQueue.take();
    String partitionKey = event.getSessionId();
    ByteBuffer data = ByteBuffer.wrap(
            event.getPayload().getBytes("UTF-8"));
    recordsPut.getAndIncrement();

    addEntry(new PutRecordsRequestEntry()
            .withPartitionKey(partitionKey)
            .withData(data));
}
 

public Optional<PutRecordsRequest> put(PutRecordsRequestEntry entry) {
    int newRequestSize = requestSize + entry.getData().remaining()
            + entry.getPartitionKey().length();
    if (entries.size() < maxCount && newRequestSize <= maxSize) {
        requestSize = newRequestSize;
        entries.add(entry);
        return Optional.empty();
    } else {
        Optional<PutRecordsRequest> ret = flush();
        put(entry);
        return ret;
    }
}
 

private static Collection<PutRecordsRequestEntry> getRecords(int numRecords) {
  return IntStream.range(0, numRecords).mapToObj(i -> {
    PutRecordsRequestEntry record = new PutRecordsRequestEntry();
    record.setData(ByteBuffer.wrap("{\"a\":1,\"b\":2,\"c\":2}".getBytes(Charsets.UTF_8)));
    record.setPartitionKey(String.valueOf(i));
    return record;
  }).collect(Collectors.toList());
}
 

private void addRecord(T tuple)
{
  try {
    Pair<String, V> keyValue = tupleToKeyValue(tuple);
    PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
    putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second)));
    putRecordsEntry.setPartitionKey(keyValue.first);
    putRecordsRequestEntryList.add(putRecordsEntry);
  } catch (AmazonClientException e) {
    throw new RuntimeException(e);
  }
}
 

public static void main(String[] args) throws Exception
{
    if (args.length != 2)
    {
        System.err.println("USAGE: SampleNormalProducer <stream name> <region>");
        System.exit(1);
    }

    String streamName = args[0];
    String regionName = args[1];

    AmazonKinesis producer = ProducerUtils.getKinesisProducer(regionName);

    System.out.println("Creating " + ProducerConfig.RECORDS_TO_TRANSMIT + " records...");
    List<PutRecordsRequestEntry> entries = new LinkedList<>();
    for (int i = 1; i <= ProducerConfig.RECORDS_TO_TRANSMIT; i++)
    {
        byte[] data = ProducerUtils.randomData(i, ProducerConfig.RECORD_SIZE_BYTES);
        entries.add(new PutRecordsRequestEntry()
                    .withPartitionKey(ProducerUtils.randomPartitionKey())
                    .withExplicitHashKey(ProducerUtils.randomExplicitHashKey())
                    .withData(ByteBuffer.wrap(data)));
    }

    PutRecordsRequest request = new PutRecordsRequest().withRecords(entries).withStreamName(streamName);

    System.out.println("Sending " + ProducerConfig.RECORDS_TO_TRANSMIT + " records...");
    producer.putRecords(request);
    System.out.println("Complete.");
}
 

private void createMessages(String streamName, long count)
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(UTF_8)));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest);
}
 

private void createDummyMessages(String streamName, int count)
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(UTF_8)));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 

private static PutRecordsRequestEntry getBadRecord() {
  PutRecordsRequestEntry badRecord = new PutRecordsRequestEntry();
  badRecord.setData(ByteBuffer.wrap("{\"a\":1,\"b\":2,\"c\":2,".getBytes(Charsets.UTF_8)));
  badRecord.setPartitionKey(String.valueOf(0));
  return badRecord;
}
 

@Override
public final Response addRecords(final Request request, final LambdaLogger logger) {

    this.documentAddedCount = request.getCount();

    DescribeStreamResult result = this.kinesisClient.describeStream(request.getStreamName());
    logger.log("Stream Status: " + result.getStreamDescription().getStreamStatus() + ". ");

    logger.log("Adding records to Stream...");

    String payload;

    for (int i = 1; i <= request.getCount(); i++) {

        payload = request.getPayload() + i;

        this.kinesisBatch.add(new PutRecordsRequestEntry()
                .withPartitionKey(request.getPartitionKey())
                .withData(ByteBuffer.wrap(payload.getBytes())));

        if (this.kinesisBatch.size() >= request.getBatchSize()) {

            try {
                logger.log("Flushing records to Stream...");
                flushBatch(request.getStreamName(), logger);
            } catch (Exception e) {
                logger.log("Exception occurred: " + e);
                this.isError = false;
            } finally {
                this.kinesisBatch.clear();
            }
        }

    }

    if (this.isError) {
        return new Response(ERROR_MESSAGE, documentAddedCount);
    } else {
        return new Response(SUCCESS_MESSAGE, documentAddedCount);
    }
}
 

/**
 * Run the kineses logger in a loop
 */
public void runKinesisLogger() {
    Object msg = null;
    String str = null;
    int i;
    List <PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    while (!me.isInterrupted()) {
        try {
            if ((msg = queue.poll()) != null) {
                i = 1;
                PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
                while(msg != null) {
                    str = serialize(msg);
                    byte [] bytes = str.getBytes();
                    PutRecordsRequestEntry putRecordsRequestEntry  = new PutRecordsRequestEntry();
                    putRecordsRequestEntry.setPartitionKey(kinesis.getPartition());
                    putRecordsRequestEntry.setData(ByteBuffer.wrap(bytes));
                    putRecordsRequestEntryList.add(putRecordsRequestEntry);

                    if (i++ == 100)
                        msg = null;
                    else
                        msg = queue.poll();
                }
                putRecordsRequest.setRecords(putRecordsRequestEntryList);
                putRecordsRequest.setStreamName(kinesis.getStream());
                PutRecordsResult putRecordsResult  = kinesis.getKinesis().putRecords(putRecordsRequest);
                putRecordsRequestEntryList.clear();
            }
            Thread.sleep(1);


            /*while ((msg = queue.poll()) != null) {
                str = serialize(msg);
                var bytes = str.getBytes();
                PutRecordRequest putRecord = new PutRecordRequest();
                putRecord.setStreamName(kinesis.getStream());
                putRecord.setPartitionKey(kinesis.getPartition());
                putRecord.setData(ByteBuffer.wrap(bytes));

                try {
                    kinesis.getKinesis().putRecord(putRecord);
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
            Thread.sleep(1); */
        } catch (Exception e) {
            e.printStackTrace();
            // return;
        }
    }
}
 

private void putRecords() {
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(AWSCommon.DEFAULT_KINESIS_STREAM);

    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();

    LOG.debug("Adding data to the Kinesis stream");
    for (int i = 0; i < expect; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(String.valueOf(i).getBytes()));

        String partition = String.format("partitionKey-%d", i);
        putRecordsRequestEntry.setPartitionKey(partition);

        LOG.debug("Added data {} (as bytes) to partition {}", i, partition);

        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    LOG.debug("Done creating the data records");

    int retries = 5;
    do {
        try {
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            PutRecordsResult putRecordsResult = awsKinesisClient.putRecords(putRecordsRequest);

            if (putRecordsResult.getFailedRecordCount() == 0) {
                LOG.debug("Done putting the data records into the stream");
            } else {
                fail("Unable to put all the records into the stream");
            }

            break;
        } catch (AmazonServiceException e) {
            retries--;

            /*
             This works around the "... Cannot deserialize instance of `...AmazonKinesisException` out of NOT_AVAILABLE token

             It may take some time for the local Kinesis backend to be fully up - even though the container is
             reportedly up and running. Therefore, it tries a few more times
             */
            LOG.trace("Failed to put the records: {}. Retrying in 2 seconds ...", e.getMessage());
            if (retries == 0) {
                LOG.error("Failed to put the records: {}", e.getMessage(), e);
                throw e;
            }


            try {
                Thread.sleep(TimeUnit.SECONDS.toMillis(2));
            } catch (InterruptedException ex) {
                break;
            }
        }
    } while (retries > 0);


}
 

/**
 * Convert the aggregated data in this record into a single
 * PutRecordsRequestEntry. This method has no side effects (i.e. it will not
 * clear the current contents of the aggregated record).
 * 
 * @return A PutRecordsRequestEntry containing all the current data in this
 *         aggregated record that can be sent to Kinesis via a
 *         PutRecordsRequest.
 */
public PutRecordsRequestEntry toPutRecordsRequestEntry() {
	return new PutRecordsRequestEntry().withExplicitHashKey(getExplicitHashKey())
			.withPartitionKey(getPartitionKey()).withData(ByteBuffer.wrap(toRecordBytes()));
}