Java Code Examples for com.amazonaws.services.kinesis.model.PutRecordsRequestEntry

The following examples show how to use com.amazonaws.services.kinesis.model.PutRecordsRequestEntry. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: TestRecordAccess.java    License: Apache License 2.0 6 votes vote down vote up
private void createJsonMessages(String streamName, int count, int idStart)
{
    String jsonFormat = "{\"id\" : %d, \"name\" : \"%s\"}";
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        long id = idStart + i;
        String name = UUID.randomUUID().toString();
        String jsonVal = String.format(jsonFormat, id, name);

        // ? with StandardCharsets.UTF_8
        putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonVal.getBytes(UTF_8)));
        putRecordsRequestEntry.setPartitionKey(Long.toString(id));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 
Example 2
Source Project: presto   Source File: MockKinesisClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest)
        throws AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        List<PutRecordsResultEntry> resultList = new ArrayList<>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 
Example 3
Source Project: attic-apex-malhar   Source File: KinesisTestProducer.java    License: Apache License 2.0 6 votes vote down vote up
private void generateRecords()
{
  // Create dummy message
  int recordNo = 1;
  while (recordNo <= sendCount) {
    String dataStr = "Record_" + recordNo;
    PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
    putRecordsEntry.setData(ByteBuffer.wrap(dataStr.getBytes()));
    putRecordsEntry.setPartitionKey(dataStr);
    putRecordsRequestEntryList.add(putRecordsEntry);
    if ( (putRecordsRequestEntryList.size() == batchSize) || (recordNo == sendCount )) {
      PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
      putRecordsRequest.setStreamName(streamName);
      putRecordsRequest.setRecords(putRecordsRequestEntryList);
      client.putRecords(putRecordsRequest);
      putRecordsRequestEntryList.clear();
    }
    recordNo++;
  }
}
 
Example 4
Source Project: presto-kinesis   Source File: TestRecordAccess.java    License: Apache License 2.0 6 votes vote down vote up
private void createDummyMessages(String streamName, int count)
        throws Exception
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 
Example 5
Source Project: presto-kinesis   Source File: TestRecordAccess.java    License: Apache License 2.0 6 votes vote down vote up
private void createJsonMessages(String streamName, int count, int idStart)
        throws Exception
{
    String jsonFormat = "{\"id\" : %d, \"name\" : \"%s\"}";
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        long id = idStart + i;
        String name = UUID.randomUUID().toString();
        String jsonVal = String.format(jsonFormat, id, name);

        // ? with StandardCharsets.UTF_8
        putRecordsRequestEntry.setData(ByteBuffer.wrap(jsonVal.getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(id));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 
Example 6
Source Project: presto-kinesis   Source File: MockKinesisClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) throws AmazonServiceException, AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        ArrayList<PutRecordsResultEntry> resultList = new ArrayList<PutRecordsResultEntry>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 
Example 7
Source Project: presto-kinesis   Source File: TestMinimalFunctionality.java    License: Apache License 2.0 6 votes vote down vote up
private void createMessages(String streamName, int count)
        throws Exception
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes()));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest);
}
 
Example 8
Source Project: presto   Source File: TestRecordAccess.java    License: Apache License 2.0 5 votes vote down vote up
private void createDummyMessages(String streamName, int count)
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(UTF_8)));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    mockClient.putRecords(putRecordsRequest);
}
 
Example 9
Source Project: presto   Source File: TestMinimalFunctionality.java    License: Apache License 2.0 5 votes vote down vote up
private void createMessages(String streamName, long count)
{
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(streamName);
    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    for (int i = 0; i < count; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(UTF_8)));
        putRecordsRequestEntry.setPartitionKey(Long.toString(i));
        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    putRecordsRequest.setRecords(putRecordsRequestEntryList);
    embeddedKinesisStream.getKinesisClient().putRecords(putRecordsRequest);
}
 
Example 10
Source Project: kinesis-aggregation   Source File: SampleNormalProducer.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception
{
    if (args.length != 2)
    {
        System.err.println("USAGE: SampleNormalProducer <stream name> <region>");
        System.exit(1);
    }

    String streamName = args[0];
    String regionName = args[1];

    AmazonKinesis producer = ProducerUtils.getKinesisProducer(regionName);

    System.out.println("Creating " + ProducerConfig.RECORDS_TO_TRANSMIT + " records...");
    List<PutRecordsRequestEntry> entries = new LinkedList<>();
    for (int i = 1; i <= ProducerConfig.RECORDS_TO_TRANSMIT; i++)
    {
        byte[] data = ProducerUtils.randomData(i, ProducerConfig.RECORD_SIZE_BYTES);
        entries.add(new PutRecordsRequestEntry()
                    .withPartitionKey(ProducerUtils.randomPartitionKey())
                    .withExplicitHashKey(ProducerUtils.randomExplicitHashKey())
                    .withData(ByteBuffer.wrap(data)));
    }

    PutRecordsRequest request = new PutRecordsRequest().withRecords(entries).withStreamName(streamName);

    System.out.println("Sending " + ProducerConfig.RECORDS_TO_TRANSMIT + " records...");
    producer.putRecords(request);
    System.out.println("Complete.");
}
 
Example 11
private void addRecord(T tuple)
{
  try {
    Pair<String, V> keyValue = tupleToKeyValue(tuple);
    PutRecordsRequestEntry putRecordsEntry = new PutRecordsRequestEntry();
    putRecordsEntry.setData(ByteBuffer.wrap(getRecord(keyValue.second)));
    putRecordsEntry.setPartitionKey(keyValue.first);
    putRecordsRequestEntryList.add(putRecordsEntry);
  } catch (AmazonClientException e) {
    throw new RuntimeException(e);
  }
}
 
Example 12
Source Project: datacollector   Source File: KinesisSourceIT.java    License: Apache License 2.0 5 votes vote down vote up
private static Collection<PutRecordsRequestEntry> getRecords(int numRecords) {
  return IntStream.range(0, numRecords).mapToObj(i -> {
    PutRecordsRequestEntry record = new PutRecordsRequestEntry();
    record.setData(ByteBuffer.wrap("{\"a\":1,\"b\":2,\"c\":2}".getBytes(Charsets.UTF_8)));
    record.setPartitionKey(String.valueOf(i));
    return record;
  }).collect(Collectors.toList());
}
 
Example 13
Source Project: aws-big-data-blog   Source File: RecordBatcher.java    License: Apache License 2.0 5 votes vote down vote up
public Optional<PutRecordsRequest> put(PutRecordsRequestEntry entry) {
    int newRequestSize = requestSize + entry.getData().remaining()
            + entry.getPartitionKey().length();
    if (entries.size() < maxCount && newRequestSize <= maxSize) {
        requestSize = newRequestSize;
        entries.add(entry);
        return Optional.empty();
    } else {
        Optional<PutRecordsRequest> ret = flush();
        put(entry);
        return ret;
    }
}
 
Example 14
@Override
protected void runOnce() throws Exception {
    ClickEvent event = inputQueue.take();
    String partitionKey = event.getSessionId();
    ByteBuffer data = ByteBuffer.wrap(
            event.getPayload().getBytes("UTF-8"));
    recordsPut.getAndIncrement();

    addEntry(new PutRecordsRequestEntry()
            .withPartitionKey(partitionKey)
            .withData(data));
}
 
Example 15
protected void addEntry(PutRecordsRequestEntry entry) {
    int newDataSize = dataSize + entry.getData().remaining() +
            entry.getPartitionKey().length();
    if (newDataSize <= 5 * 1024 * 1024 && entries.size() < 500) {
        dataSize = newDataSize;
        entries.add(entry);
    } else {
        flush();
        dataSize = 0;
        addEntry(entry);
    }
}
 
Example 16
private void putRecords() {
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(AWSCommon.DEFAULT_KINESIS_STREAM);

    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();

    LOG.debug("Adding data to the Kinesis stream");
    for (int i = 0; i < expect; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(String.valueOf(i).getBytes()));

        String partition = String.format("partitionKey-%d", i);
        putRecordsRequestEntry.setPartitionKey(partition);

        LOG.debug("Added data {} (as bytes) to partition {}", i, partition);

        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    LOG.debug("Done creating the data records");

    int retries = 5;
    do {
        try {
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            PutRecordsResult putRecordsResult = awsKinesisClient.putRecords(putRecordsRequest);

            if (putRecordsResult.getFailedRecordCount() == 0) {
                LOG.debug("Done putting the data records into the stream");
            } else {
                fail("Unable to put all the records into the stream");
            }

            break;
        } catch (AmazonServiceException e) {
            retries--;

            /*
             This works around the "... Cannot deserialize instance of `...AmazonKinesisException` out of NOT_AVAILABLE token

             It may take some time for the local Kinesis backend to be fully up - even though the container is
             reportedly up and running. Therefore, it tries a few more times
             */
            LOG.trace("Failed to put the records: {}. Retrying in 2 seconds ...", e.getMessage());
            if (retries == 0) {
                LOG.error("Failed to put the records: {}", e.getMessage(), e);
                throw e;
            }


            try {
                Thread.sleep(TimeUnit.SECONDS.toMillis(2));
            } catch (InterruptedException ex) {
                break;
            }
        }
    } while (retries > 0);


}
 
Example 17
Source Project: bidder   Source File: ZPublisher.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Run the kineses logger in a loop
 */
public void runKinesisLogger() {
    Object msg = null;
    String str = null;
    int i;
    List <PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    while (!me.isInterrupted()) {
        try {
            if ((msg = queue.poll()) != null) {
                i = 1;
                PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
                while(msg != null) {
                    str = serialize(msg);
                    byte [] bytes = str.getBytes();
                    PutRecordsRequestEntry putRecordsRequestEntry  = new PutRecordsRequestEntry();
                    putRecordsRequestEntry.setPartitionKey(kinesis.getPartition());
                    putRecordsRequestEntry.setData(ByteBuffer.wrap(bytes));
                    putRecordsRequestEntryList.add(putRecordsRequestEntry);

                    if (i++ == 100)
                        msg = null;
                    else
                        msg = queue.poll();
                }
                putRecordsRequest.setRecords(putRecordsRequestEntryList);
                putRecordsRequest.setStreamName(kinesis.getStream());
                PutRecordsResult putRecordsResult  = kinesis.getKinesis().putRecords(putRecordsRequest);
                putRecordsRequestEntryList.clear();
            }
            Thread.sleep(1);


            /*while ((msg = queue.poll()) != null) {
                str = serialize(msg);
                var bytes = str.getBytes();
                PutRecordRequest putRecord = new PutRecordRequest();
                putRecord.setStreamName(kinesis.getStream());
                putRecord.setPartitionKey(kinesis.getPartition());
                putRecord.setData(ByteBuffer.wrap(bytes));

                try {
                    kinesis.getKinesis().putRecord(putRecord);
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
            Thread.sleep(1); */
        } catch (Exception e) {
            e.printStackTrace();
            // return;
        }
    }
}
 
Example 18
@Override
public final Response addRecords(final Request request, final LambdaLogger logger) {

    this.documentAddedCount = request.getCount();

    DescribeStreamResult result = this.kinesisClient.describeStream(request.getStreamName());
    logger.log("Stream Status: " + result.getStreamDescription().getStreamStatus() + ". ");

    logger.log("Adding records to Stream...");

    String payload;

    for (int i = 1; i <= request.getCount(); i++) {

        payload = request.getPayload() + i;

        this.kinesisBatch.add(new PutRecordsRequestEntry()
                .withPartitionKey(request.getPartitionKey())
                .withData(ByteBuffer.wrap(payload.getBytes())));

        if (this.kinesisBatch.size() >= request.getBatchSize()) {

            try {
                logger.log("Flushing records to Stream...");
                flushBatch(request.getStreamName(), logger);
            } catch (Exception e) {
                logger.log("Exception occurred: " + e);
                this.isError = false;
            } finally {
                this.kinesisBatch.clear();
            }
        }

    }

    if (this.isError) {
        return new Response(ERROR_MESSAGE, documentAddedCount);
    } else {
        return new Response(SUCCESS_MESSAGE, documentAddedCount);
    }
}
 
Example 19
Source Project: datacollector   Source File: KinesisSourceIT.java    License: Apache License 2.0 4 votes vote down vote up
private static PutRecordsRequestEntry getBadRecord() {
  PutRecordsRequestEntry badRecord = new PutRecordsRequestEntry();
  badRecord.setData(ByteBuffer.wrap("{\"a\":1,\"b\":2,\"c\":2,".getBytes(Charsets.UTF_8)));
  badRecord.setPartitionKey(String.valueOf(0));
  return badRecord;
}
 
Example 20
Source Project: kinesis-aggregation   Source File: AggRecord.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Convert the aggregated data in this record into a single
 * PutRecordsRequestEntry. This method has no side effects (i.e. it will not
 * clear the current contents of the aggregated record).
 * 
 * @return A PutRecordsRequestEntry containing all the current data in this
 *         aggregated record that can be sent to Kinesis via a
 *         PutRecordsRequest.
 */
public PutRecordsRequestEntry toPutRecordsRequestEntry() {
	return new PutRecordsRequestEntry().withExplicitHashKey(getExplicitHashKey())
			.withPartitionKey(getPartitionKey()).withData(ByteBuffer.wrap(toRecordBytes()));
}