Java Code Examples for com.amazonaws.services.kinesis.model.PutRecordsResult

The following examples show how to use com.amazonaws.services.kinesis.model.PutRecordsResult. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: MockKinesisClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest)
        throws AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        List<PutRecordsResultEntry> resultList = new ArrayList<>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 
Example 2
Source Project: presto-kinesis   Source File: MockKinesisClient.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) throws AmazonServiceException, AmazonClientException
{
    // Setup method to add a batch of new records:
    InternalStream theStream = this.getStream(putRecordsRequest.getStreamName());
    if (theStream != null) {
        PutRecordsResult result = new PutRecordsResult();
        ArrayList<PutRecordsResultEntry> resultList = new ArrayList<PutRecordsResultEntry>();
        for (PutRecordsRequestEntry entry : putRecordsRequest.getRecords()) {
            PutRecordResult putResult = theStream.putRecord(entry.getData(), entry.getPartitionKey());
            resultList.add((new PutRecordsResultEntry()).withShardId(putResult.getShardId()).withSequenceNumber(putResult.getSequenceNumber()));
        }

        result.setRecords(resultList);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 
Example 3
private void flushBatch(final String streamName, final LambdaLogger logger) {
    final PutRecordsResult result = this.kinesisClient.putRecords(new PutRecordsRequest()
            .withStreamName(streamName)
            .withRecords(this.kinesisBatch));

    result.getRecords().forEach(r -> {
        if (!(StringUtils.hasValue(r.getErrorCode()))) {
            String successMessage = "Successfully processed record with sequence number: " + r.getSequenceNumber()
                    + ", shard id: " + r.getShardId();
            logger.log(successMessage);
        } else {
            this.documentAddedCount--;

            String errorMessage = "Did not process record with sequence number: " + r.getSequenceNumber()
                    + ", error code: " + r.getErrorCode()
                    + ", error message: " + r.getErrorMessage();
            logger.log(errorMessage);
            this.isError = true;
        }
    });


    // You may also implement a retry logic only for failed records (e.g. Create a list for failed records,
    // add error records to that list and finally retry all failed records until a max retry count is reached.)
    /*
    if (result.getFailedRecordCount() != null && result.getFailedRecordCount() > 0) {
        result.getRecords().forEach(r -> {
            if ((r != null) && (StringUtils.hasValue(r.getErrorCode()))) {
                // add this record to the retry list.
            }
        });
    }
    */
}
 
Example 4
private void putRecords() {
    PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
    putRecordsRequest.setStreamName(AWSCommon.DEFAULT_KINESIS_STREAM);

    List<PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();

    LOG.debug("Adding data to the Kinesis stream");
    for (int i = 0; i < expect; i++) {
        PutRecordsRequestEntry putRecordsRequestEntry = new PutRecordsRequestEntry();
        putRecordsRequestEntry.setData(ByteBuffer.wrap(String.valueOf(i).getBytes()));

        String partition = String.format("partitionKey-%d", i);
        putRecordsRequestEntry.setPartitionKey(partition);

        LOG.debug("Added data {} (as bytes) to partition {}", i, partition);

        putRecordsRequestEntryList.add(putRecordsRequestEntry);
    }

    LOG.debug("Done creating the data records");

    int retries = 5;
    do {
        try {
            putRecordsRequest.setRecords(putRecordsRequestEntryList);
            PutRecordsResult putRecordsResult = awsKinesisClient.putRecords(putRecordsRequest);

            if (putRecordsResult.getFailedRecordCount() == 0) {
                LOG.debug("Done putting the data records into the stream");
            } else {
                fail("Unable to put all the records into the stream");
            }

            break;
        } catch (AmazonServiceException e) {
            retries--;

            /*
             This works around the "... Cannot deserialize instance of `...AmazonKinesisException` out of NOT_AVAILABLE token

             It may take some time for the local Kinesis backend to be fully up - even though the container is
             reportedly up and running. Therefore, it tries a few more times
             */
            LOG.trace("Failed to put the records: {}. Retrying in 2 seconds ...", e.getMessage());
            if (retries == 0) {
                LOG.error("Failed to put the records: {}", e.getMessage(), e);
                throw e;
            }


            try {
                Thread.sleep(TimeUnit.SECONDS.toMillis(2));
            } catch (InterruptedException ex) {
                break;
            }
        }
    } while (retries > 0);


}
 
Example 5
Source Project: bidder   Source File: ZPublisher.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Run the kineses logger in a loop
 */
public void runKinesisLogger() {
    Object msg = null;
    String str = null;
    int i;
    List <PutRecordsRequestEntry> putRecordsRequestEntryList = new ArrayList<>();
    while (!me.isInterrupted()) {
        try {
            if ((msg = queue.poll()) != null) {
                i = 1;
                PutRecordsRequest putRecordsRequest = new PutRecordsRequest();
                while(msg != null) {
                    str = serialize(msg);
                    byte [] bytes = str.getBytes();
                    PutRecordsRequestEntry putRecordsRequestEntry  = new PutRecordsRequestEntry();
                    putRecordsRequestEntry.setPartitionKey(kinesis.getPartition());
                    putRecordsRequestEntry.setData(ByteBuffer.wrap(bytes));
                    putRecordsRequestEntryList.add(putRecordsRequestEntry);

                    if (i++ == 100)
                        msg = null;
                    else
                        msg = queue.poll();
                }
                putRecordsRequest.setRecords(putRecordsRequestEntryList);
                putRecordsRequest.setStreamName(kinesis.getStream());
                PutRecordsResult putRecordsResult  = kinesis.getKinesis().putRecords(putRecordsRequest);
                putRecordsRequestEntryList.clear();
            }
            Thread.sleep(1);


            /*while ((msg = queue.poll()) != null) {
                str = serialize(msg);
                var bytes = str.getBytes();
                PutRecordRequest putRecord = new PutRecordRequest();
                putRecord.setStreamName(kinesis.getStream());
                putRecord.setPartitionKey(kinesis.getPartition());
                putRecord.setData(ByteBuffer.wrap(bytes));

                try {
                    kinesis.getKinesis().putRecord(putRecord);
                } catch (Exception ex) {
                    ex.printStackTrace();
                }
            }
            Thread.sleep(1); */
        } catch (Exception e) {
            e.printStackTrace();
            // return;
        }
    }
}
 
Example 6
Source Project: beam   Source File: AmazonKinesisMock.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public PutRecordsResult putRecords(PutRecordsRequest putRecordsRequest) {
  throw new RuntimeException("Not implemented");
}