Java Code Examples for com.amazonaws.services.kinesis.model.GetRecordsRequest#setLimit()

The following examples show how to use com.amazonaws.services.kinesis.model.GetRecordsRequest#setLimit() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KinesisTestConsumer.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up 
public String processNextIterator(String iterator)
{
  GetRecordsRequest getRequest = new GetRecordsRequest();
  getRequest.setLimit(1000);

  getRequest.setShardIterator(iterator);
  // call "get" operation and get everything in this shard range
  GetRecordsResult getResponse = client.getRecords(getRequest);

  iterator = getResponse.getNextShardIterator();

  List<Record> records = getResponse.getRecords();
  processResponseRecords(records);

  return iterator;
}
Example 2
Source File: KinesisUtil.java    From datacollector with Apache License 2.0 6 votes vote down vote up 
public static List<com.amazonaws.services.kinesis.model.Record> getPreviewRecords(
    ClientConfiguration awsClientConfig,
    KinesisConfigBean conf,
    int maxBatchSize,
    GetShardIteratorRequest getShardIteratorRequest
) throws StageException {
  AmazonKinesis kinesisClient = getKinesisClient(awsClientConfig, conf);

  GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
  String shardIterator = getShardIteratorResult.getShardIterator();

  GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
  getRecordsRequest.setShardIterator(shardIterator);
  getRecordsRequest.setLimit(maxBatchSize);

  GetRecordsResult getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
  return getRecordsResult.getRecords();
}
Example 3
Source File: KinesisProxy.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
Example 4
Source File: KinesisProxy.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
Example 5
Source File: KinesisProxy.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * {@inheritDoc}
 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
	final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
	getRecordsRequest.setShardIterator(shardIterator);
	getRecordsRequest.setLimit(maxRecordsToGet);

	GetRecordsResult getRecordsResult = null;

	int retryCount = 0;
	while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
		try {
			getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
		} catch (SdkClientException ex) {
			if (isRecoverableSdkClientException(ex)) {
				long backoffMillis = fullJitterBackoff(
					getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
				LOG.warn("Got recoverable SdkClientException. Backing off for "
					+ backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
				Thread.sleep(backoffMillis);
			} else {
				throw ex;
			}
		}
	}

	if (getRecordsResult == null) {
		throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries +
			" retry attempts failed.");
	}

	return getRecordsResult;
}
Example 6
Source File: KinesisRecordSet.java    From presto with Apache License 2.0 4 votes vote down vote up 
/**
 * Retrieves the next batch of records from Kinesis using the shard iterator.
 * <p>
 * Most of the time this results in one getRecords call.  However we allow for
 * a call to return an empty list, and we'll try again if we are far enough
 * away from the latest record.
 */
private void getKinesisRecords()
        throws ResourceNotFoundException
{
    // Normally this loop will execute once, but we have to allow for the odd Kinesis
    // behavior, per the docs:
    // A single call to getRecords might return an empty record list, even when the shard contains
    // more records at later sequence numbers
    boolean fetchedRecords = false;
    int attempts = 0;
    while (!fetchedRecords && attempts < fetchAttempts) {
        Duration duration = nanosSince(lastReadTime);
        if (duration.toMillis() <= sleepTime) {
            try {
                Thread.sleep(duration.toMillis());
            }
            catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RuntimeException("thread interrupted");
            }
        }
        getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(batchSize);

        getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest);
        lastReadTime = System.nanoTime();

        shardIterator = getRecordsResult.getNextShardIterator();
        kinesisRecords = getRecordsResult.getRecords();
        if (isLogBatches) {
            log.info("Fetched %d records from Kinesis.  MillisBehindLatest=%d", kinesisRecords.size(), getRecordsResult.getMillisBehindLatest());
        }

        fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT);
        attempts++;
    }

    listIterator = kinesisRecords.iterator();
    batchesRead++;
    messagesRead += kinesisRecords.size();
}
Example 7
Source File: KinesisRecordSet.java    From presto-kinesis with Apache License 2.0 4 votes vote down vote up 
/**
 * Retrieves the next batch of records from Kinesis using the shard iterator.
 *
 * Most of the time this results in one getRecords call.  However we allow for
 * a call to return an empty list, and we'll try again if we are far enough
 * away from the latest record.
 */
private void getKinesisRecords()
        throws ResourceNotFoundException
{
    // Normally this loop will execute once, but we have to allow for the odd Kinesis
    // behavior, per the docs:
    // A single call to getRecords might return an empty record list, even when the shard contains
    // more records at later sequence numbers
    boolean fetchedRecords = false;
    int attempts = 0;
    while (!fetchedRecords && attempts < fetchAttempts) {
        long now = System.currentTimeMillis();
        if (now - lastReadTime <= sleepTime) {
            try {
                Thread.sleep(now - lastReadTime);
            }
            catch (InterruptedException e) {
                log.error("Sleep interrupted.", e);
            }
        }
        getRecordsRequest = new GetRecordsRequest();
        getRecordsRequest.setShardIterator(shardIterator);
        getRecordsRequest.setLimit(batchSize);

        getRecordsResult = clientManager.getClient().getRecords(getRecordsRequest);
        lastReadTime = System.currentTimeMillis();

        shardIterator = getRecordsResult.getNextShardIterator();
        kinesisRecords = getRecordsResult.getRecords();
        if (kinesisConnectorConfig.isLogBatches()) {
            log.info("Fetched %d records from Kinesis.  MillisBehindLatest=%d", kinesisRecords.size(), getRecordsResult.getMillisBehindLatest());
        }

        fetchedRecords = (kinesisRecords.size() > 0 || getMillisBehindLatest() <= MILLIS_BEHIND_LIMIT);
        attempts++;
    }

    listIterator = kinesisRecords.iterator();
    batchesRead++;
    messagesRead += kinesisRecords.size();
}