Java Code Examples for com.amazonaws.services.kinesis.model.GetShardIteratorRequest#setTimestamp()

The following examples show how to use com.amazonaws.services.kinesis.model.GetShardIteratorRequest#setTimestamp() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KinesisRecordSet.java From presto with Apache License 2.0

5 votes

private void getIterator()
        throws ResourceNotFoundException
{
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(split.getStreamName());
    getShardIteratorRequest.setShardId(split.getShardId());

    // Explanation: when we have a sequence number from a prior read or checkpoint, always use it.
    // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration.
    // If starting at a timestamp, use the session variable STARTING_TIMESTAMP when given, otherwise
    // fallback on starting at STARTING_OFFSET_SECONDS from timestamp.
    if (lastReadSequenceNumber == null) {
        if (isIteratorFromTimestamp(session)) {
            getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP");
            long iteratorStartTimestamp = getIteratorStartTimestamp(session);
            if (iteratorStartTimestamp == 0) {
                long startTimestamp = System.currentTimeMillis() - (getIteratorOffsetSeconds(session) * 1000);
                getShardIteratorRequest.setTimestamp(new Date(startTimestamp));
            }
            else {
                getShardIteratorRequest.setTimestamp(new Date(iteratorStartTimestamp));
            }
        }
        else {
            getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");
        }
    }
    else {
        getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER");
        getShardIteratorRequest.setStartingSequenceNumber(lastReadSequenceNumber);
    }

    GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}

Example 2

Source File: KinesisProxy.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * {@inheritDoc}
 */
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, @Nullable Object startingMarker) throws InterruptedException {
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
		.withStreamName(shard.getStreamName())
		.withShardId(shard.getShard().getShardId())
		.withShardIteratorType(shardIteratorType);

	switch (ShardIteratorType.fromValue(shardIteratorType)) {
		case TRIM_HORIZON:
		case LATEST:
			break;
		case AT_TIMESTAMP:
			if (startingMarker instanceof Date) {
				getShardIteratorRequest.setTimestamp((Date) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_TIMESTAMP. Must be a Date object.");
			}
			break;
		case AT_SEQUENCE_NUMBER:
		case AFTER_SEQUENCE_NUMBER:
			if (startingMarker instanceof String) {
				getShardIteratorRequest.setStartingSequenceNumber((String) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. Must be a String.");
			}
	}
	return getShardIterator(getShardIteratorRequest);
}

Example 3

Source File: KinesisProxy.java From flink with Apache License 2.0

5 votes

/**
 * {@inheritDoc}
 */
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, @Nullable Object startingMarker) throws InterruptedException {
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
		.withStreamName(shard.getStreamName())
		.withShardId(shard.getShard().getShardId())
		.withShardIteratorType(shardIteratorType);

	switch (ShardIteratorType.fromValue(shardIteratorType)) {
		case TRIM_HORIZON:
		case LATEST:
			break;
		case AT_TIMESTAMP:
			if (startingMarker instanceof Date) {
				getShardIteratorRequest.setTimestamp((Date) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_TIMESTAMP. Must be a Date object.");
			}
			break;
		case AT_SEQUENCE_NUMBER:
		case AFTER_SEQUENCE_NUMBER:
			if (startingMarker instanceof String) {
				getShardIteratorRequest.setStartingSequenceNumber((String) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. Must be a String.");
			}
	}
	return getShardIterator(getShardIteratorRequest);
}

Example 4

Source File: KinesisSource.java From datacollector with Apache License 2.0

5 votes

private void previewProcess(
    int maxBatchSize,
    BatchMaker batchMaker
) throws IOException, StageException {
  ClientConfiguration awsClientConfig = AWSUtil.getClientConfiguration(conf.proxyConfig);

  String shardId = KinesisUtil.getLastShardId(awsClientConfig, conf, conf.streamName);

  GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
  getShardIteratorRequest.setStreamName(conf.streamName);
  getShardIteratorRequest.setShardId(shardId);
  getShardIteratorRequest.setShardIteratorType(conf.initialPositionInStream.name());

  if (conf.initialPositionInStream == InitialPositionInStream.AT_TIMESTAMP) {
    getShardIteratorRequest.setTimestamp(new Date(conf.initialTimestamp));
  }

  if (!getContext().isPreview() && conf.maxBatchSize > maxBatchSize) {
    getContext().reportError(Errors.KINESIS_18, maxBatchSize);
  }

  List<com.amazonaws.services.kinesis.model.Record> results = KinesisUtil.getPreviewRecords(
      awsClientConfig,
      conf,
      Math.min(conf.maxBatchSize, maxBatchSize),
      getShardIteratorRequest
  );

  int batchSize = results.size() > maxBatchSize ? maxBatchSize : results.size();

  for (int index = 0; index < batchSize; index++) {
    com.amazonaws.services.kinesis.model.Record record = results.get(index);
    UserRecord userRecord = new UserRecord(record);
    KinesisUtil.processKinesisRecord(
        getShardIteratorRequest.getShardId(),
        userRecord,
        parserFactory
    ).forEach(batchMaker::addRecord);
  }
}

Example 5

Source File: KinesisRecordSet.java From presto-kinesis with Apache License 2.0

5 votes

private void getIterator()
        throws ResourceNotFoundException
{
    GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
    getShardIteratorRequest.setStreamName(split.getStreamName());
    getShardIteratorRequest.setShardId(split.getShardId());

    // Explanation: when we have a sequence number from a prior read or checkpoint, always use it.
    // Otherwise, decide if starting at a timestamp or the trim horizon based on configuration.
    // If starting at a timestamp, sue the session variable ITER_START_TIMESTAMP when given, otherwise
    // fallback on starting at ITER_OFFSET_SECONDS from timestamp.
    if (lastReadSeqNo == null) {
        // Important: shard iterator type AT_TIMESTAMP requires 1.11.x or above of the AWS SDK.
        if (SessionVariables.getIterFromTimestamp(session)) {
            getShardIteratorRequest.setShardIteratorType("AT_TIMESTAMP");
            long iterStartTs = SessionVariables.getIterStartTimestamp(session);
            if (iterStartTs == 0) {
                long startTs = System.currentTimeMillis() - (SessionVariables.getIterOffsetSeconds(session) * 1000);
                getShardIteratorRequest.setTimestamp(new Date(startTs));
            }
            else {
                getShardIteratorRequest.setTimestamp(new Date(iterStartTs));
            }
        }
        else {
            getShardIteratorRequest.setShardIteratorType("TRIM_HORIZON");
        }
    }
    else {
        getShardIteratorRequest.setShardIteratorType("AFTER_SEQUENCE_NUMBER");
        getShardIteratorRequest.setStartingSequenceNumber(lastReadSeqNo);
    }

    GetShardIteratorResult getShardIteratorResult = clientManager.getClient().getShardIterator(getShardIteratorRequest);
    shardIterator = getShardIteratorResult.getShardIterator();
}

Example 6

Source File: KinesisProxy.java From flink with Apache License 2.0

5 votes

/**
 * {@inheritDoc}
 */
@Override
public String getShardIterator(StreamShardHandle shard, String shardIteratorType, @Nullable Object startingMarker) throws InterruptedException {
	GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest()
		.withStreamName(shard.getStreamName())
		.withShardId(shard.getShard().getShardId())
		.withShardIteratorType(shardIteratorType);

	switch (ShardIteratorType.fromValue(shardIteratorType)) {
		case TRIM_HORIZON:
		case LATEST:
			break;
		case AT_TIMESTAMP:
			if (startingMarker instanceof Date) {
				getShardIteratorRequest.setTimestamp((Date) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_TIMESTAMP. Must be a Date object.");
			}
			break;
		case AT_SEQUENCE_NUMBER:
		case AFTER_SEQUENCE_NUMBER:
			if (startingMarker instanceof String) {
				getShardIteratorRequest.setStartingSequenceNumber((String) startingMarker);
			} else {
				throw new IllegalArgumentException("Invalid object given for GetShardIteratorRequest() when ShardIteratorType is AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER. Must be a String.");
			}
	}
	return getShardIterator(getShardIteratorRequest);
}