Java Code Examples for com.amazonaws.services.kinesis.model.DescribeStreamResult

The following examples show how to use com.amazonaws.services.kinesis.model.DescribeStreamResult. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: DynamoDBStreamsProxy.java    License: Apache License 2.0 6 votes vote down vote up
private List<StreamShardHandle> getShardsOfStream(
		String streamName,
		@Nullable String lastSeenShardId)
		throws InterruptedException {
	List<StreamShardHandle> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);
		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new StreamShardHandle(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 
Example 2
Source Project: flink   Source File: DynamoDBStreamsProxy.java    License: Apache License 2.0 6 votes vote down vote up
private List<StreamShardHandle> getShardsOfStream(
		String streamName,
		@Nullable String lastSeenShardId)
		throws InterruptedException {
	List<StreamShardHandle> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);
		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new StreamShardHandle(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 
Example 3
@Test
void testProvisionProducerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(new KinesisProducerProperties());
	String name = "test-stream";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ProducerDestination destination = provisioner.provisionProducerDestination(name,
			extendedProducerProperties);

	verify(amazonKinesisMock).describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 
Example 4
Source Project: beam   Source File: SimplifiedKinesisClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void shouldListAllShards() throws Exception {
  Shard shard1 = new Shard().withShardId(SHARD_1);
  Shard shard2 = new Shard().withShardId(SHARD_2);
  Shard shard3 = new Shard().withShardId(SHARD_3);
  when(kinesis.describeStream(STREAM, null))
      .thenReturn(
          new DescribeStreamResult()
              .withStreamDescription(
                  new StreamDescription().withShards(shard1, shard2).withHasMoreShards(true)));
  when(kinesis.describeStream(STREAM, SHARD_2))
      .thenReturn(
          new DescribeStreamResult()
              .withStreamDescription(
                  new StreamDescription().withShards(shard3).withHasMoreShards(false)));

  List<Shard> shards = underTest.listShards(STREAM);

  assertThat(shards).containsOnly(shard1, shard2, shard3);
}
 
Example 5
Source Project: flink   Source File: DynamoDBStreamsProxy.java    License: Apache License 2.0 6 votes vote down vote up
private List<StreamShardHandle> getShardsOfStream(
		String streamName,
		@Nullable String lastSeenShardId)
		throws InterruptedException {
	List<StreamShardHandle> shardsOfStream = new ArrayList<>();

	DescribeStreamResult describeStreamResult;
	do {
		describeStreamResult = describeStream(streamName, lastSeenShardId);
		List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
		for (Shard shard : shards) {
			shardsOfStream.add(new StreamShardHandle(streamName, shard));
		}

		if (shards.size() != 0) {
			lastSeenShardId = shards.get(shards.size() - 1).getShardId();
		}
	} while (describeStreamResult.getStreamDescription().isHasMoreShards());

	return shardsOfStream;
}
 
Example 6
Source Project: presto   Source File: KinesisSplitManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 * <p>
 * Gets from the internal cache unless not yet created or too old.
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription internalStreamDescription = this.streamMap.get(streamName);
    if (internalStreamDescription == null || System.currentTimeMillis() - internalStreamDescription.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        internalStreamDescription = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            internalStreamDescription.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        }
        while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, internalStreamDescription);
    }

    return internalStreamDescription;
}
 
Example 7
Source Project: presto   Source File: MockKinesisClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest)
        throws AmazonClientException
{
    InternalStream theStream = this.getStream(describeStreamRequest.getStreamName());
    if (theStream == null) {
        throw new AmazonClientException("This stream does not exist!");
    }

    StreamDescription desc = new StreamDescription();
    desc = desc.withStreamName(theStream.getStreamName()).withStreamStatus(theStream.getStreamStatus()).withStreamARN(theStream.getStreamAmazonResourceName());

    if (describeStreamRequest.getExclusiveStartShardId() == null || describeStreamRequest.getExclusiveStartShardId().isEmpty()) {
        desc.setShards(this.getShards(theStream));
        desc.setHasMoreShards(false);
    }
    else {
        // Filter from given shard Id, or may not have any more
        String startId = describeStreamRequest.getExclusiveStartShardId();
        desc.setShards(this.getShards(theStream, startId));
        desc.setHasMoreShards(false);
    }

    DescribeStreamResult result = new DescribeStreamResult();
    result = result.withStreamDescription(desc);
    return result;
}
 
Example 8
Source Project: Flink-CEPplus   Source File: KinesisProxy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 
Example 9
Source Project: flink   Source File: KinesisProxy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 
Example 10
@Test
void testProvisionConsumerSuccessfulWithExistingStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeStreamResult);

	ConsumerDestination destination = provisioner.provisionConsumerDestination(name,
			group, extendedConsumerProperties);

	verify(amazonKinesisMock).describeStream(any(DescribeStreamRequest.class));

	assertThat(destination.getName()).isEqualTo(name);
}
 
Example 11
@Test
void testProvisionConsumerExistingStreamUpdateShards() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor = ArgumentCaptor
			.forClass(UpdateShardCountRequest.class);
	String name = "test-stream";
	String group = "test-group";
	int targetShardCount = 2;
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setMinShardCount(targetShardCount);
	binderProperties.setAutoAddShards(true);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());

	DescribeStreamResult describeOriginalStream = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	DescribeStreamResult describeUpdatedStream = describeStreamResultWithShards(
			Arrays.asList(new Shard(), new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeOriginalStream).thenReturn(describeUpdatedStream);

	provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(1)).updateShardCount(updateShardCaptor.capture());

	assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
	assertThat(updateShardCaptor.getValue().getScalingType())
			.isEqualTo(ScalingType.UNIFORM_SCALING.name());
	assertThat(updateShardCaptor.getValue().getTargetShardCount())
			.isEqualTo(targetShardCount);
}
 
Example 12
@Test
void testProvisionProducerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	ExtendedProducerProperties<KinesisProducerProperties> extendedProducerProperties =
			new ExtendedProducerProperties<>(
			new KinesisProducerProperties());

	String name = "test-stream";
	Integer shards = 1;

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, shards))
			.thenReturn(new CreateStreamResult());

	ProducerDestination destination = provisioner.provisionProducerDestination(name,
			extendedProducerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock).createStream(name, shards);

	assertThat(destination.getName()).isEqualTo(name);
}
 
Example 13
@Test
void testProvisionProducerUpdateShards() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	ArgumentCaptor<UpdateShardCountRequest> updateShardCaptor = ArgumentCaptor
			.forClass(UpdateShardCountRequest.class);
	String name = "test-stream";
	String group = "test-group";
	int targetShardCount = 2;
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setMinShardCount(targetShardCount);
	binderProperties.setAutoAddShards(true);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());

	DescribeStreamResult describeOriginalStream = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	DescribeStreamResult describeUpdatedStream = describeStreamResultWithShards(
			Arrays.asList(new Shard(), new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenReturn(describeOriginalStream).thenReturn(describeUpdatedStream);

	provisioner.provisionConsumerDestination(name, group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(1)).updateShardCount(updateShardCaptor.capture());
	assertThat(updateShardCaptor.getValue().getStreamName()).isEqualTo(name);
	assertThat(updateShardCaptor.getValue().getScalingType())
			.isEqualTo(ScalingType.UNIFORM_SCALING.name());
	assertThat(updateShardCaptor.getValue().getTargetShardCount())
			.isEqualTo(targetShardCount);
}
 
Example 14
@Test
void testProvisionConsumerSuccessfulWithNewStream() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	DescribeStreamResult describeStreamResult = describeStreamResultWithShards(
			Collections.singletonList(new Shard()));

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("I got nothing"))
			.thenReturn(describeStreamResult);

	when(amazonKinesisMock.createStream(name, instanceCount * concurrency))
			.thenReturn(new CreateStreamResult());

	ConsumerDestination destination = provisioner.provisionConsumerDestination(name,
			group, extendedConsumerProperties);

	verify(amazonKinesisMock, times(2))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock).createStream(name, instanceCount * concurrency);

	assertThat(destination.getName()).isEqualTo(name);
}
 
Example 15
@Test
public void testAutoCreateStreamForNonExistingStream() throws Exception {
	KinesisTestBinder binder = getBinder();
	DirectChannel output = createBindableChannel("output", new BindingProperties());
	ExtendedConsumerProperties<KinesisConsumerProperties> consumerProperties = createConsumerProperties();
	Date testDate = new Date();
	consumerProperties.getExtension().setShardIteratorType(
			ShardIteratorType.AT_TIMESTAMP.name() + ":" + testDate.getTime());
	String testStreamName = "nonexisting" + System.currentTimeMillis();
	Binding<?> binding = binder.bindConsumer(testStreamName, "test", output,
			consumerProperties);
	binding.unbind();

	DescribeStreamResult streamResult = AMAZON_KINESIS.describeStream(testStreamName);
	String createdStreamName = streamResult.getStreamDescription().getStreamName();
	int createdShards = streamResult.getStreamDescription().getShards().size();
	String createdStreamStatus = streamResult.getStreamDescription()
			.getStreamStatus();

	assertThat(createdStreamName).isEqualTo(testStreamName);
	assertThat(createdShards).isEqualTo(consumerProperties.getInstanceCount()
			* consumerProperties.getConcurrency());
	assertThat(createdStreamStatus).isEqualTo(StreamStatus.ACTIVE.toString());

	KinesisShardOffset shardOffset = TestUtils.getPropertyValue(binding,
			"lifecycle.streamInitialSequence", KinesisShardOffset.class);
	assertThat(shardOffset.getIteratorType())
			.isEqualTo(ShardIteratorType.AT_TIMESTAMP);
	assertThat(shardOffset.getTimestamp()).isEqualTo(testDate);
}
 
Example 16
private List<Shard> describeStream(String stream) {
	String exclusiveStartShardId = null;

	DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest()
			.withStreamName(stream);

	List<Shard> shardList = new ArrayList<>();

	while (true) {
		DescribeStreamResult describeStreamResult;

		describeStreamRequest.withExclusiveStartShardId(exclusiveStartShardId);
		describeStreamResult = AMAZON_KINESIS.describeStream(describeStreamRequest);
		StreamDescription streamDescription = describeStreamResult
				.getStreamDescription();
		if (StreamStatus.ACTIVE.toString()
				.equals(streamDescription.getStreamStatus())) {
			shardList.addAll(streamDescription.getShards());

			if (streamDescription.getHasMoreShards()) {
				exclusiveStartShardId = shardList.get(shardList.size() - 1)
						.getShardId();
				continue;
			}
			else {
				return shardList;
			}
		}
		try {
			Thread.sleep(100);
		}
		catch (InterruptedException ex) {
			Thread.currentThread().interrupt();
			throw new IllegalStateException(ex);
		}
	}
}
 
Example 17
Source Project: beam   Source File: KinesisMockWriteTest.java    License: Apache License 2.0 5 votes vote down vote up
private AmazonKinesis getMockedAmazonKinesisClient() {
  int statusCode = isExistingStream ? 200 : 404;
  SdkHttpMetadata httpMetadata = mock(SdkHttpMetadata.class);
  when(httpMetadata.getHttpStatusCode()).thenReturn(statusCode);

  DescribeStreamResult streamResult = mock(DescribeStreamResult.class);
  when(streamResult.getSdkHttpMetadata()).thenReturn(httpMetadata);

  AmazonKinesis client = mock(AmazonKinesis.class);
  when(client.describeStream(any(String.class))).thenReturn(streamResult);

  return client;
}
 
Example 18
Source Project: beam   Source File: AmazonKinesisMock.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId) {
  if (rateLimitDescribeStream-- > 0) {
    throw new LimitExceededException("DescribeStream rate limit exceeded");
  }
  int nextShardId = 0;
  if (exclusiveStartShardId != null) {
    nextShardId = parseInt(exclusiveStartShardId) + 1;
  }
  boolean hasMoreShards = nextShardId + 1 < shardedData.size();

  List<Shard> shards = new ArrayList<>();
  if (nextShardId < shardedData.size()) {
    shards.add(new Shard().withShardId(Integer.toString(nextShardId)));
  }

  HttpResponse response = new HttpResponse(null, null);
  response.setStatusCode(200);
  DescribeStreamResult result = new DescribeStreamResult();
  result.setSdkHttpMetadata(SdkHttpMetadata.from(response));
  result.withStreamDescription(
      new StreamDescription()
          .withHasMoreShards(hasMoreShards)
          .withShards(shards)
          .withStreamName(streamName));
  return result;
}
 
Example 19
Source Project: datacollector   Source File: KinesisSourceIT.java    License: Apache License 2.0 5 votes vote down vote up
private static boolean streamActive(AmazonKinesis client, String streamName) {
  try {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    DescribeStreamResult describeStreamResult = client.describeStream(describeStreamRequest);
    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if ("ACTIVE".equals(streamStatus)) {
      return true;
    }
  } catch (Exception e) {
    return false;
  }
  return false;
}
 
Example 20
Source Project: presto-kinesis   Source File: KinesisSplitManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Internal method to retrieve the stream description and get the shards from AWS.
 *
 * Gets from the internal cache unless not yet created or too old.
 *
 * @param streamName Kinesis stream name
 * @return Returns kinesis stream description
 */
protected InternalStreamDescription getStreamDescription(String streamName)
{
    InternalStreamDescription desc = this.streamMap.get(streamName);
    if (desc == null || System.currentTimeMillis() - desc.getCreateTimeStamp() >= MAX_CACHE_AGE_MILLIS) {
        desc = new InternalStreamDescription(streamName);

        DescribeStreamRequest describeStreamRequest = clientManager.getDescribeStreamRequest();
        describeStreamRequest.setStreamName(streamName);

        // Collect shards from Kinesis
        String exclusiveStartShardId = null;
        List<Shard> shards = new ArrayList<>();
        do {
            describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
            DescribeStreamResult describeStreamResult = clientManager.getClient().describeStream(describeStreamRequest);

            String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
            if (!streamStatus.equals("ACTIVE") && !streamStatus.equals("UPDATING")) {
                throw new ResourceNotFoundException("Stream not Active");
            }

            desc.addAllShards(describeStreamResult.getStreamDescription().getShards());

            if (describeStreamResult.getStreamDescription().getHasMoreShards() && (shards.size() > 0)) {
                exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
            }
            else {
                exclusiveStartShardId = null;
            }
        } while (exclusiveStartShardId != null);

        this.streamMap.put(streamName, desc);
    }

    return desc;
}
 
Example 21
Source Project: presto-kinesis   Source File: MockKinesisClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) throws AmazonServiceException, AmazonClientException
{
    InternalStream theStream = this.getStream(describeStreamRequest.getStreamName());
    if (theStream != null) {
        StreamDescription desc = new StreamDescription();
        desc = desc.withStreamName(theStream.getStreamName()).withStreamStatus(theStream.getStreamStatus()).withStreamARN(theStream.getStreamARN());

        if (describeStreamRequest.getExclusiveStartShardId() == null || describeStreamRequest.getExclusiveStartShardId().isEmpty()) {
            desc.setShards(this.getShards(theStream));
            desc.setHasMoreShards(false);
        }
        else {
            // Filter from given shard Id, or may not have any more
            String startId = describeStreamRequest.getExclusiveStartShardId();
            desc.setShards(this.getShards(theStream, startId));
            desc.setHasMoreShards(false);
        }

        DescribeStreamResult result = new DescribeStreamResult();
        result = result.withStreamDescription(desc);
        return result;
    }
    else {
        throw new AmazonClientException("This stream does not exist!");
    }
}
 
Example 22
Source Project: flink   Source File: KinesisProxy.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this
 * Kinesis stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article,
 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">
 *   "Exponential Backoff and Jitter"</a>.
 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher.
 * This jitter backoff approach will help distribute calls across the fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 *
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId)
		throws InterruptedException {
	final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
	describeStreamRequest.setStreamName(streamName);
	describeStreamRequest.setExclusiveStartShardId(startShardId);

	DescribeStreamResult describeStreamResult = null;

	// Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
	int attemptCount = 0;
	while (describeStreamResult == null) { // retry until we get a result
		try {
			describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
		} catch (LimitExceededException le) {
			long backoffMillis = fullJitterBackoff(
					describeStreamBaseBackoffMillis,
					describeStreamMaxBackoffMillis,
					describeStreamExpConstant,
					attemptCount++);
			LOG.warn(String.format("Got LimitExceededException when describing stream %s. "
					+ "Backing off for %d millis.", streamName, backoffMillis));
			Thread.sleep(backoffMillis);
		} catch (ResourceNotFoundException re) {
			throw new RuntimeException("Error while getting stream details", re);
		}
	}

	String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
	if (!(streamStatus.equals(StreamStatus.ACTIVE.toString())
			|| streamStatus.equals(StreamStatus.UPDATING.toString()))) {
		if (LOG.isWarnEnabled()) {
			LOG.warn(String.format("The status of stream %s is %s ; result of the current "
							+ "describeStream operation will not contain any shard information.",
					streamName, streamStatus));
		}
	}

	return describeStreamResult;
}
 
Example 23
Source Project: presto   Source File: MockKinesisClient.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DescribeStreamResult describeStream(String streamName)
        throws AmazonServiceException, AmazonClientException
{
    return null;
}
 
Example 24
Source Project: presto   Source File: MockKinesisClient.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DescribeStreamResult describeStream(String streamName, String exclusiveStartShardId)
        throws AmazonServiceException, AmazonClientException
{
    return null;
}
 
Example 25
Source Project: presto   Source File: MockKinesisClient.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DescribeStreamResult describeStream(String streamName, Integer integer, String exclusiveStartShardId)
        throws AmazonServiceException, AmazonClientException
{
    return null;
}
 
Example 26
Source Project: Flink-CEPplus   Source File: ManualExactlyOnceTest.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	LOG.info("Starting exactly once test");

	final String streamName = "flink-test-" + UUID.randomUUID().toString();
	final String accessKey = pt.getRequired("accessKey");
	final String secretKey = pt.getRequired("secretKey");
	final String region = pt.getRequired("region");

	Properties configProps = new Properties();
	configProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
	configProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
	configProps.setProperty(AWSConfigConstants.AWS_REGION, region);
	AmazonKinesis client = AWSUtil.createKinesisClient(configProps);

	// create a stream for the test:
	client.createStream(streamName, 1);

	// wait until stream has been created
	DescribeStreamResult status = client.describeStream(streamName);
	LOG.info("status {}" , status);
	while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
		status = client.describeStream(streamName);
		LOG.info("Status of stream {}", status);
		Thread.sleep(1000);
	}

	final Configuration flinkConfig = new Configuration();
	flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m");
	flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");

	MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberTaskManagers(1)
		.setNumberSlotsPerTaskManager(8)
		.setConfiguration(flinkConfig)
		.build());
	flink.before();

	final int flinkPort = flink.getRestAddres().getPort();

	try {
		final AtomicReference<Throwable> producerError = new AtomicReference<>();
		Thread producerThread = KinesisEventsGeneratorProducerThread.create(
			TOTAL_EVENT_COUNT, 2,
			accessKey, secretKey, region, streamName,
			producerError, flinkPort, flinkConfig);
		producerThread.start();

		final AtomicReference<Throwable> consumerError = new AtomicReference<>();
		Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(
			TOTAL_EVENT_COUNT, 200, 2, 500, 500,
			accessKey, secretKey, region, streamName,
			consumerError, flinkPort, flinkConfig);
		consumerThread.start();

		boolean deadlinePassed = false;
		long deadline = System.currentTimeMillis() + (1000 * 2 * 60); // wait at most for two minutes
		// wait until both producer and consumer finishes, or an unexpected error is thrown
		while ((consumerThread.isAlive() || producerThread.isAlive()) &&
			(producerError.get() == null && consumerError.get() == null)) {
			Thread.sleep(1000);
			if (System.currentTimeMillis() >= deadline) {
				LOG.warn("Deadline passed");
				deadlinePassed = true;
				break; // enough waiting
			}
		}

		if (producerThread.isAlive()) {
			producerThread.interrupt();
		}

		if (consumerThread.isAlive()) {
			consumerThread.interrupt();
		}

		if (producerError.get() != null) {
			LOG.info("+++ TEST failed! +++");
			throw new RuntimeException("Producer failed", producerError.get());
		}
		if (consumerError.get() != null) {
			LOG.info("+++ TEST failed! +++");
			throw new RuntimeException("Consumer failed", consumerError.get());
		}

		if (!deadlinePassed) {
			LOG.info("+++ TEST passed! +++");
		} else {
			LOG.info("+++ TEST failed! +++");
		}

	} finally {
		client.deleteStream(streamName);
		client.shutdown();

		// stopping flink
		flink.after();
	}
}
 
Example 27
Source Project: flink   Source File: ManualExactlyOnceTest.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	LOG.info("Starting exactly once test");

	final String streamName = "flink-test-" + UUID.randomUUID().toString();
	final String accessKey = pt.getRequired("accessKey");
	final String secretKey = pt.getRequired("secretKey");
	final String region = pt.getRequired("region");

	Properties configProps = new Properties();
	configProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
	configProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
	configProps.setProperty(AWSConfigConstants.AWS_REGION, region);
	AmazonKinesis client = AWSUtil.createKinesisClient(configProps);

	// create a stream for the test:
	client.createStream(streamName, 1);

	// wait until stream has been created
	DescribeStreamResult status = client.describeStream(streamName);
	LOG.info("status {}" , status);
	while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
		status = client.describeStream(streamName);
		LOG.info("Status of stream {}", status);
		Thread.sleep(1000);
	}

	final Configuration flinkConfig = new Configuration();
	flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m");
	flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");

	MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberTaskManagers(1)
		.setNumberSlotsPerTaskManager(8)
		.setConfiguration(flinkConfig)
		.build());
	flink.before();

	final int flinkPort = flink.getRestAddres().getPort();

	try {
		final AtomicReference<Throwable> producerError = new AtomicReference<>();
		Thread producerThread = KinesisEventsGeneratorProducerThread.create(
			TOTAL_EVENT_COUNT, 2,
			accessKey, secretKey, region, streamName,
			producerError, flinkPort, flinkConfig);
		producerThread.start();

		final AtomicReference<Throwable> consumerError = new AtomicReference<>();
		Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(
			TOTAL_EVENT_COUNT, 200, 2, 500, 500,
			accessKey, secretKey, region, streamName,
			consumerError, flinkPort, flinkConfig);
		consumerThread.start();

		boolean deadlinePassed = false;
		long deadline = System.currentTimeMillis() + (1000 * 2 * 60); // wait at most for two minutes
		// wait until both producer and consumer finishes, or an unexpected error is thrown
		while ((consumerThread.isAlive() || producerThread.isAlive()) &&
			(producerError.get() == null && consumerError.get() == null)) {
			Thread.sleep(1000);
			if (System.currentTimeMillis() >= deadline) {
				LOG.warn("Deadline passed");
				deadlinePassed = true;
				break; // enough waiting
			}
		}

		if (producerThread.isAlive()) {
			producerThread.interrupt();
		}

		if (consumerThread.isAlive()) {
			consumerThread.interrupt();
		}

		if (producerError.get() != null) {
			LOG.info("+++ TEST failed! +++");
			throw new RuntimeException("Producer failed", producerError.get());
		}
		if (consumerError.get() != null) {
			LOG.info("+++ TEST failed! +++");
			throw new RuntimeException("Consumer failed", consumerError.get());
		}

		if (!deadlinePassed) {
			LOG.info("+++ TEST passed! +++");
		} else {
			LOG.info("+++ TEST failed! +++");
		}

	} finally {
		client.deleteStream(streamName);
		client.shutdown();

		// stopping flink
		flink.after();
	}
}
 
Example 28
@Override
public final Response addRecords(final Request request, final LambdaLogger logger) {

    this.documentAddedCount = request.getCount();

    DescribeStreamResult result = this.kinesisClient.describeStream(request.getStreamName());
    logger.log("Stream Status: " + result.getStreamDescription().getStreamStatus() + ". ");

    logger.log("Adding records to Stream...");

    String payload;

    for (int i = 1; i <= request.getCount(); i++) {

        payload = request.getPayload() + i;

        this.kinesisBatch.add(new PutRecordsRequestEntry()
                .withPartitionKey(request.getPartitionKey())
                .withData(ByteBuffer.wrap(payload.getBytes())));

        if (this.kinesisBatch.size() >= request.getBatchSize()) {

            try {
                logger.log("Flushing records to Stream...");
                flushBatch(request.getStreamName(), logger);
            } catch (Exception e) {
                logger.log("Exception occurred: " + e);
                this.isError = false;
            } finally {
                this.kinesisBatch.clear();
            }
        }

    }

    if (this.isError) {
        return new Response(ERROR_MESSAGE, documentAddedCount);
    } else {
        return new Response(SUCCESS_MESSAGE, documentAddedCount);
    }
}
 
Example 29
private static DescribeStreamResult describeStreamResultWithShards(
		List<Shard> shards) {
	return new DescribeStreamResult().withStreamDescription(new StreamDescription()
			.withShards(shards).withStreamStatus(StreamStatus.ACTIVE)
			.withHasMoreShards(Boolean.FALSE));
}
 
Example 30
Source Project: beam   Source File: AmazonKinesisMock.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DescribeStreamResult describeStream(DescribeStreamRequest describeStreamRequest) {
  throw new RuntimeException("Not implemented");
}