org.springframework.cloud.stream.provisioning.ProvisioningException Java Examples

The following examples show how to use org.springframework.cloud.stream.provisioning.ProvisioningException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KinesisStreamProvisioner.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 6 votes vote down vote up
@Override
public ProducerDestination provisionProducerDestination(String name,
		ExtendedProducerProperties<KinesisProducerProperties> properties)
		throws ProvisioningException {

	if (logger.isInfoEnabled()) {
		logger.info("Using Kinesis stream for outbound: " + name);
	}

	if (properties.getHeaderMode() == null) {
		properties.setHeaderMode(HeaderMode.embeddedHeaders);
	}

	return new KinesisProducerDestination(name,
			createOrUpdate(name, properties.getPartitionCount()));
}
 
Example #2
Source File: KinesisStreamProvisioner.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 6 votes vote down vote up
@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
		ExtendedConsumerProperties<KinesisConsumerProperties> properties)
		throws ProvisioningException {

	if (properties.getExtension().isDynamoDbStreams()) {
		if (logger.isInfoEnabled()) {
			logger.info("Using DynamoDB table in DynamoDB Streams support for inbound: " + name);
		}
		return new KinesisConsumerDestination(name, Collections.emptyList());
	}

	if (logger.isInfoEnabled()) {
		logger.info("Using Kinesis stream for inbound: " + name);
	}

	if (properties.getHeaderMode() == null) {
		properties.setHeaderMode(HeaderMode.embeddedHeaders);
	}

	int shardCount = properties.getInstanceCount() * properties.getConcurrency();

	return new KinesisConsumerDestination(name, createOrUpdate(name, shardCount));
}
 
Example #3
Source File: RabbitExchangeQueueProvisioner.java    From spring-cloud-stream-binder-rabbit with Apache License 2.0 6 votes vote down vote up
private Exchange buildExchange(RabbitCommonProperties properties,
		String exchangeName) {
	try {
		ExchangeBuilder builder = new ExchangeBuilder(exchangeName,
				properties.getExchangeType());
		builder.durable(properties.isExchangeDurable());
		if (properties.isExchangeAutoDelete()) {
			builder.autoDelete();
		}
		if (properties.isDelayedExchange()) {
			builder.delayed();
		}
		return builder.build();
	}
	catch (Exception e) {
		throw new ProvisioningException("Failed to create exchange object", e);
	}
}
 
Example #4
Source File: KafkaTopicProvisioner.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
private void createTopic(AdminClient adminClient, String name, int partitionCount,
		boolean tolerateLowerPartitionsOnBroker, KafkaTopicProperties properties) {
	try {
		createTopicIfNecessary(adminClient, name, partitionCount,
				tolerateLowerPartitionsOnBroker, properties);
	}
	// TODO: Remove catching Throwable. See this thread:
	// TODO:
	// https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940
	catch (Throwable throwable) {
		if (throwable instanceof Error) {
			throw (Error) throwable;
		}
		else {
			// TODO:
			// https://github.com/spring-cloud/spring-cloud-stream-binder-kafka/pull/514#discussion_r241075940
			throw new ProvisioningException("Provisioning exception", throwable);
		}
	}
}
 
Example #5
Source File: PubSubChannelProvisioner.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
private Topic makeSureTopicExists(String topicName, boolean autoCreate) {
	Topic topic = this.pubSubAdmin.getTopic(topicName);
	if (topic == null) {
		if (autoCreate) {
			try {
				topic = this.pubSubAdmin.createTopic(topicName);
			}
			catch (AlreadyExistsException alreadyExistsException) {
				// Ignore concurrent topic creation - we're good as long as topic was created and exists
				LOGGER.info("Failed to auto-create topic '" + topicName + "' because it already exists.");
			}
		}
		else {
			throw new ProvisioningException("Non-existing '" + topicName + "' topic.");
		}
	}

	return topic;
}
 
Example #6
Source File: PubSubChannelProvisionerTests.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Test
public void testProvisionConsumerDestination_noTopicException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Non-existing 'topic_A' topic.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);
	when(this.pubSubAdminMock.getTopic("topic_A")).thenReturn(null);

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 
Example #7
Source File: TestChannelBinderProvisioner.java    From spring-cloud-stream with Apache License 2.0 5 votes vote down vote up
/**
 * Will provision consumer destination as SI {@link DirectChannel}.
 */
@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
		ConsumerProperties properties) throws ProvisioningException {
	SubscribableChannel destination = this.provisionDestination(name, false);
	if (this.source != null) {
		this.source.setChannel(destination);
	}
	return new SpringIntegrationConsumerDestination(name, destination);
}
 
Example #8
Source File: TestChannelBinderProvisioner.java    From spring-cloud-stream with Apache License 2.0 5 votes vote down vote up
/**
 * Will provision producer destination as an SI {@link PublishSubscribeChannel}. <br>
 * This provides convenience of registering additional subscriber (handler in the test
 * method) along side of being able to call {@link OutputDestination#receive()} to get
 * a {@link Message} for additional assertions.
 */
@Override
public ProducerDestination provisionProducerDestination(String name,
		ProducerProperties properties) throws ProvisioningException {
	SubscribableChannel destination = this.provisionDestination(name, true);
	this.target.setChannel(destination);
	return new SpringIntegrationProducerDestination(name, destination);
}
 
Example #9
Source File: KafkaBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testAutoAddPartitionsDisabledFailsIfTopicUnderPartitionedAndAutoRebalanceDisabled()
		throws Throwable {
	KafkaBinderConfigurationProperties configurationProperties = createConfigurationProperties();

	String testTopicName = "existing" + System.currentTimeMillis();
	invokeCreateTopic(testTopicName, 1, 1);
	configurationProperties.setAutoAddPartitions(false);
	Binder binder = getBinder(configurationProperties);
	GenericApplicationContext context = new GenericApplicationContext();
	context.refresh();

	ExtendedConsumerProperties<KafkaConsumerProperties> consumerProperties = createConsumerProperties();
	DirectChannel output = createBindableChannel("output",
			createConsumerBindingProperties(consumerProperties));
	// this consumer must consume from partition 2
	consumerProperties.setInstanceCount(3);
	consumerProperties.setInstanceIndex(2);
	consumerProperties.getExtension().setAutoRebalanceEnabled(false);
	expectedProvisioningException.expect(ProvisioningException.class);
	expectedProvisioningException.expectMessage(
			"The number of expected partitions was: 3, but 1 has been found instead");
	Binding binding = binder.bindConsumer(testTopicName, "test", output,
			consumerProperties);
	if (binding != null) {
		binding.unbind();
	}
}
 
Example #10
Source File: KafkaTopicProvisioner.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private ConsumerDestination createDlqIfNeedBe(AdminClient adminClient, String name,
		String group, ExtendedConsumerProperties<KafkaConsumerProperties> properties,
		boolean anonymous, int partitions) {

	if (properties.getExtension().isEnableDlq() && !anonymous) {
		String dlqTopic = StringUtils.hasText(properties.getExtension().getDlqName())
				? properties.getExtension().getDlqName()
				: "error." + name + "." + group;
		int dlqPartitions = properties.getExtension().getDlqPartitions() == null
				? partitions
				: properties.getExtension().getDlqPartitions();
		try {
			createTopicAndPartitions(adminClient, dlqTopic, dlqPartitions,
					properties.getExtension().isAutoRebalanceEnabled(),
					properties.getExtension().getTopic());
		}
		catch (Throwable throwable) {
			if (throwable instanceof Error) {
				throw (Error) throwable;
			}
			else {
				throw new ProvisioningException("provisioning exception", throwable);
			}
		}
		return new KafkaConsumerDestination(name, partitions, dlqTopic);
	}
	return null;
}
 
Example #11
Source File: KafkaTopicProvisioner.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Override
public ProducerDestination provisionProducerDestination(final String name,
		ExtendedProducerProperties<KafkaProducerProperties> properties) {

	if (logger.isInfoEnabled()) {
		logger.info("Using kafka topic for outbound: " + name);
	}
	KafkaTopicUtils.validateTopicName(name);
	try (AdminClient adminClient = AdminClient.create(this.adminClientProperties)) {
		createTopic(adminClient, name, properties.getPartitionCount(), false,
				properties.getExtension().getTopic());
		int partitions = 0;
		Map<String, TopicDescription> topicDescriptions = new HashMap<>();
		if (this.configurationProperties.isAutoCreateTopics()) {
			this.metadataRetryOperations.execute(context -> {
				try {
					if (logger.isDebugEnabled()) {
						logger.debug("Attempting to retrieve the description for the topic: " + name);
					}
					DescribeTopicsResult describeTopicsResult = adminClient
							.describeTopics(Collections.singletonList(name));
					KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult
							.all();
					topicDescriptions.putAll(all.get(this.operationTimeout, TimeUnit.SECONDS));
				}
				catch (Exception ex) {
					throw new ProvisioningException("Problems encountered with partitions finding", ex);
				}
				return null;
			});
		}
		TopicDescription topicDescription = topicDescriptions.get(name);
		if (topicDescription != null) {
			partitions = topicDescription.partitions().size();
		}
		return new KafkaProducerDestination(name, partitions);
	}
}
 
Example #12
Source File: PubSubChannelProvisionerTests.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Test
public void testProvisionConsumerDestination_wrongTopicException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Existing 'topic_A.group_A' subscription is for a different topic 'topic_B'.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);
	when(this.pubSubAdminMock.getSubscription("topic_A.group_A")).thenReturn(Subscription.newBuilder().setTopic("topic_B").build());

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 
Example #13
Source File: PubSubChannelProvisionerTests.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Test
public void testProvisionConsumerDestination_noSubscriptionException() {
	this.expectedEx.expect(ProvisioningException.class);
	this.expectedEx.expectMessage("Non-existing 'topic_A.group_A' subscription.");

	when(this.pubSubConsumerProperties.isAutoCreateResources()).thenReturn(false);

	PubSubConsumerDestination result = (PubSubConsumerDestination) this.pubSubChannelProvisioner
			.provisionConsumerDestination("topic_A", "group_A", this.properties);
}
 
Example #14
Source File: RocketMQTopicProvisioner.java    From spring-cloud-alibaba with Apache License 2.0 5 votes vote down vote up
@Override
public ProducerDestination provisionProducerDestination(String name,
		ExtendedProducerProperties<RocketMQProducerProperties> properties)
		throws ProvisioningException {
	checkTopic(name);
	return new RocketProducerDestination(name);
}
 
Example #15
Source File: PubSubChannelProvisioner.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Override
public ConsumerDestination provisionConsumerDestination(String topicName, String group,
		ExtendedConsumerProperties<PubSubConsumerProperties> properties)
		throws ProvisioningException {

	Topic topic = makeSureTopicExists(topicName, properties.getExtension().isAutoCreateResources());

	String subscriptionName;
	Subscription subscription;
	if (StringUtils.hasText(group)) {
		// Use <topicName>.<group> as subscription name
		subscriptionName = topicName + "." + group;
		subscription = this.pubSubAdmin.getSubscription(subscriptionName);
	}
	else {
		// Generate anonymous random group since one wasn't provided
		subscriptionName = "anonymous." + topicName + "." + UUID.randomUUID().toString();
		subscription = this.pubSubAdmin.createSubscription(subscriptionName, topicName);
		this.anonymousGroupSubscriptionNames.add(subscriptionName);
	}

	// make sure subscription exists
	if (subscription == null) {
		if (properties.getExtension().isAutoCreateResources()) {
			this.pubSubAdmin.createSubscription(subscriptionName, topicName);
		}
		else {
			throw new ProvisioningException("Non-existing '" + subscriptionName + "' subscription.");
		}
	}
	else if (!subscription.getTopic().equals(topic.getName())) {
		throw new ProvisioningException(
				"Existing '" + subscriptionName + "' subscription is for a different topic '"
						+ subscription.getTopic() + "'.");
	}
	return new PubSubConsumerDestination(subscriptionName);
}
 
Example #16
Source File: PubSubChannelProvisioner.java    From spring-cloud-gcp with Apache License 2.0 5 votes vote down vote up
@Override
public ProducerDestination provisionProducerDestination(String topic,
		ExtendedProducerProperties<PubSubProducerProperties> properties)
		throws ProvisioningException {
	makeSureTopicExists(topic, properties.getExtension().isAutoCreateResources());

	return new PubSubProducerDestination(topic);
}
 
Example #17
Source File: KinesisStreamProvisionerTests.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
@Test
void testProvisionConsumerResourceNotFoundException() {
	AmazonKinesis amazonKinesisMock = mock(AmazonKinesis.class);
	KinesisBinderConfigurationProperties binderProperties = new KinesisBinderConfigurationProperties();
	binderProperties.setAutoCreateStream(false);
	KinesisStreamProvisioner provisioner = new KinesisStreamProvisioner(
			amazonKinesisMock, binderProperties);
	int instanceCount = 1;
	int concurrency = 1;

	ExtendedConsumerProperties<KinesisConsumerProperties> extendedConsumerProperties =
			new ExtendedConsumerProperties<>(
			new KinesisConsumerProperties());
	extendedConsumerProperties.setInstanceCount(instanceCount);
	extendedConsumerProperties.setConcurrency(concurrency);

	String name = "test-stream";
	String group = "test-group";

	when(amazonKinesisMock.describeStream(any(DescribeStreamRequest.class)))
			.thenThrow(new ResourceNotFoundException("Stream not found"));

	assertThatThrownBy(() -> provisioner.provisionConsumerDestination(name, group,
			extendedConsumerProperties))
			.isInstanceOf(ProvisioningException.class)
			.hasMessageContaining(
					"The stream [test-stream] was not found and auto creation is disabled.")
			.hasCauseInstanceOf(ResourceNotFoundException.class);

	verify(amazonKinesisMock, times(1))
			.describeStream(any(DescribeStreamRequest.class));

	verify(amazonKinesisMock, never()).createStream(name,
			instanceCount * concurrency);
}
 
Example #18
Source File: KinesisMessageChannelBinder.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
@Override
protected MessageProducer createConsumerEndpoint(ConsumerDestination destination,
		String group,
		ExtendedConsumerProperties<KinesisConsumerProperties> properties) {

	ConsumerDestination destinationToUse = destination;

	if (properties.getExtension().isDynamoDbStreams()) {
		DescribeTableResult describeTableResult = this.dynamoDBClient.describeTable(destinationToUse.getName());
		String latestStreamArn = describeTableResult.getTable().getLatestStreamArn();
		if (StringUtils.hasText(latestStreamArn)) {
			destinationToUse = new KinesisConsumerDestination(latestStreamArn, Collections.emptyList());
		}
		else {
			throw new ProvisioningException("The DynamoDB table ["
					+ destinationToUse.getName()
					+ "] doesn't have Streams enabled.");
		}
	}
	else {
		this.streamsInUse.add(destinationToUse.getName());
	}

	MessageProducer adapter;
	if (this.configurationProperties.isKplKclEnabled()) {
		adapter = createKclConsumerEndpoint(destinationToUse, group, properties);
	}
	else {
		adapter = createKinesisConsumerEndpoint(destinationToUse, group, properties);
	}

	return adapter;
}
 
Example #19
Source File: KinesisStreamProvisioner.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
private List<Shard> createOrUpdate(String stream, int shards) {
	List<Shard> shardList = new ArrayList<>();
	try {
		shardList = getShardList(stream);
	}
	catch (ResourceNotFoundException ex) {
		if (!this.configurationProperties.isAutoCreateStream()) {
			throw new ProvisioningException(
					"The stream [" + stream + "] was not found and auto creation is disabled.", ex);
		}
		if (logger.isInfoEnabled()) {
			logger.info("Stream '" + stream + "' not found. Create one...");
		}

		this.amazonKinesis.createStream(stream, Math.max(this.configurationProperties.getMinShardCount(), shards));

	}

	int effectiveShardCount = Math.max(this.configurationProperties.getMinShardCount(), shards);

	if ((shardList.size() < effectiveShardCount)
			&& this.configurationProperties.isAutoAddShards()) {
		return updateShardCount(stream, shardList.size(), effectiveShardCount);
	}

	return shardList;
}
 
Example #20
Source File: KinesisStreamProvisioner.java    From spring-cloud-stream-binder-aws-kinesis with Apache License 2.0 5 votes vote down vote up
private List<Shard> getShardList(String stream, int retryCount) {
	List<Shard> shardList = new ArrayList<>();

	if (retryCount > configurationProperties.getDescribeStreamRetries()) {
		ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException(
				"The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
		resourceNotFoundException.setServiceName("Kinesis");

		throw new ProvisioningException(
				"Kinesis org.springframework.cloud.stream.binder.kinesis.provisioning error",
				resourceNotFoundException);
	}

	ListShardsRequest listShardsRequest = new ListShardsRequest().withStreamName(stream);

	try {
		ListShardsResult listShardsResult = amazonKinesis.listShards(listShardsRequest);

		shardList.addAll(listShardsResult.getShards());

	}
	catch (LimitExceededException limitExceededException) {
		logger.info("Got LimitExceededException when describing stream [" + stream + "]. " + "Backing off for ["
				+ this.configurationProperties.getDescribeStreamBackoff() + "] millis.");

		try {
			Thread.sleep(this.configurationProperties.getDescribeStreamBackoff());
			getShardList(stream, retryCount++);
		}
		catch (InterruptedException ex) {
			Thread.currentThread().interrupt();
			throw new ProvisioningException(
					"The [describeStream] thread for the stream [" + stream + "] has been interrupted.", ex);
		}
	}

	return shardList;
}
 
Example #21
Source File: RocketMQTopicProvisioner.java    From spring-cloud-alibaba with Apache License 2.0 5 votes vote down vote up
@Override
public ConsumerDestination provisionConsumerDestination(String name, String group,
		ExtendedConsumerProperties<RocketMQConsumerProperties> properties)
		throws ProvisioningException {
	checkTopic(name);
	return new RocketConsumerDestination(name);
}
 
Example #22
Source File: KafkaTopicProvisioner.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 4 votes vote down vote up
private ConsumerDestination doProvisionConsumerDestination(final String name,
		final String group,
		ExtendedConsumerProperties<KafkaConsumerProperties> properties) {

	if (properties.getExtension().isDestinationIsPattern()) {
		Assert.isTrue(!properties.getExtension().isEnableDlq(),
				"enableDLQ is not allowed when listening to topic patterns");
		if (logger.isDebugEnabled()) {
			logger.debug("Listening to a topic pattern - " + name
					+ " - no provisioning performed");
		}
		return new KafkaConsumerDestination(name);
	}
	KafkaTopicUtils.validateTopicName(name);
	boolean anonymous = !StringUtils.hasText(group);
	Assert.isTrue(!anonymous || !properties.getExtension().isEnableDlq(),
			"DLQ support is not available for anonymous subscriptions");
	if (properties.getInstanceCount() == 0) {
		throw new IllegalArgumentException("Instance count cannot be zero");
	}
	int partitionCount = properties.getInstanceCount() * properties.getConcurrency();
	ConsumerDestination consumerDestination = new KafkaConsumerDestination(name);
	try (AdminClient adminClient = createAdminClient()) {
		createTopic(adminClient, name, partitionCount,
				properties.getExtension().isAutoRebalanceEnabled(),
				properties.getExtension().getTopic());
		if (this.configurationProperties.isAutoCreateTopics()) {
			DescribeTopicsResult describeTopicsResult = adminClient
					.describeTopics(Collections.singletonList(name));
			KafkaFuture<Map<String, TopicDescription>> all = describeTopicsResult
					.all();
			try {
				Map<String, TopicDescription> topicDescriptions = all
						.get(this.operationTimeout, TimeUnit.SECONDS);
				TopicDescription topicDescription = topicDescriptions.get(name);
				int partitions = topicDescription.partitions().size();
				consumerDestination = createDlqIfNeedBe(adminClient, name, group,
						properties, anonymous, partitions);
				if (consumerDestination == null) {
					consumerDestination = new KafkaConsumerDestination(name,
							partitions);
				}
			}
			catch (Exception ex) {
				throw new ProvisioningException("provisioning exception", ex);
			}
		}
	}
	return consumerDestination;
}