com.microsoft.azure.eventhubs.EventHubClient Java Examples

The following examples show how to use com.microsoft.azure.eventhubs.EventHubClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AzureIoTService.java    From para with Apache License 2.0 6 votes vote down vote up
/**
 * No-args constructor.
 */
public AzureIoTService() {
	if (!StringUtils.isBlank(SERVICE_ACCESS_KEY)) {
		if (!StringUtils.isBlank(EVENTHUB_ENDPOINT)) {
			final ArrayList<EventHubClient> recievers = new ArrayList<>();
			for (int i = 0; i < PARTITIONS_COUNT; i++) {
				recievers.add(receiveEventsAsync(Integer.toString(i)));
			}
			Para.addDestroyListener(new DestroyListener() {
				public void onDestroy() {
					for (EventHubClient recvr : recievers) {
						recvr.close();
					}
				}
			});
		}
		try {
			registryManager = RegistryManager.createFromConnectionString(SERVICE_CONN_STR);
		} catch (Exception ex) {
			logger.warn("Couldn't initialize Azure registry manager: {}", ex.getMessage());
		}
	}
}
 
Example #2
Source File: PutAzureEventHub.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
protected void sendMessage(final byte[] buffer) throws ProcessException {

        final EventHubClient sender = senderQueue.poll();
        if(null != sender) {
            try {
                sender.sendSync(new EventData(buffer));
            } catch (final ServiceBusException sbe) {
                throw new ProcessException("Caught exception trying to send message to eventbus", sbe);
            } finally {
                senderQueue.offer(sender);
            }
        }else{
            throw new ProcessException("No EventHubClients are configured for sending");
        }

    }
 
Example #3
Source File: PutAzureEventHub.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@OnScheduled
public final void setupClient(final ProcessContext context) throws ProcessException{
    final String policyName = context.getProperty(ACCESS_POLICY).getValue();
    final String policyKey = context.getProperty(POLICY_PRIMARY_KEY).getValue();
    final String namespace = context.getProperty(NAMESPACE).getValue();
    final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue();


    final int numThreads = context.getMaxConcurrentTasks();
    senderQueue = new LinkedBlockingQueue<>(numThreads);
    for (int i = 0; i < numThreads; i++) {
        final EventHubClient client = createEventHubClient(namespace, eventHubName, policyName, policyKey);
        if(null != client) {
            senderQueue.offer(client);
        }
    }
}
 
Example #4
Source File: PutAzureEventHub.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * Prepare at least one Event hub sender based on this instance of processor.
 *
 * @param context of this processor instance from which all connectivity information properties are taken.
 */
protected void populateSenderQueue(ProcessContext context) {
    if(senderQueue.size() == 0){
        final int numThreads = context.getMaxConcurrentTasks();
        senderQueue = new LinkedBlockingQueue<>(numThreads);
        executor = Executors.newScheduledThreadPool(4);
        final boolean useManagedIdentiy = context.getProperty(USE_MANAGED_IDENTITY).asBoolean();
        final String policyName, policyKey;
        if(useManagedIdentiy) {
            policyName = AzureEventHubUtils.MANAGED_IDENTITY_POLICY;
            policyKey =null;
        } else {
            policyName = context.getProperty(ACCESS_POLICY).getValue();
            policyKey = context.getProperty(POLICY_PRIMARY_KEY).getValue();
        }
        final String namespace = context.getProperty(NAMESPACE).getValue();
        final String eventHubName = context.getProperty(EVENT_HUB_NAME).getValue();
        for (int i = 0; i < numThreads; i++) {
            final EventHubClient client = createEventHubClient(namespace, eventHubName, policyName, policyKey, executor);
            if(null != client) {
                senderQueue.offer(client);
            }
        }
    }
}
 
Example #5
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAllAttributesAreLiftedToProperties() {
    MockedEventhubClientMockPutAzureEventHub processor = new PutAzureEventHubTest.MockedEventhubClientMockPutAzureEventHub();
    MockitoAnnotations.initMocks(processor);

    EventHubClient eventHubClient = processor.getEventHubClient();
    when(eventHubClient.send(any(EventData.class)))
    .thenReturn(CompletableFuture.completedFuture(null));

    testRunner = TestRunners.newTestRunner(processor);
    setUpStandardTestConfig();

    MockFlowFile flowFile = new MockFlowFile(1234);
    ImmutableMap<String, String> demoAttributes = ImmutableMap.of("A", "a", "B", "b", "D", "d", "C", "c");
    flowFile.putAttributes(demoAttributes);

    testRunner.enqueue(flowFile);
    testRunner.run(1, true);
    ArgumentCaptor<EventData> eventDataCaptor = ArgumentCaptor.forClass(EventData.class);

    Mockito.verify(eventHubClient).send(eventDataCaptor.capture());

    EventData event = eventDataCaptor.getValue();
    assertTrue(event.getProperties().entrySet().containsAll(demoAttributes.entrySet()));
}
 
Example #6
Source File: EventHubConsoleConsumer.java    From samza with Apache License 2.0 6 votes vote down vote up
private static void consumeEvents(String ehName, String namespace, String keyName, String token)
    throws EventHubException, IOException, ExecutionException, InterruptedException {
  ConnectionStringBuilder connStr = new ConnectionStringBuilder().setNamespaceName(namespace)
      .setEventHubName(ehName)
      .setSasKeyName(keyName)
      .setSasKey(token);

  EventHubClient client = EventHubClient.createSync(connStr.toString(), Executors.newFixedThreadPool(10));

  EventHubRuntimeInformation runTimeInfo = client.getRuntimeInformation().get();
  int numPartitions = runTimeInfo.getPartitionCount();
  for (int partition = 0; partition < numPartitions; partition++) {
    PartitionReceiver receiver =
        client.createReceiverSync(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, String.valueOf(partition),
            EventPosition.fromStartOfStream());
    receiver.receive(10).handle((records, throwable) -> handleComplete(receiver, records, throwable));
  }
}
 
Example #7
Source File: EventHubSystemProducer.java    From samza with Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Void> sendToEventHub(String streamId, EventData eventData, Object partitionKey,
    EventHubClient eventHubClient) {
  if (PartitioningMethod.ROUND_ROBIN.equals(partitioningMethod)) {
    return eventHubClient.send(eventData);
  } else if (PartitioningMethod.EVENT_HUB_HASHING.equals(partitioningMethod)) {
    if (partitionKey == null) {
      throw new SamzaException("Partition key cannot be null for EventHub hashing");
    }
    return eventHubClient.send(eventData, convertPartitionKeyToString(partitionKey));
  } else if (PartitioningMethod.PARTITION_KEY_AS_PARTITION.equals(partitioningMethod)) {
    if (!(partitionKey instanceof Integer)) {
      String msg = "Partition key should be of type Integer";
      throw new SamzaException(msg);
    }

    Integer numPartition = streamPartitionSenders.get(streamId).size();
    Integer destinationPartition = (Integer) partitionKey % numPartition;

    PartitionSender sender = streamPartitionSenders.get(streamId).get(destinationPartition);
    return sender.send(eventData);
  } else {
    throw new SamzaException("Unknown partitioning method " + partitioningMethod);
  }
}
 
Example #8
Source File: PutAzureEventHub.java    From nifi with Apache License 2.0 6 votes vote down vote up
/**
 * @param namespace name of the Eventhub namespace (part of the domain name)
 * @param eventHubName name of the eventhub, a message broker entity. Like topic.
 * @param policyName technically it is username bound to eventhub namespace or hub and privileges.
 * @param policyKey password belonging to the above policy
 * @param executor thread executor to perform the client connection.
 * @return An initialized eventhub client based on supplied parameters.
 * @throws ProcessException when creation of event hub fails due to formatting of conection string. Authorization or even network connectivity.
 */
protected EventHubClient createEventHubClient(
    final String namespace,
    final String eventHubName,
    final String policyName,
    final String policyKey,
    final ScheduledExecutorService executor)
    throws ProcessException{

    try {
        EventHubClientImpl.USER_AGENT = "ApacheNiFi-azureeventhub/3.1.1";
        final String connectionString;
        if(policyName == AzureEventHubUtils.MANAGED_IDENTITY_POLICY) {
            connectionString = AzureEventHubUtils.getManagedIdentityConnectionString(namespace, eventHubName);
        } else{
            connectionString = getConnectionString(namespace, eventHubName, policyName, policyKey);
        }
        return EventHubClient.createFromConnectionStringSync(connectionString, executor);
    } catch (IOException | EventHubException | IllegalConnectionStringFormatException e) {
        getLogger().error("Failed to create EventHubClient due to {}", new Object[]{e.getMessage()}, e);
        throw new ProcessException(e);
    }
}
 
Example #9
Source File: SamzaEventHubClientManager.java    From samza with Apache License 2.0 6 votes vote down vote up
@Override
public void init() {
  String remoteHost = String.format(EVENTHUB_REMOTE_HOST_FORMAT, eventHubNamespace);
  LOG.info("Initializing SamzaEventHubClientManager for namespace: " + eventHubNamespace);
  try {
    ConnectionStringBuilder connectionStringBuilder = new ConnectionStringBuilder()
        .setNamespaceName(eventHubNamespace)
        .setEventHubName(entityPath)
        .setSasKeyName(sasKeyName)
        .setSasKey(sasKey);

    ThreadFactoryBuilder threadFactoryBuilder = new ThreadFactoryBuilder().setNameFormat("Samza EventHubClient Thread-%d").setDaemon(true);
    eventHubClientExecutor = Executors.newFixedThreadPool(numClientThreads, threadFactoryBuilder.build());
    eventHubClient = EventHubClient.createSync(connectionStringBuilder.toString(), retryPolicy, eventHubClientExecutor);
  } catch (IOException | EventHubException e) {
    String msg = String.format("Creation of EventHub client failed for eventHub EntityPath: %s on remote host %s:%d",
            entityPath, remoteHost, ClientConstants.AMQPS_PORT);
    LOG.error(msg, e);
    throw new SamzaException(msg, e);
  }
  LOG.info("SamzaEventHubClientManager initialized for namespace: " + eventHubNamespace);
}
 
Example #10
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testMessageIsSentWithPartitioningKeyIfSpecifiedAndPopulated() {
    MockedEventhubClientMockPutAzureEventHub processor = new PutAzureEventHubTest.MockedEventhubClientMockPutAzureEventHub();
    MockitoAnnotations.initMocks(processor);

    EventHubClient eventHubClient = processor.getEventHubClient();
    when(eventHubClient.send(any(EventData.class), anyString()))
    .thenReturn(CompletableFuture.completedFuture(null));

    when(eventHubClient.send(any(EventData.class)))
    .thenThrow(new RuntimeException("Partition-key-less method called despite key is defined and required."));

    testRunner = TestRunners.newTestRunner(processor);
    setUpStandardTestConfig();
    testRunner.setProperty(PutAzureEventHub.PARTITIONING_KEY_ATTRIBUTE_NAME, TEST_PARTITIONING_KEY_ATTRIBUTE_NAME);

    MockFlowFile flowFile = new MockFlowFile(1234);
    flowFile.putAttributes(ImmutableMap.of(TEST_PARTITIONING_KEY_ATTRIBUTE_NAME, TEST_PARTITIONING_KEY));
    testRunner.enqueue(flowFile);
    testRunner.run(1, true);

    Mockito.verify(eventHubClient).send(any(EventData.class), eq(TEST_PARTITIONING_KEY));
}
 
Example #11
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testMessageIsSentWithoutPartitioningKeyIfNotSpecifiedOrNotPopulated() {
    MockedEventhubClientMockPutAzureEventHub processor = new PutAzureEventHubTest.MockedEventhubClientMockPutAzureEventHub();
    MockitoAnnotations.initMocks(processor);

    EventHubClient eventHubClient = processor.getEventHubClient();
    when(eventHubClient.send(any(EventData.class), anyString()))
    .thenThrow(new RuntimeException("Partition-key-full method called despite key is Not required or not populated."));

    when(eventHubClient.send(any(EventData.class)))
    .thenReturn(CompletableFuture.completedFuture(null));

    testRunner = TestRunners.newTestRunner(processor);
    setUpStandardTestConfig();

    MockFlowFile flowFile = new MockFlowFile(1234);
    flowFile.putAttributes(ImmutableMap.of(TEST_PARTITIONING_KEY_ATTRIBUTE_NAME, TEST_PARTITIONING_KEY));

    // Key not specified
    testRunner.enqueue(flowFile);
    testRunner.run(1, true);

    Mockito.verify(eventHubClient, never()).send(any(EventData.class), eq(TEST_PARTITIONING_KEY));
    Mockito.verify(eventHubClient).send(any(EventData.class));

    // Key wanted but not available
    testRunner.setProperty(PutAzureEventHub.PARTITIONING_KEY_ATTRIBUTE_NAME, "Non-existing-attribute");

    testRunner.enqueue(flowFile);
    testRunner.run(1, true);

    Mockito.verify(eventHubClient, never()).send(any(EventData.class), eq(TEST_PARTITIONING_KEY));
    Mockito.verify(eventHubClient, times(2)).send(any(EventData.class));
}
 
Example #12
Source File: PutAzureEventHub.java    From nifi with Apache License 2.0 5 votes vote down vote up
@OnStopped
public void tearDown() {
    EventHubClient sender;
    while ((sender = senderQueue.poll()) != null) {
        sender.close();
    }
}
 
Example #13
Source File: PutAzureEventHub.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * @param buffer Block of data to be sent as a message body. Entire array is used. See Event hub limits for body size.
 * @param partitioningKey A hint for Eventhub message broker how to distribute messages consistently amongst multiple partitions.
 * @param userProperties A key value set of customary information that is attached in User defined properties part of the message.
 * @return future object for referencing a success/failure of this message sending.
 * @throws ProcessException
 *
 * @see <a href="https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-quotas">Event Hubs Quotas</a>
 */
protected CompletableFuture<Void> sendMessage(final byte[] buffer, String partitioningKey, Map<String, Object> userProperties) throws ProcessException {

    final EventHubClient sender = senderQueue.poll();
    if(sender == null) {
        throw new ProcessException("No EventHubClients are configured for sending");
    }

    // Create message with properties
    final EventData eventData = EventData.create(buffer);
    final Map<String, Object> properties = eventData.getProperties();
    if(userProperties != null && properties != null) {
        properties.putAll(userProperties);
    }

    // Send with optional partition key
    final CompletableFuture<Void> eventFuture;
    if(StringUtils.isNotBlank(partitioningKey)) {
        eventFuture = sender.send(eventData, partitioningKey);
    }else {
        eventFuture = sender.send(eventData);
    }

    senderQueue.offer(sender);

    return eventFuture;
}
 
Example #14
Source File: GetAzureEventHub.java    From nifi with Apache License 2.0 5 votes vote down vote up
protected void setupReceiver(final String connectionString, final ScheduledExecutorService executor) throws ProcessException {
    try {
        EventHubClientImpl.USER_AGENT = "ApacheNiFi-azureeventhub/3.1.1";
        eventHubClient = EventHubClient.createFromConnectionStringSync(connectionString, executor);
    } catch (IOException | EventHubException e) {
        throw new ProcessException(e);
    }
}
 
Example #15
Source File: EventHubConsumerSource.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Override
public List<ConfigIssue> init(Info info, Context context) {
  List<ConfigIssue> issues = new ArrayList<>();
  this.context = context;

  consumerConfigBean.dataFormatConfig.stringBuilderPoolSize = getNumberOfThreads();
  consumerConfigBean.dataFormatConfig.init(
      context,
      consumerConfigBean.dataFormat,
      Groups.DATA_FORMAT.name(),
      "dataFormatConfig",
      DataFormatConstants.MAX_OVERRUN_LIMIT,
      issues
  );
  parserFactory = consumerConfigBean.dataFormatConfig.getParserFactory();
  errorQueue = new ArrayBlockingQueue<>(100);
  errorList = new ArrayList<>(100);

  // validate the connection info
  if(issues.size() == 0) {
    try {
      EventHubClient ehClient = eventHubCommon.createEventHubClient("event-hub-consumer-pool-%d");
      EventHubRuntimeInformation ehInfo = ehClient.getRuntimeInformation().get();
      ehClient.close().get();
    } catch (Exception e) {
      issues.add(context.createConfigIssue(
          Groups.EVENT_HUB.toString(),
          EventHubCommon.CONF_NAME_SPACE,
          Errors.EVENT_HUB_02,
          e.getMessage()
      ));
    }
  }

  return issues;
}
 
Example #16
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testBatchProcessesUptoMaximum() {
    MockedEventhubClientMockPutAzureEventHub processor = new PutAzureEventHubTest.MockedEventhubClientMockPutAzureEventHub();
    MockitoAnnotations.initMocks(processor);

    EventHubClient eventHubClient = processor.getEventHubClient();

    CompletableFuture<Void> failedFuture = new CompletableFuture<Void>();
    failedFuture.completeExceptionally(new IllegalArgumentException());

    when(eventHubClient.send(any(EventData.class)))
    .thenReturn(failedFuture)
    .thenReturn(CompletableFuture.completedFuture(null));

    testRunner = TestRunners.newTestRunner(processor);
    setUpStandardTestConfig();

    List<MockFlowFile> flowFiles = Arrays.asList(new MockFlowFile(1), new MockFlowFile(2), new MockFlowFile(3),
            new MockFlowFile(4), new MockFlowFile(5), new MockFlowFile(6));

    flowFiles.stream().forEachOrdered(ff -> testRunner.enqueue(ff));

    testRunner.setProperty(PutAzureEventHub.MAX_BATCH_SIZE, "4");
    testRunner.run(1, true);

    Mockito.verify(eventHubClient, times(4)).send(any(EventData.class));
    testRunner.assertTransferCount(PutAzureEventHub.REL_SUCCESS, 3);
    testRunner.assertTransferCount(PutAzureEventHub.REL_FAILURE, 1);
}
 
Example #17
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHubClient createEventHubClient(
        final String namespace,
        final String eventHubName,
        final String policyName,
        final String policyKey,
        final ScheduledExecutorService executor) throws ProcessException {
    return null;
}
 
Example #18
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHubClient createEventHubClient(
        final String namespace,
        final String eventHubName,
        final String policyName,
        final String policyKey,
        final ScheduledExecutorService executor) throws ProcessException {
    return null;
}
 
Example #19
Source File: PutAzureEventHubTest.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHubClient createEventHubClient(
        final String namespace,
        final String eventHubName,
        final String policyName,
        final String policyKey,
        final ScheduledExecutorService executor) throws ProcessException {
    return client;
}
 
Example #20
Source File: AzureIoTService.java    From para with Apache License 2.0 5 votes vote down vote up
private static EventHubClient receiveEventsAsync(final String partitionId) {
	EventHubClient client = null;
	try {
		client = EventHubClient.createSync(EVENTHUB_CONN_STR, Para.getExecutorService());
		client.createReceiver(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, partitionId,
					EventPosition.fromEnqueuedTime(Instant.now())).
				thenAccept(new Receiver(partitionId));
	} catch (Exception e) {
		logger.warn("Couldn't start receiving messages from Azure cloud: {}", e.getMessage());
	}
	return client;
}
 
Example #21
Source File: EventHubCommon.java    From datacollector with Apache License 2.0 5 votes vote down vote up
public EventHubClient createEventHubClient(String threadNamePattern) throws IOException, EventHubException {
  final ConnectionStringBuilder connStr = new ConnectionStringBuilder()
      .setNamespaceName(commonConf.namespaceName)
      .setEventHubName(commonConf.eventHubName)
      .setSasKey(commonConf.sasKey.get())
      .setSasKeyName(commonConf.sasKeyName);


  final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(
      new ThreadFactoryBuilder().setNameFormat(threadNamePattern).build()
  );

  return EventHubClient.createSync(connStr.toString(), scheduledExecutorService);
}
 
Example #22
Source File: TestEventHubSystemAdmin.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testStartpointResolverShouldResolveTheStartpointTimestampToCorrectOffset() throws EventHubException {
  // Initialize variables required for testing.
  EventHubSystemAdmin mockEventHubSystemAdmin = Mockito.mock(EventHubSystemAdmin.class);
  EventHubConfig eventHubConfig = Mockito.mock(EventHubConfig.class);
  SystemStreamPartition systemStreamPartition = new SystemStreamPartition("test-system", "test-stream", new Partition(0));
  String mockedOffsetToReturn = "100";

  // Setup the mock variables.
  EventHubClientManager mockEventHubClientManager = Mockito.mock(EventHubClientManager.class);
  EventHubClient mockEventHubClient = Mockito.mock(EventHubClient.class);
  PartitionReceiver mockPartitionReceiver = Mockito.mock(PartitionReceiver.class);
  EventData mockEventData = Mockito.mock(EventData.class);
  EventData.SystemProperties mockSystemProperties = Mockito.mock(EventData.SystemProperties.class);

  // Configure the mock variables to return the appropriate values.
  Mockito.when(mockEventHubSystemAdmin.getOrCreateStreamEventHubClient("test-stream")).thenReturn(mockEventHubClientManager);
  Mockito.when(mockEventHubClientManager.getEventHubClient()).thenReturn(mockEventHubClient);
  Mockito.when(mockEventHubClient.createReceiverSync(Mockito.anyString(), Mockito.anyString(), Mockito.any())).thenReturn(mockPartitionReceiver);
  Mockito.when(mockPartitionReceiver.receiveSync(1)).thenReturn(Arrays.asList(mockEventData));
  Mockito.when(mockEventData.getSystemProperties()).thenReturn(mockSystemProperties);
  Mockito.when(mockSystemProperties.getOffset()).thenReturn(mockedOffsetToReturn);

  // Test the Offset resolver.
  EventHubSamzaOffsetResolver resolver = new EventHubSamzaOffsetResolver(mockEventHubSystemAdmin, eventHubConfig);
  String resolvedOffset = resolver.visit(systemStreamPartition, new StartpointTimestamp(100L));
  Assert.assertEquals(mockedOffsetToReturn, resolvedOffset);
}
 
Example #23
Source File: ITestEventHubSystemProducer.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testReceive() throws EventHubException {
  EventHubClientManagerFactory clientFactory = new EventHubClientManagerFactory();
  EventHubClientManager wrapper = clientFactory
          .getEventHubClientManager(SYSTEM_NAME, STREAM_NAME1, new EventHubConfig(createEventHubConfig()));
  wrapper.init();
  EventHubClient client = wrapper.getEventHubClient();
  PartitionReceiver receiver =
          client.createReceiverSync(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, "0",
                  EventPosition.fromStartOfStream());
  receiveMessages(receiver, 300);
}
 
Example #24
Source File: EventHubSystemAdmin.java    From samza with Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) {
  Map<String, SystemStreamMetadata> requestedMetadata = new HashMap<>();

  try {
    for (String streamName : streamNames) {
      if (!streamPartitions.containsKey(streamName)) {
        LOG.debug(String.format("Partition ids for Stream=%s not found", streamName));

        EventHubClientManager eventHubClientManager = getOrCreateStreamEventHubClient(streamName);
        EventHubClient ehClient = eventHubClientManager.getEventHubClient();

        CompletableFuture<EventHubRuntimeInformation> runtimeInfo = ehClient.getRuntimeInformation();
        long timeoutMs = eventHubConfig.getRuntimeInfoWaitTimeMS(systemName);
        EventHubRuntimeInformation ehInfo = runtimeInfo.get(timeoutMs, TimeUnit.MILLISECONDS);
        LOG.info(String.format("Adding partition ids=%s for stream=%s. EHRuntimetInfo=%s",
            Arrays.toString(ehInfo.getPartitionIds()), streamName, printEventHubRuntimeInfo(ehInfo)));
        streamPartitions.put(streamName, ehInfo.getPartitionIds());
      }
      String[] partitionIds = streamPartitions.get(streamName);
      Map<Partition, SystemStreamPartitionMetadata> sspMetadataMap = getPartitionMetadata(streamName, partitionIds);
      SystemStreamMetadata systemStreamMetadata = new SystemStreamMetadata(streamName, sspMetadataMap);
      requestedMetadata.put(streamName, systemStreamMetadata);
    }
    return requestedMetadata;
  } catch (Exception e) {
    String msg = String.format("Error while fetching EventHubRuntimeInfo for System:%s", systemName);
    LOG.error(msg, e);
    throw new SamzaException(msg, e);
  }
}
 
Example #25
Source File: GetAzureEventHub.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
protected void setupReceiver(final String connectionString) throws ProcessException {
    try {
        eventHubClient = EventHubClient.createFromConnectionString(connectionString).get();
    } catch (InterruptedException | ExecutionException | IOException | ServiceBusException e) {
        throw new ProcessException(e);
    }
}
 
Example #26
Source File: PutAzureEventHub.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
protected EventHubClient createEventHubClient(final String namespace, final String eventHubName, final String policyName, final String policyKey) throws ProcessException{

        try {
            return EventHubClient.createFromConnectionString(getConnectionString(namespace, eventHubName, policyName, policyKey)).get();
        } catch (InterruptedException | ExecutionException | IOException | ServiceBusException | IllegalConnectionStringFormatException e) {
            getLogger().error("Failed to create EventHubClient due to {}", e);
            throw new ProcessException(e);
        }
    }
 
Example #27
Source File: PutAzureEventHub.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@OnStopped
public void tearDown() {
    EventHubClient sender;
    while ((sender = senderQueue.poll()) != null) {
        sender.close();
    }
}
 
Example #28
Source File: SamzaEventHubClientManager.java    From samza with Apache License 2.0 4 votes vote down vote up
@Override
public EventHubClient getEventHubClient() {
  return eventHubClient;
}
 
Example #29
Source File: PutAzureEventHubTest.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
protected EventHubClient createEventHubClient(final String namespace, final String eventHubName, final String policyName, final String policyKey) throws ProcessException {
    return null;
}
 
Example #30
Source File: PutAzureEventHubTest.java    From localization_nifi with Apache License 2.0 4 votes vote down vote up
@Override
protected EventHubClient createEventHubClient(final String namespace, final String eventHubName, final String policyName, final String policyKey) throws ProcessException {
    return null;
}