com.microsoft.azure.storage.queue.CloudQueue Java Examples

@Test
public void testCloudStorageAccountClientUriVerify() throws URISyntaxException, StorageException {
    StorageCredentialsAccountAndKey cred = new StorageCredentialsAccountAndKey(ACCOUNT_NAME, ACCOUNT_KEY);
    CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true);

    CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient();
    CloudBlobContainer container = blobClient.getContainerReference("container1");
    assertEquals(cloudStorageAccount.getBlobEndpoint().toString() + "/container1", container.getUri().toString());

    CloudQueueClient queueClient = cloudStorageAccount.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference("queue1");
    assertEquals(cloudStorageAccount.getQueueEndpoint().toString() + "/queue1", queue.getUri().toString());

    CloudTableClient tableClient = cloudStorageAccount.createCloudTableClient();
    CloudTable table = tableClient.getTableReference("table1");
    assertEquals(cloudStorageAccount.getTableEndpoint().toString() + "/table1", table.getUri().toString());

    CloudFileClient fileClient = cloudStorageAccount.createCloudFileClient();
    CloudFileShare share = fileClient.getShareReference("share1");
    assertEquals(cloudStorageAccount.getFileEndpoint().toString() + "/share1", share.getUri().toString());
}
 

private static void testQueueDownloadAttributes(LocationMode optionsLocationMode, LocationMode clientLocationMode,
        StorageLocation initialLocation, List<RetryContext> retryContextList, List<RetryInfo> retryInfoList)
        throws URISyntaxException, StorageException {
    CloudQueueClient client = TestHelper.createCloudQueueClient();
    CloudQueue queue = client.getQueueReference(QueueTestHelper.generateRandomQueueName());

    MultiLocationTestHelper helper = new MultiLocationTestHelper(queue.getServiceClient().getStorageUri(),
            initialLocation, retryContextList, retryInfoList);

    queue.getServiceClient().getDefaultRequestOptions().setLocationMode(clientLocationMode);
    QueueRequestOptions options = new QueueRequestOptions();

    options.setLocationMode(optionsLocationMode);
    options.setRetryPolicyFactory(helper.retryPolicy);

    try {
        queue.downloadAttributes(options, helper.operationContext);
    }
    catch (StorageException ex) {
        assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ex.getHttpStatusCode());
    }
    finally {
        helper.close();
    }
}
 

/**
 * This method create a queue if it doesn't exist
 */
public boolean createQueueIfNotExists(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    boolean creationResult;
    try {
        creationResult = queueRef.createIfNotExists(null, AzureStorageUtils.getTalendOperationContext());
    } catch (StorageException e) {
        if (!e.getErrorCode().equals(StorageErrorCodeStrings.QUEUE_BEING_DELETED)) {
            throw e;
        }
        LOGGER.warn(messages.getMessage("error.QueueDeleted", queueRef.getName()));
        // Documentation doesn't specify how many seconds at least to wait.
        // 40 seconds before retrying.
        // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/delete-queue3
        try {
            Thread.sleep(40000);
        } catch (InterruptedException eint) {
            throw new RuntimeException(messages.getMessage("error.InterruptedException"));
        }
        creationResult = queueRef.createIfNotExists(null, AzureStorageUtils.getTalendOperationContext());
        LOGGER.debug(messages.getMessage("debug.QueueCreated", queueRef.getName()));
    }

    return creationResult;
}
 

void postMessageToQueue(String queuePath, String content) throws StorageException, URISyntaxException {
    // Ensure
    Ensure.stringIsNotBlank(queuePath, "queuePath");
    Ensure.stringIsNotBlank(content, "content");

    CloudQueue queue = new CloudQueue(new URI(queuePath));
    CloudQueueMessage queueMessage = new CloudQueueMessage(content);
    queue.addMessage(queueMessage);
}
 

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();

    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    session.exportTo(flowFile, baos);
    final String flowFileContent = baos.toString();

    CloudQueueMessage message = new CloudQueueMessage(flowFileContent);
    CloudQueueClient cloudQueueClient;
    CloudQueue cloudQueue;

    final int ttl = context.getProperty(TTL).asTimePeriod(TimeUnit.SECONDS).intValue();
    final int delay = context.getProperty(VISIBILITY_DELAY).asTimePeriod(TimeUnit.SECONDS).intValue();
    final String queue = context.getProperty(QUEUE).evaluateAttributeExpressions(flowFile).getValue().toLowerCase();

    try {
        cloudQueueClient = createCloudQueueClient(context, flowFile);
        cloudQueue = cloudQueueClient.getQueueReference(queue);

        final OperationContext operationContext = new OperationContext();
        AzureStorageUtils.setProxy(operationContext, context);

        cloudQueue.addMessage(message, ttl, delay, null, operationContext);
    } catch (URISyntaxException | StorageException e) {
        getLogger().error("Failed to write the message to Azure Queue Storage due to {}", new Object[]{e});
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    session.transfer(flowFile, REL_SUCCESS);
    final long transmissionMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    session.getProvenanceReporter().send(flowFile, cloudQueue.getUri().toString(), transmissionMillis);
}
 

@Test
@Category({ DevFabricTests.class, DevStoreTests.class, SecondaryTests.class })
public void testQueueMaximumExecutionTime() throws URISyntaxException, StorageException {
    OperationContext opContext = new OperationContext();
    setDelay(opContext, 2500);

    // set the maximum execution time
    QueueRequestOptions options = new QueueRequestOptions();
    options.setMaximumExecutionTimeInMs(2000);

    // set the location mode to secondary, secondary request should fail
    // so set the timeout low to save time failing (or fail with a timeout)
    options.setLocationMode(LocationMode.SECONDARY_THEN_PRIMARY);
    options.setTimeoutIntervalInMs(1000);

    CloudQueueClient queueClient = TestHelper.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference(generateRandomName("queue"));

    try {
        // 1. download attributes will fail as the queue does not exist
        // 2. the executor will attempt to retry as it is accessing secondary
        // 3. maximum execution time should prevent the retry from being made
        queue.downloadAttributes(options, opContext);
        fail("Maximum execution time was reached but request did not fail.");
    }
    catch (StorageException e) {
        assertEquals(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION, e.getMessage());
    }
}
 

@Test
public void testUserAgentString() throws URISyntaxException, StorageException {
    // Test with a blob request
    CloudBlobClient blobClient = TestHelper.createCloudBlobClient();
    CloudBlobContainer container = blobClient.getContainerReference("container1");
    OperationContext sendingRequestEventContext = new OperationContext();
    sendingRequestEventContext.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() {

        @Override
        public void eventOccurred(SendingRequestEvent eventArg) {
            assertEquals(
                    Constants.HeaderConstants.USER_AGENT_PREFIX
                            + "/"
                            + Constants.HeaderConstants.USER_AGENT_VERSION
                            + " "
                            + String.format(Utility.LOCALE_US, "(Android %s; %s; %s)",
                            android.os.Build.VERSION.RELEASE, android.os.Build.BRAND,
                            android.os.Build.MODEL), ((HttpURLConnection) eventArg.getConnectionObject())
                            .getRequestProperty(Constants.HeaderConstants.USER_AGENT));
        }
    });
    container.exists(null, null, sendingRequestEventContext);

    // Test with a queue request
    CloudQueueClient queueClient = TestHelper.createCloudQueueClient();
    CloudQueue queue = queueClient.getQueueReference("queue1");
    queue.exists(null, sendingRequestEventContext);

    // Test with a table request
    CloudTableClient tableClient = TestHelper.createCloudTableClient();
    CloudTable table = tableClient.getTableReference("table1");
    table.exists(null, sendingRequestEventContext);
}
 

@Test
public void testGetReturnValues() {
    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-2"), dummyCredential));
        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-3"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());
        while (reader.advance()) {
            // read all records
        }

        Map<String, Object> returnedValues = reader.getReturnValues();
        assertNotNull(returnedValues);
        assertEquals(3, returnedValues.get(TAzureStorageQueueListDefinition.RETURN_NB_QUEUE));

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test(expected = NoSuchElementException.class)
public void testGetCurrentWhenNotAdvancable() {
    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());
        assertFalse(reader.advance());
        reader.getCurrent();

    } catch (InvalidKeyException | URISyntaxException | IOException | StorageException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test(expected = NoSuchElementException.class)
public void testGetCurrentWhenNotStartable() {
    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertFalse(reader.start());
        reader.getCurrent();

    } catch (InvalidKeyException | URISyntaxException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testGetCurrent() {
    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());
        IndexedRecord current = reader.getCurrent();
        assertNotNull(current);
        assertEquals("queue-1", current.get(0));

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testAdvanceAsNonStartable() {

    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertFalse(reader.start());
        assertFalse(reader.advance());

    } catch (InvalidKeyException | URISyntaxException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testAdvanceAsNonAdvancable() {

    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());
        assertFalse(reader.advance());

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testAdvanceAsAdvancable() {

    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-2"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());
        assertTrue(reader.advance());

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testStartAsNonStartable() {

    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertFalse(reader.start());

    } catch (InvalidKeyException | URISyntaxException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testStartAsStartable() {

    AzureStorageQueueSource source = new AzureStorageQueueSource();
    ValidationResult vr = source.initialize(getDummyRuntimeContiner(), properties);
    assertNotNull(vr);
    assertEquals(ValidationResult.OK.getStatus(), vr.getStatus());
    reader = (AzureStorageQueueListReader) source.createReader(getDummyRuntimeContiner());
    reader.queueService = queueService;

    final List<CloudQueue> list = new ArrayList<>();
    try {

        list.add(new CloudQueue(new URI("https://storagesample.queue.core.windows.net/queue-1"), dummyCredential));
        when(queueService.listQueues()).thenReturn(new Iterable<CloudQueue>() {

            @Override
            public Iterator<CloudQueue> iterator() {
                return new DummyCloudQueueIterator(list);
            }
        });

        assertTrue(reader.start());

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

public void deleteMessage(String queueName, CloudQueueMessage message)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    queueRef.deleteMessage(message, null, AzureStorageUtils.getTalendOperationContext());
}
 

public Iterable<CloudQueueMessage> retrieveMessages(String queueName, int numberOfMessages, int visibilityTimeoutInSeconds)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.retrieveMessages(numberOfMessages, visibilityTimeoutInSeconds, null, AzureStorageUtils.getTalendOperationContext());
}
 

public Iterable<CloudQueueMessage> peekMessages(String queueName, int numberOfMessages)
        throws InvalidKeyException, URISyntaxException, StorageException {

    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.peekMessages(numberOfMessages, null, AzureStorageUtils.getTalendOperationContext());
}
 

@Override
public List<NamedThing> getSchemaNames(RuntimeContainer container) throws IOException {
    List<NamedThing> result = new ArrayList<>();
    try {
        CloudQueueClient client = getStorageQueueClient(container);
        for (CloudQueue q : client.listQueues()) {
            result.add(new SimpleNamedThing(q.getName(), q.getName()));
        }
    } catch (InvalidKeyException | URISyntaxException e) {
        throw new ComponentException(e);
    }
    return result;
}
 

@javax.enterprise.inject.Produces
@Named("azureQueueClient")
public CloudQueue createQueueClient() throws Exception {
    StorageCredentials credentials = StorageCredentials.tryParseCredentials(System.getProperty("azurite.credentials"));
    URI uri = new URI(System.getProperty("azurite.queue.service.url") + QUEUE_NAME);
    return new CloudQueue(uri, credentials);
}
 

public void clear(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    queueRef.clear(null, AzureStorageUtils.getTalendOperationContext());
}
 

public long getApproximateMessageCount(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.getApproximateMessageCount();
}
 

public Iterable<CloudQueue> listQueues() throws InvalidKeyException, URISyntaxException {

        CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
        return client.listQueues(null, QueueListingDetails.NONE, null, AzureStorageUtils.getTalendOperationContext());
    }
 

public DummyCloudQueueIterator(List<CloudQueue> list) {
    super();
    this.it = list.iterator();
}
 

@Override
public CloudQueue next() {
    return it.next();
}
 

@AfterClass
public static void removeQueues() throws Throwable {
    for (CloudQueue q : queueClient.listQueues(TEST_QUEUE_NAME_CREATE)) {
        q.delete();
    }
}
 

public boolean deleteQueueIfExists(String queueName) throws InvalidKeyException, URISyntaxException, StorageException {
    CloudQueueClient client = connection.getCloudStorageAccount().createCloudQueueClient();
    CloudQueue queueRef = client.getQueueReference(queueName);
    return queueRef.deleteIfExists(null, AzureStorageUtils.getTalendOperationContext());
}
 

public CloudQueue getCloudQueue(RuntimeContainer runtime, String queue)
        throws InvalidKeyException, URISyntaxException, StorageException {
    return getStorageQueueClient(runtime).getQueueReference(queue);
}
 

@Test
public void testAppendQueue() throws Exception {

    StorageCredentials creds = getStorageCredentials("camelqueue", System.getenv(AZURE_STORAGE_QUEUE));
    Assume.assumeNotNull("Credentials not null", creds);

    OperationContext.setLoggingEnabledByDefault(true);

    CamelContext camelctx = createCamelContext(creds);
    camelctx.addRoutes(new RouteBuilder() {
        public void configure() throws Exception {
            from("direct:createQueue")
            .to("azure-queue://camelqueue/queue1?credentials=#creds&operation=createQueue");

            from("direct:listQueues")
            .to("azure-queue://camelqueue?credentials=#creds&operation=listQueues");

            from("direct:deleteQueue")
            .to("azure-queue://camelqueue/queue1?credentials=#creds&operation=deleteQueue");

            from("direct:addMessage")
            .to("azure-queue://camelqueue/queue1?credentials=#creds&operation=addMessage");

            from("direct:retrieveMessage")
            .to("azure-queue://camelqueue/queue1?credentials=#creds&operation=retrieveMessage");
        }
    });

    camelctx.start();
    try {
        ProducerTemplate producer = camelctx.createProducerTemplate();

        Iterator<?> it = producer.requestBody("direct:listQueues", null, Iterable.class).iterator();
        Assert.assertFalse("No more queues", it.hasNext());

        producer.sendBody("direct:addMessage", "SomeMsg");

        it = producer.requestBody("direct:listQueues", null, Iterable.class).iterator();
        Assert.assertTrue("Has queues", it.hasNext());
        CloudQueue queue = (CloudQueue) it.next();
        Assert.assertEquals("queue1", queue.getName());
        Assert.assertFalse("No more queues", it.hasNext());

        try {
            CloudQueueMessage msg = producer.requestBody("direct:retrieveMessage", null, CloudQueueMessage.class);
            Assert.assertNotNull("Retrieve a message", msg);
            Assert.assertEquals("SomeMsg", msg.getMessageContentAsString());
        } finally {
            queue.delete();
        }

    } finally {
        camelctx.close();
    }
}