com.microsoft.azure.storage.blob.ListBlobItem Java Examples

private void listBlobsInDirectoryWithValidation(CloudBlobContainer cloudBlobContainer, String directoryName, Boolean zeroContent)
        throws URISyntaxException, StorageException {

    CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);
    Set<String> blobsWithZeroLength = new HashSet<>();

    for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
        if (blobItem instanceof CloudBlobDirectory) {
            listBlobsInDirectoryWithValidation(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix(), zeroContent);
        } else if (blobItem instanceof CloudPageBlob) {
            validateBlobItemLength(blobItem, zeroContent, blobsWithZeroLength);
        } else if (blobItem instanceof CloudBlockBlob) {
            validateBlobItemLength(blobItem, zeroContent, blobsWithZeroLength);
        } else {
            LOGGER.error("Azure Adls Gen 2 Cloud Storage Item that is present at URI: {} cannot be classify as CloudBlob, CloudPageBlob and " +
                    "CloudBlockBlob. ", blobItem.getUri().getPath());
            throw new TestFailException(String.format("Azure Adls Gen 2 Cloud Storage Item that is present at URI: %s cannot be classify as" +
                    " CloudBlob, CloudPageBlob and CloudBlockBlob. ", blobItem.getUri().getPath()));
        }
    }
}
 

private void listBlobsInDirectory(CloudBlobContainer cloudBlobContainer, String directoryName)
        throws URISyntaxException, StorageException {

    CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);

    for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
        if (blobItem instanceof CloudBlobDirectory) {
            listBlobsInDirectory(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix());
        } else if (blobItem instanceof CloudPageBlob) {
            Log.log(LOGGER, format(" Azure Adls Gen 2 Cloud Page Blob is present with Name: [%s] and with bytes of content: [%d] at URI: [%s] ",
                    ((CloudPageBlob) blobItem).getName(), ((CloudPageBlob) blobItem).getProperties().getLength(), blobItem.getUri().getPath()));
        } else if (blobItem instanceof CloudBlockBlob) {
            Log.log(LOGGER, format(" Azure Adls Gen 2 Cloud Block Blob is present with Name: [%s] and with bytes of content: [%d] at URI: [%s] ",
                    ((CloudBlockBlob) blobItem).getName(), ((CloudBlockBlob) blobItem).getProperties().getLength(), blobItem.getUri().getPath()));
        } else {
            LOGGER.error("Azure Adls Gen 2 Cloud Storage Item that is present at URI: [{}] cannot be classify as CloudBlob, CloudPageBlob and " +
                    "CloudBlockBlob. ", blobItem.getUri().getPath());
            throw new TestFailException(String.format("Azure Adls Gen 2 Cloud Storage Item that is present at URI: [%s] cannot be classify as" +
                    " CloudBlob, CloudPageBlob and CloudBlockBlob. ", blobItem.getUri().getPath()));
        }
    }
}
 

private void deleteBlobsInDirectory(CloudBlobContainer cloudBlobContainer, String directoryName)
        throws URISyntaxException, StorageException {

    CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(directoryName);
    for (ListBlobItem blobItem : blobDirectory.listBlobs()) {
        if (blobItem instanceof CloudBlobDirectory) {
            deleteBlobsInDirectory(cloudBlobContainer, ((CloudBlobDirectory) blobItem).getPrefix());
        } else if (blobItem instanceof CloudPageBlob) {
            CloudPageBlob cloudPageBlob = cloudBlobContainer.getPageBlobReference(((CloudPageBlob) blobItem).getName());
            cloudPageBlob.deleteIfExists();
        } else if (blobItem instanceof CloudBlockBlob) {
            CloudBlockBlob cloudBlockBlob = cloudBlobContainer.getBlockBlobReference(((CloudBlockBlob) blobItem).getName());
            cloudBlockBlob.deleteIfExists();
        }
    }
}
 

/**
 * List Logs with well defined time range
 *
 * @throws URISyntaxException
 * @throws StorageException
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testCloudAnalyticsClientListLogsStartEndTime() throws URISyntaxException, StorageException, IOException {
    this.container.create();
    this.client.LogContainer = this.container.getName();
    int numBlobs = 72;
    Calendar now = new GregorianCalendar();
    now.add(GregorianCalendar.DAY_OF_MONTH, -3);
    List<String> blobNames = AnalyticsTestHelper.CreateLogs(this.container, StorageService.BLOB, 72, now,
            Granularity.HOUR);

    assertEquals(numBlobs, blobNames.size());

    Calendar start = new GregorianCalendar();
    start.add(GregorianCalendar.DAY_OF_MONTH, -2);
    Calendar end = new GregorianCalendar();
    end.add(GregorianCalendar.DAY_OF_MONTH, -1);
    for (ListBlobItem blob : this.client.listLogBlobs(StorageService.BLOB, start.getTime(), end.getTime(), null,
            null, null, null)) {
        assertEquals(CloudBlockBlob.class, blob.getClass());
        assertTrue(blobNames.remove(((CloudBlockBlob) blob).getName()));
    }
    assertTrue(blobNames.size() == 48);
}
 

/**
 * List all logs
 *
 * @throws URISyntaxException
 * @throws StorageException
 * @throws IOException
 * @throws InterruptedException
 */
public void testCloudAnalyticsClientListLogs() throws URISyntaxException, StorageException, IOException {
    this.container.create();
    this.client.LogContainer = this.container.getName();
    int numBlobs = 13;
    Calendar now = new GregorianCalendar();

    now.add(GregorianCalendar.MONTH, -13);
    List<String> blobNames = AnalyticsTestHelper.CreateLogs(this.container, StorageService.BLOB, 13, now,
            Granularity.MONTH);

    assertEquals(numBlobs, blobNames.size());

    for (ListBlobItem blob : this.client.listLogBlobs(StorageService.BLOB)) {
        assertEquals(CloudBlockBlob.class, blob.getClass());
        assertTrue(blobNames.remove(((CloudBlockBlob) blob).getName()));
    }
    assertTrue(blobNames.size() == 0);
}
 

public void deleteIfExist(RuntimeContainer runtimeContainer) {

        try {
            for (RemoteBlob rmtb : createRemoteBlobFilter()) {
                for (ListBlobItem blob : azureStorageBlobService.listBlobs(containerName, rmtb.prefix, rmtb.include)) {
                    if (blob instanceof CloudBlockBlob) {
                        // FIXME - problem with blobs with space in name...
                        boolean successfulyDeleted = azureStorageBlobService.deleteBlobBlockIfExist((CloudBlockBlob) blob);
                        if (!successfulyDeleted) {
                            LOGGER.warn(messages.getMessage("warn.FaildDelete", ((CloudBlockBlob) blob).getName()));
                        }
                    }
                }
            }
        } catch (StorageException | URISyntaxException | InvalidKeyException e) {
            LOGGER.error(e.getLocalizedMessage());
            if (dieOnError) {
                throw new ComponentException(e);
            }
        }
    }
 

@Test
public void testStartAsNonStartable() {
    try {

        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(new ArrayList<CloudBlockBlob>());
            }
        });

        properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
        properties.remoteBlobs.include.setValue(Arrays.asList(true));
        properties.remoteBlobs.prefix.setValue(Arrays.asList("dummyFilter"));

        boolean startable = reader.start();
        assertFalse(startable);
        assertFalse(reader.advance());

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

public Set<String> children(String account, String container, BlobPath path) throws URISyntaxException, StorageException {
    final var blobsBuilder = new HashSet<String>();
    final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
    final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
    final String keyPath = path.buildAsString();
    final EnumSet<BlobListingDetails> enumBlobListingDetails = EnumSet.of(BlobListingDetails.METADATA);

    for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath, false, enumBlobListingDetails, null, client.v2().get())) {
        if (blobItem instanceof CloudBlobDirectory) {
            final URI uri = blobItem.getUri();
            LOGGER.trace(() -> new ParameterizedMessage("blob url [{}]", uri));
            // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
            // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /.
            // Lastly, we add the length of keyPath to the offset to strip this container's path.
            final String uriPath = uri.getPath();
            blobsBuilder.add(uriPath.substring(1 + container.length() + 1 + keyPath.length(), uriPath.length() - 1));
        }
    }
    return Set.copyOf(blobsBuilder);
}
 

@Before
public void addMockRules() {
  CloudStorageAccount cloudStorageAccount = mock(CloudStorageAccount.class);
  CloudBlobClient cloudBlobClient = mock(CloudBlobClient.class);
  CloudBlobContainer cloudBlobContainer = mock(CloudBlobContainer.class);

  ListBlobItem listBlobItem = mock(ListBlobItem.class);
  List<ListBlobItem> lst = new ArrayList<>();
  lst.add(listBlobItem);
  PowerMockito.mockStatic(CloudStorageAccount.class);
  try {
    doReturn(cloudStorageAccount).when(CloudStorageAccount.class, "parse", Mockito.anyString());
    doReturn(cloudBlobClient).when(cloudStorageAccount).createCloudBlobClient();
    doReturn(cloudBlobContainer).when(cloudBlobClient).getContainerReference(Mockito.anyString());
    doReturn(true).when(cloudBlobContainer).exists();
    when(cloudBlobContainer.listBlobs()).thenReturn(lst);
    when(listBlobItem.getUri()).thenReturn(new URI("http://www.google.com"));

  } catch (Exception e) {
    Assert.fail("Could not initalize mocks, underlying reason " + e.getLocalizedMessage());
  }
}
 

private void deleteStaleBlobs() throws Exception {
    final Date expiryDate = Date.from(ZonedDateTime.now().minusWeeks(1).toInstant());

    final CloudBlobDirectory directoryReference = blobContainer.getDirectoryReference(request.storageLocation.clusterId + "/" + request.storageLocation.datacenterId);

    for (final ListBlobItem blob : directoryReference.listBlobs(null, true, EnumSet.noneOf(BlobListingDetails.class), null, null)) {
        if (!(blob instanceof CloudBlob)) {
            continue;
        }

        final BlobProperties properties = ((CloudBlob) blob).getProperties();
        if (properties == null || properties.getLastModified() == null) {
            continue;
        }

        if (properties.getLastModified().before(expiryDate)) {
            ((CloudBlob) blob).delete();
        }
    }
}
 

private Map<String, Long> getSnapshotFileKeys(CloudBlobContainer container, String keyPrefix) {
  Map<String, Long> snapshotFiles = new HashMap<>();

  try {
    for (ListBlobItem item : container.listBlobs(keyPrefix, true)) {
      if (item instanceof CloudPageBlob) {
        CloudPageBlob cloudBlob = (CloudPageBlob) item;
        snapshotFiles.put(cloudBlob.getName(), getOriginalFileSize(cloudBlob));
      }
    }
  } catch (StorageException e) {
    logger.error("Unable to retrieve metadata.", e);
    // all or none
    snapshotFiles = new HashMap<>();
  }
  return snapshotFiles;
}
 

/**
 * For a set of remote storage objects under a remote location and a given prefix/path
 * returns their properties wrapped in ObjectSummary objects
 *
 * @param remoteStorageLocation location, i.e. container for Azure
 * @param prefix                the prefix/path to list under
 * @return a collection of storage summary objects
 * @throws StorageProviderException Azure storage exception
 */
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation, String prefix)
throws StorageProviderException
{
  StorageObjectSummaryCollection storageObjectSummaries;

  try
  {
    CloudBlobContainer container = azStorageClient.getContainerReference(remoteStorageLocation);
    Iterable<ListBlobItem> listBlobItemIterable = container.listBlobs(
        prefix,       // List the BLOBs under this prefix
        true          // List the BLOBs as a flat list, i.e. do not list directories
    );
    storageObjectSummaries = new StorageObjectSummaryCollection(listBlobItemIterable);
  }
  catch (URISyntaxException | StorageException ex)
  {
    logger.debug("Failed to list objects: {}", ex);
    throw new StorageProviderException(ex);
  }
  return storageObjectSummaries;
}
 

@Override
public void deleteByTenant(final String tenant) {

    try {
        final CloudBlobContainer container = getContainer();
        final CloudBlobDirectory tenantDirectory = container.getDirectoryReference(sanitizeTenant(tenant));

        LOG.info("Deleting Azure Storage blob folder (tenant) from container {} for tenant {}", container.getName(),
                tenant);

        final ResultSegment<ListBlobItem> blobs = tenantDirectory.listBlobsSegmented();
        ResultContinuation token = null;
        do {
            token = blobs.getContinuationToken();
            blobs.getResults().stream().filter(CloudBlob.class::isInstance).map(CloudBlob.class::cast)
                    .forEach(this::deleteBlob);
        } while (token != null);

    } catch (final URISyntaxException | StorageException e) {
        throw new ArtifactStoreException("Failed to delete tenant directory from Azure storage", e);
    }
}
 

/**
 * List Logs with open ended time range
 *
 * @throws URISyntaxException
 * @throws StorageException
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testCloudAnalyticsClientListLogsStartTime() throws URISyntaxException, StorageException, IOException {
    this.container.create();
    this.client.LogContainer = this.container.getName();
    int numBlobs = 48;
    Calendar now = new GregorianCalendar();
    now.add(GregorianCalendar.DAY_OF_MONTH, -2);
    List<String> blobNames = AnalyticsTestHelper.CreateLogs(this.container, StorageService.BLOB, 48, now,
            Granularity.HOUR);

    assertEquals(numBlobs, blobNames.size());

    Calendar start = new GregorianCalendar();
    start.add(GregorianCalendar.DAY_OF_MONTH, -1);
    for (ListBlobItem blob : this.client.listLogBlobs(StorageService.BLOB, start.getTime(), null, null, null, null,
            null)) {
        assertEquals(CloudBlockBlob.class, blob.getClass());
        assertTrue(blobNames.remove(((CloudBlockBlob) blob).getName()));
    }
    assertTrue(blobNames.size() == 24);
}
 

/**
 * Validates that the log given is of the correct log type.
 * 
 * @param current
 *            the current log
 * @return whether or not the log is of the correct type.
 */
private boolean isCorrectLogType(ListBlobItem current) {
    HashMap<String, String> metadata = ((CloudBlob) current).getMetadata();
    String logType = metadata.get("LogType");

    if (logType == null) {
        return true;
    }

    if (this.operations.contains(LoggingOperations.READ) && logType.contains("read")) {
        return true;
    }

    if (this.operations.contains(LoggingOperations.WRITE) && logType.contains("write")) {
        return true;
    }

    if (this.operations.contains(LoggingOperations.DELETE) && logType.contains("delete")) {
        return true;
    }

    return false;
}
 

public SdxTestDto deleteAllFolders(TestContext testContext, SdxTestDto sdxTestDto, SdxClient sdxClient) {
    String containerName = getContainerName(sdxTestDto.getRequest().getCloudStorage().getBaseLocation());
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    try {
        for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                ((CloudBlob) blob).deleteIfExists();
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                deleteBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException(String.format("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned the error: %s", e));
    }

    return sdxTestDto;
}
 

public void deleteAllFolders() throws StorageException, URISyntaxException {
    String containerName = getContainerName();
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    try {
        for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                ((CloudBlob) blob).deleteIfExists();
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                deleteBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw e;
    }
}
 

@Override
public Path downloadFileToDir(final Path destinationDir, final Path remotePrefix, final Predicate<String> keyFilter) throws Exception {

    final Iterable<ListBlobItem> blobItemsIterable = list(remotePrefix);
    final List<ListBlobItem> blobItems = new ArrayList<>();

    for (final ListBlobItem listBlobItem : blobItemsIterable) {
        if (keyFilter.test(listBlobItem.getUri().getPath())) {
            blobItems.add(listBlobItem);
        }
    }

    if (blobItems.size() != 1) {
        throw new IllegalStateException(format("There is not one key which satisfies key filter: %s", blobItems.toString()));
    }

    final String blobItemPath = blobItems.get(0).getUri().getPath();
    final String fileName = blobItemPath.split("/")[blobItemPath.split("/").length - 1];

    final Path destination = destinationDir.resolve(fileName);

    downloadFile(destination, objectKeyToRemoteReference(remotePrefix.resolve(fileName)));

    return destination;
}
 

public void testDownloadBlob() {
    try {
        // Loop through each blob item in the container.
        for (ListBlobItem blobItem : container.listBlobs()) {
            // If the item is a blob, not a virtual directory.
            if (blobItem instanceof CloudBlob) {
                // Download the item and save it to a file with the same
                // name.
                CloudBlob blob = (CloudBlob) blobItem;
                blob.download(new FileOutputStream("C:\\mydownloads\\" + blob.getName()));
            }
        }
    } catch (Exception e) {
        // Output the stack trace.
        e.printStackTrace();
    }
}
 

public static List<String> listAllBlobbs(String containerName) {

    List<String> blobsList = new ArrayList<>();
    CloudBlobContainer container = AzureConnectionManager.getContainer(containerName, true);
    // Loop over blobs within the container and output the URI to each of them.
    if (container != null) {
      for (ListBlobItem blobItem : container.listBlobs()) {
        blobsList.add(blobItem.getUri().toString());
      }
    }
    return blobsList;
  }
 

@Override
public Iterable<ListBlobItem> listBlobs(String prefix,
    boolean useFlatBlobListing, EnumSet<BlobListingDetails> listingDetails,
    BlobRequestOptions options, OperationContext opContext)
    throws URISyntaxException, StorageException {
  return WrappingIterator.wrap(directory.listBlobs(prefix,
      useFlatBlobListing, listingDetails, options, opContext));
}
 

@NotNull
public static String listBlobs(@NotNull final StorageInputs inputs) throws Exception {
    final CloudBlobClient blobClient = getCloudBlobClient(inputs);
    final CloudBlobContainer container = blobClient.getContainerReference(inputs.getContainerName());
    final List<String> blobList = new ArrayList<>();
    for (final ListBlobItem blobItem : container.listBlobs()) {
        final String path = blobItem.getUri().getPath();
        blobList.add(path.substring(path.lastIndexOf(FORWARD_SLASH) + 1));
    }
    return StringUtilities.join(blobList, COMMA);
}
 

@Test
public void testListBlob() {
    try {
        // Loop over blobs within the container and output the URI to each
        // of them.
        for (ListBlobItem blobItem : container.listBlobs()) {
            System.out.println(blobItem.getUri());
        }
    } catch (Exception e) {
        // Output the stack trace.
        e.printStackTrace();
    }
}
 

@Test
public void testRunAtDriverValid() {
    properties.remoteBlobs.include.setValue(Arrays.asList(true));
    properties.remoteBlobs.prefix.setValue(Arrays.asList("block1"));
    ValidationResult validationResult = deleteBlock.initialize(runtimeContainer, properties);
    assertEquals(ValidationResult.OK.getStatus(), validationResult.getStatus());
    deleteBlock.azureStorageBlobService = blobService;
    try {
        final List<CloudBlockBlob> list = new ArrayList<>();
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));

        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(list);
            }
        });

        when(blobService.deleteBlobBlockIfExist(any(CloudBlockBlob.class))).thenReturn(true);

        deleteBlock.runAtDriver(runtimeContainer);
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        fail("should not throw " + e.getMessage());
    }

}
 

@Test
public void testRunAtDriverNotSuccessfullyDeleted() {
    properties.remoteBlobs.include.setValue(Arrays.asList(true));
    properties.remoteBlobs.prefix.setValue(Arrays.asList("block1"));
    ValidationResult validationResult = deleteBlock.initialize(runtimeContainer, properties);
    assertEquals(ValidationResult.OK.getStatus(), validationResult.getStatus());
    deleteBlock.azureStorageBlobService = blobService;

    try {
        final List<CloudBlockBlob> list = new ArrayList<>();
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));

        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(list);
            }
        });

        when(blobService.deleteBlobBlockIfExist(any(CloudBlockBlob.class))).thenReturn(false);
        deleteBlock.runAtDriver(runtimeContainer);

    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test(expected = NoSuchElementException.class)
public void getCurrentOnNonAdvancableReader() {
    try {
        final List<CloudBlockBlob> list = new ArrayList<>();
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));

        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(list);
            }
        });

        properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
        properties.remoteBlobs.include.setValue(Arrays.asList(true));
        properties.remoteBlobs.prefix.setValue(Arrays.asList("someFilter"));

        boolean startable = reader.start();
        assertTrue(startable);
        assertNotNull(reader.getCurrent());
        assertFalse(reader.advance());

        reader.getCurrent();
        fail("should throw NoSuchElementException");

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

@Test
public void testStartAsStartabke() {

    try {
        final List<CloudBlockBlob> list = new ArrayList<>();
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob1.txt")));
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob2.txt")));
        list.add(new CloudBlockBlob(new URI("https://storagesample.blob.core.windows.net/mycontainer/blob3.txt")));
        when(blobService.listBlobs(anyString(), anyString(), anyBoolean())).thenReturn(new Iterable<ListBlobItem>() {

            @Override
            public Iterator<ListBlobItem> iterator() {
                return new DummyListBlobItemIterator(list);
            }
        });

        properties.remoteBlobs = new RemoteBlobsTable("RemoteBlobsTable");
        properties.remoteBlobs.include.setValue(Arrays.asList(true));
        properties.remoteBlobs.prefix.setValue(Arrays.asList("someFilter"));

        boolean startable = reader.start();
        assertTrue(startable);
        assertNotNull(reader.getCurrent());
        while (reader.advance()) {
            assertNotNull(reader.getCurrent());
        }
        assertNotNull(reader.getReturnValues());
        assertEquals(3, reader.getReturnValues().get(ComponentDefinition.RETURN_TOTAL_RECORD_COUNT));

    } catch (InvalidKeyException | URISyntaxException | StorageException | IOException e) {
        fail("should not throw " + e.getMessage());
    }
}
 

public void listAllFolders(String baseLocation) {
    String containerName = getContainerName(baseLocation);
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    Log.log(LOGGER, format(" Azure Blob Storage URI: %s", cloudBlobContainer.getStorageUri()));
    Log.log(LOGGER, format(" Azure Blob Container: %s", cloudBlobContainer.getName()));

    try {
        for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                if (((CloudBlob) blob).exists()) {
                    Log.log(LOGGER, format(" Azure Adls Gen 2 Blob is present with Name: %s and with bytes of content: %d at URI: %s ",
                            ((CloudBlob) blob).getName(), ((CloudBlob) blob).getProperties().getLength(), blobUriPath));
                }
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                listBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException(String.format("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned the error: %s", e));
    }
}
 

@Override
public boolean start() throws IOException {
    String mycontainer = properties.container.getValue();
    List<CloudBlob> blobs = new ArrayList<>();
    // build a list with remote blobs to fetch
    List<RemoteBlob> remoteBlobs = ((AzureStorageSource) getCurrentSource()).getRemoteBlobs();
    try {

        for (RemoteBlob rmtb : remoteBlobs) {
            for (ListBlobItem blob : azureStorageBlobService.listBlobs(mycontainer, rmtb.prefix, rmtb.include)) {
                if (blob instanceof CloudBlob) {
                    blobs.add((CloudBlob) blob);
                }
            }
        }

        startable = !blobs.isEmpty();
        blobsIterator = blobs.iterator();
    } catch (StorageException | URISyntaxException | InvalidKeyException e) {
        LOGGER.error(e.getLocalizedMessage());
        if (properties.dieOnError.getValue()) {
            throw new ComponentException(e);
        }
    }

    if (startable) {
        dataCount++;
        currentBlob = blobsIterator.next();
        IndexedRecord dataRecord = new GenericData.Record(properties.schema.schema.getValue());
        dataRecord.put(0, currentBlob.getName());
        Schema rootSchema = RootSchemaUtils.createRootSchema(properties.schema.schema.getValue(), properties.outOfBandSchema);
        currentRecord = new GenericData.Record(rootSchema);
        currentRecord.put(0, dataRecord);
        currentRecord.put(1, dataRecord);
    }
    return startable;
}
 

@Override
public ListBlobItem next() {
  ListBlobItem unwrapped = present.next();
  if (unwrapped instanceof CloudBlobDirectory) {
    return new CloudBlobDirectoryWrapperImpl((CloudBlobDirectory) unwrapped);
  } else if (unwrapped instanceof CloudBlockBlob) {
    return new CloudBlockBlobWrapperImpl((CloudBlockBlob) unwrapped);
  } else if (unwrapped instanceof CloudPageBlob) {
    return new CloudPageBlobWrapperImpl((CloudPageBlob) unwrapped);
  } else {
    return unwrapped;
  }
}