Java Code Examples for com.microsoft.azure.storage.blob.CloudBlobContainer#listBlobs()

private Map<String, Long> getSnapshotFileKeys(CloudBlobContainer container, String keyPrefix) {
  Map<String, Long> snapshotFiles = new HashMap<>();

  try {
    for (ListBlobItem item : container.listBlobs(keyPrefix, true)) {
      if (item instanceof CloudPageBlob) {
        CloudPageBlob cloudBlob = (CloudPageBlob) item;
        snapshotFiles.put(cloudBlob.getName(), getOriginalFileSize(cloudBlob));
      }
    }
  } catch (StorageException e) {
    logger.error("Unable to retrieve metadata.", e);
    // all or none
    snapshotFiles = new HashMap<>();
  }
  return snapshotFiles;
}
 

/**
 * For a set of remote storage objects under a remote location and a given prefix/path
 * returns their properties wrapped in ObjectSummary objects
 *
 * @param remoteStorageLocation location, i.e. container for Azure
 * @param prefix                the prefix/path to list under
 * @return a collection of storage summary objects
 * @throws StorageProviderException Azure storage exception
 */
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation, String prefix)
throws StorageProviderException
{
  StorageObjectSummaryCollection storageObjectSummaries;

  try
  {
    CloudBlobContainer container = azStorageClient.getContainerReference(remoteStorageLocation);
    Iterable<ListBlobItem> listBlobItemIterable = container.listBlobs(
        prefix,       // List the BLOBs under this prefix
        true          // List the BLOBs as a flat list, i.e. do not list directories
    );
    storageObjectSummaries = new StorageObjectSummaryCollection(listBlobItemIterable);
  }
  catch (URISyntaxException | StorageException ex)
  {
    logger.debug("Failed to list objects: {}", ex);
    throw new StorageProviderException(ex);
  }
  return storageObjectSummaries;
}
 

public SdxTestDto deleteAllFolders(TestContext testContext, SdxTestDto sdxTestDto, SdxClient sdxClient) {
    String containerName = getContainerName(sdxTestDto.getRequest().getCloudStorage().getBaseLocation());
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    try {
        for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                ((CloudBlob) blob).deleteIfExists();
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                deleteBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException(String.format("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned the error: %s", e));
    }

    return sdxTestDto;
}
 

public void deleteAllFolders() throws StorageException, URISyntaxException {
    String containerName = getContainerName();
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    try {
        for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                ((CloudBlob) blob).deleteIfExists();
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                deleteBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw e;
    }
}
 

public Set<String> children(String account, String container, BlobPath path) throws URISyntaxException, StorageException {
    final var blobsBuilder = new HashSet<String>();
    final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
    final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
    final String keyPath = path.buildAsString();
    final EnumSet<BlobListingDetails> enumBlobListingDetails = EnumSet.of(BlobListingDetails.METADATA);

    for (ListBlobItem blobItem : blobContainer.listBlobs(keyPath, false, enumBlobListingDetails, null, client.v2().get())) {
        if (blobItem instanceof CloudBlobDirectory) {
            final URI uri = blobItem.getUri();
            LOGGER.trace(() -> new ParameterizedMessage("blob url [{}]", uri));
            // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
            // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /.
            // Lastly, we add the length of keyPath to the offset to strip this container's path.
            final String uriPath = uri.getPath();
            blobsBuilder.add(uriPath.substring(1 + container.length() + 1 + keyPath.length(), uriPath.length() - 1));
        }
    }
    return Set.copyOf(blobsBuilder);
}
 

public static List<String> listAllBlobbs(String containerName) {

    List<String> blobsList = new ArrayList<>();
    CloudBlobContainer container = AzureConnectionManager.getContainer(containerName, true);
    // Loop over blobs within the container and output the URI to each of them.
    if (container != null) {
      for (ListBlobItem blobItem : container.listBlobs()) {
        blobsList.add(blobItem.getUri().toString());
      }
    }
    return blobsList;
  }
 

public Iterable<ListBlobItem> listBlobs(final String containerName, final String prefix, final boolean useFlatBlobListing)
        throws URISyntaxException, StorageException, InvalidKeyException {
    CloudBlobClient cloudBlobClient = connection.getCloudStorageAccount().createCloudBlobClient();
    CloudBlobContainer cloudBlobContainer = cloudBlobClient.getContainerReference(containerName);
    return cloudBlobContainer.listBlobs(prefix, useFlatBlobListing, EnumSet.noneOf(BlobListingDetails.class), null,
            AzureStorageUtils.getTalendOperationContext());
}
 

@NotNull
public static String listBlobs(@NotNull final StorageInputs inputs) throws Exception {
    final CloudBlobClient blobClient = getCloudBlobClient(inputs);
    final CloudBlobContainer container = blobClient.getContainerReference(inputs.getContainerName());
    final List<String> blobList = new ArrayList<>();
    for (final ListBlobItem blobItem : container.listBlobs()) {
        final String path = blobItem.getUri().getPath();
        blobList.add(path.substring(path.lastIndexOf(FORWARD_SLASH) + 1));
    }
    return StringUtilities.join(blobList, COMMA);
}
 

public void listAllFolders(String baseLocation) {
    String containerName = getContainerName(baseLocation);
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    Log.log(LOGGER, format(" Azure Blob Storage URI: %s", cloudBlobContainer.getStorageUri()));
    Log.log(LOGGER, format(" Azure Blob Container: %s", cloudBlobContainer.getName()));

    try {
        for (ListBlobItem blob : cloudBlobContainer.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                if (((CloudBlob) blob).exists()) {
                    Log.log(LOGGER, format(" Azure Adls Gen 2 Blob is present with Name: %s and with bytes of content: %d at URI: %s ",
                            ((CloudBlob) blob).getName(), ((CloudBlob) blob).getProperties().getLength(), blobUriPath));
                }
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                listBlobsInDirectory(cloudBlobContainer, blobDirectory.getPrefix());
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException(String.format("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned the error: %s", e));
    }
}
 

public void deleteFiles(String container, String path) throws URISyntaxException, StorageException {
    final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
    // container name must be lower case.
    LOGGER.trace(() -> new ParameterizedMessage("delete files container [{}], path [{}]", container, path));
    // list the blobs using a flat blob listing mode
    final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
    for (final ListBlobItem blobItem : blobContainer.listBlobs(path, true, EnumSet.noneOf(BlobListingDetails.class), null,
        client.v2().get())) {
        final String blobName = blobNameFromUri(blobItem.getUri());
        LOGGER.trace(() -> new ParameterizedMessage("removing blob [{}] full URI was [{}]", blobName, blobItem.getUri()));
        // don't call {@code #deleteBlob}, use the same client
        final CloudBlockBlob azureBlob = blobContainer.getBlockBlobReference(blobName);
        azureBlob.delete(DeleteSnapshotsOption.NONE, null, null, client.v2().get());
    }
}
 

public Map<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix)
    throws URISyntaxException, StorageException {
    // NOTE: this should be here: if (prefix == null) prefix = "";
    // however, this is really inefficient since deleteBlobsByPrefix enumerates everything and
    // then does a prefix match on the result; it should just call listBlobsByPrefix with the prefix!
    final var blobsBuilder = new HashMap<String, BlobMetaData>();
    final EnumSet<BlobListingDetails> enumBlobListingDetails = EnumSet.of(BlobListingDetails.METADATA);
    final Tuple<CloudBlobClient, Supplier<OperationContext>> client = client();
    final CloudBlobContainer blobContainer = client.v1().getContainerReference(container);
    LOGGER.trace(() -> new ParameterizedMessage("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix));
    for (final ListBlobItem blobItem : blobContainer.listBlobs(keyPath + (prefix == null ? "" : prefix), false,
        enumBlobListingDetails, null, client.v2().get())) {
        final URI uri = blobItem.getUri();
        LOGGER.trace(() -> new ParameterizedMessage("blob url [{}]", uri));
        // uri.getPath is of the form /container/keyPath.* and we want to strip off the /container/
        // this requires 1 + container.length() + 1, with each 1 corresponding to one of the /
        final String blobPath = uri.getPath().substring(1 + container.length() + 1);
        if (blobItem instanceof CloudBlob) {
            final BlobProperties properties = ((CloudBlob) blobItem).getProperties();
            final String name = blobPath.substring(keyPath.length());
            LOGGER.trace(() -> new ParameterizedMessage("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength()));
            blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
        }
    }

    return Map.copyOf(blobsBuilder);
}
 

public void listSelectedDirectory(String baseLocation, String selectedDirectory, Boolean zeroContent) {
    String containerName = getContainerName(baseLocation);
    CloudBlobContainer cloudBlobContainer = getCloudBlobContainer(containerName);

    Log.log(LOGGER, format(" Azure Blob Storage URI: %s", cloudBlobContainer.getStorageUri()));
    Log.log(LOGGER, format(" Azure Blob Container: %s", cloudBlobContainer.getName()));
    Log.log(LOGGER, format(" Azure Blob Directory: %s", selectedDirectory));

    try {
        CloudBlobDirectory logsDirectory = cloudBlobContainer.getDirectoryReference("cluster-logs");
        CloudBlobDirectory selectedLogsDirectory = logsDirectory.getDirectoryReference(selectedDirectory);
        Set<String> blobsWithZeroLength = new HashSet<>();

        Iterable<ListBlobItem> blobListing = cloudBlobContainer.listBlobs("cluster-logs/" + selectedDirectory, true);
        List<ListBlobItem> listBlobItems = StreamSupport
                .stream(blobListing.spliterator(), false)
                .collect(Collectors.toList());
        Log.log(LOGGER, format(" Azure Blob Directory: %s contains %d sub-objects.",
                selectedDirectory, listBlobItems.size()));

        for (ListBlobItem blob : selectedLogsDirectory.listBlobs()) {
            String blobName = blob.getUri().getPath().split("/", 3)[2];
            String blobUriPath = blob.getUri().getPath();

            if (blob instanceof CloudBlob) {
                if (((CloudBlob) blob).exists()) {
                    validateBlobItemLength(blob, zeroContent, blobsWithZeroLength);
                } else {
                    LOGGER.error("Azure Adls Gen 2 Blob is NOT present with Name: {} and with bytes of content: {} at URI: {}",
                            ((CloudBlob) blob).getName(), ((CloudBlob) blob).getProperties().getLength(), blobUriPath);
                    throw new TestFailException(String.format("Azure Adls Gen 2 Blob is NOT present with Name: %s", ((CloudBlob) blob).getName()));
                }
            } else {
                if (blobName.endsWith("/")) {
                    blobName = blobName.replaceAll(".$", "");
                }
                CloudBlobDirectory blobDirectory = cloudBlobContainer.getDirectoryReference(blobName);
                listBlobsInDirectoryWithValidation(cloudBlobContainer, blobDirectory.getPrefix(), zeroContent);
            }
        }
    } catch (StorageException | URISyntaxException e) {
        LOGGER.error("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned with error!", e);
        throw new TestFailException(String.format("Azure Adls Gen 2 Blob couldn't process the call. So it has been returned the error: %s", e));
    }
}
 

@Override
protected List<BlobInfo> performListing(final ProcessContext context, final Long minTimestamp) throws IOException {
    String containerName = context.getProperty(AzureStorageUtils.CONTAINER).evaluateAttributeExpressions().getValue();
    String prefix = context.getProperty(PROP_PREFIX).evaluateAttributeExpressions().getValue();
    if (prefix == null) {
        prefix = "";
    }
    final List<BlobInfo> listing = new ArrayList<>();
    try {
        CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null);
        CloudBlobContainer container = blobClient.getContainerReference(containerName);

        final OperationContext operationContext = new OperationContext();
        AzureStorageUtils.setProxy(operationContext, context);

        for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, operationContext)) {
            if (blob instanceof CloudBlob) {
                CloudBlob cloudBlob = (CloudBlob) blob;
                BlobProperties properties = cloudBlob.getProperties();
                StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri();

                Builder builder = new BlobInfo.Builder()
                                          .primaryUri(uri.getPrimaryUri().toString())
                                          .blobName(cloudBlob.getName())
                                          .containerName(containerName)
                                          .contentType(properties.getContentType())
                                          .contentLanguage(properties.getContentLanguage())
                                          .etag(properties.getEtag())
                                          .lastModifiedTime(properties.getLastModified().getTime())
                                          .length(properties.getLength());

                if (uri.getSecondaryUri() != null) {
                    builder.secondaryUri(uri.getSecondaryUri().toString());
                }

                if (blob instanceof CloudBlockBlob) {
                    builder.blobType(AzureStorageUtils.BLOCK);
                } else {
                    builder.blobType(AzureStorageUtils.PAGE);
                }
                listing.add(builder.build());
            }
        }
    } catch (Throwable t) {
        throw new IOException(ExceptionUtils.getRootCause(t));
    }
    return listing;
}