Java Code Examples for com.amazonaws.services.s3.AmazonS3Client#listObjects()

The following examples show how to use com.amazonaws.services.s3.AmazonS3Client#listObjects() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AWSTestUtils.java    From aws-ant-tasks with Apache License 2.0 6 votes vote down vote up 
public static void emptyAndDeleteBucket(AmazonS3Client client,
        String bucketName) {
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries()
                .iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator
                    .next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    client.deleteBucket(bucketName);
}
Example 2
Source File: S3TableConfigClient.java    From presto with Apache License 2.0 5 votes vote down vote up 
/**
 * Call S3 to get the most recent object list.
 * <p>
 * This is an object list request to AWS in the given "directory".
 */
private List<S3ObjectSummary> getObjectSummaries()
{
    AmazonS3Client s3client = clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());

    List<S3ObjectSummary> result = new ArrayList<>();
    try {
        log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
        ListObjectsRequest request = new ListObjectsRequest()
                .withBucketName(directoryURI.getBucket())
                .withPrefix(directoryURI.getKey() + "/")
                .withDelimiter("/")
                .withMaxKeys(25);
        ObjectListing response;

        do {
            response = s3client.listObjects(request);

            result.addAll(response.getObjectSummaries());
            request.setMarker(response.getNextMarker());
        }
        while (response.isTruncated());

        log.info("Completed getting S3 object listing.");
    }
    catch (AmazonClientException e) {
        log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
    }
    return result;
}
Example 3
Source File: S3ClientImpl.java    From exhibitor with Apache License 2.0 5 votes vote down vote up 
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws Exception
{
    RefCountedClient    holder = client.get();
    AmazonS3Client      amazonS3Client = holder.useClient();
    try
    {
        return amazonS3Client.listObjects(request);
    }
    finally
    {
        holder.release();
    }
}
Example 4
Source File: TerrapinUtil.java    From terrapin with Apache License 2.0 5 votes vote down vote up 
static public List<Pair<Path, Long>> getS3FileList(AWSCredentials credentials,
    String s3Bucket, String s3KeyPrefix) {
  List<Pair<Path, Long>> fileSizePairList = Lists.newArrayListWithCapacity(
      Constants.MAX_ALLOWED_SHARDS);
  AmazonS3Client s3Client = new AmazonS3Client(credentials);
  // List files and build the path using the s3n: prefix.
  // Note that keys > marker are retrieved where the > is by lexicographic order.
  String prefix = s3KeyPrefix;
  String marker = prefix;
  while (true) {
    boolean reachedEnd = false;
    ObjectListing listing = s3Client.listObjects(new ListObjectsRequest().
        withBucketName(s3Bucket).
        withMarker(marker));
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();

    if (summaries.isEmpty()) {
      break;
    }

    for (S3ObjectSummary summary: summaries) {
      if (summary.getKey().startsWith(prefix)) {
        fileSizePairList.add(new ImmutablePair(new Path("s3n", s3Bucket, "/" + summary.getKey()),
            summary.getSize()));
        if (fileSizePairList.size() > Constants.MAX_ALLOWED_SHARDS) {
          throw new RuntimeException("Too many files " + fileSizePairList.size());
        }
      } else {
        // We found a key which does not match the prefix, stop.
        reachedEnd = true;
        break;
      }
    }
    if (reachedEnd) {
      break;
    }
    marker = summaries.get(summaries.size() - 1).getKey();
  }
  return fileSizePairList;
}