Java Code Examples for com.amazonaws.services.s3.model.ObjectListing#getObjectSummaries()

@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 

@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 

private Collection<String> getKeys(final String bucket, final int start, final int count) {
    final Set<String> keys = new HashSet<>();
    int index = 0;
    ObjectListing listing = this.s3Client.listObjects(bucket);
    do {
        for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
            if (index < start) {
                logger.debug("skipping {} at index={} start={}", summary.getKey(), index++, start);
                continue;
            }
            keys.add(summary.getKey());

            if (keys.size() == count) {
                logger.debug("retrieved {}/{} keys, returning early", keys.size(), count);
                return keys;
            }
        }

        logger.debug("got {} keys from {}", listing.getObjectSummaries().size(), listing);
        listing = this.s3Client.listNextBatchOfObjects(listing);
    } while (listing.isTruncated());

    return keys;
}
 

/**
 * @throws Exception If error.
 */
@Override protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(),
        IgniteS3TestSuite.getSecretKey());

    AmazonS3 s3 = new AmazonS3Client(cred);

    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                s3.deleteObject(bucketName, sum.getKey());

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    }
    catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}
 

@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 

@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 

private void deleteObjects(AmazonS3 service, ObjectListing listing) throws MultiObjectDeleteException, IOException {
	do {
		if (Thread.currentThread().isInterrupted()) {
			throw new IOException(FileOperationMessages.getString("IOperationHandler.interrupted")); //$NON-NLS-1$
		}
		List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
		if (!objectSummaries.isEmpty()) {
			List<KeyVersion> keys = new ArrayList<KeyVersion>(objectSummaries.size());
			for (S3ObjectSummary object: objectSummaries) {
				keys.add(new KeyVersion(object.getKey()));
			}
			DeleteObjectsRequest request = new DeleteObjectsRequest(listing.getBucketName()).withKeys(keys).withQuiet(true);
			service.deleteObjects(request); // quiet
		}
		listing = service.listNextBatchOfObjects(listing);
	} while (listing.isTruncated());
}
 

public void listingObjects(String bucketName, String key) throws AmazonServiceException {
  /**
   * List objects in your bucket by prefix - There are many options for
   * listing the objects in your bucket. Keep in mind that buckets with many
   * objects might truncate their results when listing their objects, so be
   * sure to check if the returned object listing is truncated, and use the
   * AmazonS3.listNextBatchOfObjects(...) operation to retrieve additional
   * results.
   */
  System.out.println("Listing objects");
  ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucketName).withPrefix("My");
  ObjectListing objectListing = s3Client.listObjects(request);
  for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
    System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
  }
  System.out.println();
}
 

public void delete(String bucketName, String pathPrefix) {
    ObjectListing listing = null;
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, pathPrefix) : helper.client().listNextBatchOfObjects(listing);

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);

        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        }
        req.withKeys(keys);

        helper.client().deleteObjects(req);
    } while (listing.isTruncated());
}
 

public static void listBucketItems(String bucketName) {
System.out.println( "Connecting to AWS" );
System.out.println( "Listing files in bucket "+ bucketName );
AmazonS3 s3 = new AmazonS3Client();
Region usWest2 = Region.getRegion(Regions.US_WEST_2);
s3.setRegion(usWest2);
System.out.println("Listing buckets");
ObjectListing objectListing = s3.listObjects(new ListObjectsRequest()
	.withBucketName(bucketName));
for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
	System.out.println(" - " + objectSummary.getKey() + "  " +
	"(size = " + objectSummary.getSize() + ")");
	}
System.out.println();
}
 

@AfterClass
public static void oneTimeTearDown() {
    // Empty the bucket before deleting it.
    try {
        ObjectListing objectListing = client.listObjects(BUCKET_NAME);

        while (true) {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                client.deleteObject(BUCKET_NAME, objectSummary.getKey());
            }

            if (objectListing.isTruncated()) {
                objectListing = client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }

        DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
        client.deleteBucket(dbr);
    } catch (final AmazonS3Exception e) {
        System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
    }

    if (client.doesBucketExist(BUCKET_NAME)) {
        Assert.fail("Incomplete teardown, subsequent tests might fail");
    }

}
 

/**
 * Download to executor
 */
@Override
public Integer invoke(TransferManager transferManager, File base, VirtualChannel channel) throws IOException, InterruptedException {
    if(!base.exists()) {
        if (!base.mkdirs()) {
            throw new IOException("Failed to create directory : " + base);
        }
    }

    int totalCount;
    Downloads downloads = new Downloads();
    ObjectListing objectListing = null;

    do {
        objectListing = transferManager.getAmazonS3Client().listObjects(new ListObjectsRequest()
                .withBucketName(bucketName)
                .withPrefix(pathPrefix)
                .withMarker(objectListing != null ? objectListing.getNextMarker() : null));

        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
            downloads.startDownload(transferManager, base, pathPrefix, summary);
        }

    } while (objectListing.getNextMarker() != null);

    // Grab # of files copied
    totalCount = downloads.count();

    // Finish the asynchronous downloading process
    downloads.finishDownloading();

    return totalCount;
}
 

@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
  Map<String, NoteInfo> notesInfo = new HashMap<>();
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith(".zpln")) {
          try {
            NoteInfo info = getNoteInfo(objectSummary.getKey());
            notesInfo.put(info.getId(), info);
          } catch (IOException e) {
            LOGGER.warn(e.getMessage());
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Fail to list objects in S3", ace);
  }
  return notesInfo;
}
 

@Test
public void testBlobListRecursive() throws Exception {
    ObjectListing listing = client.listObjects(containerName);
    assertThat(listing.getObjectSummaries()).isEmpty();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "prefix/blob1",
            BYTE_SOURCE.openStream(), metadata);
    client.putObject(containerName, "prefix/blob2",
            BYTE_SOURCE.openStream(), metadata);

    ImmutableList.Builder<String> builder = ImmutableList.builder();
    listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withDelimiter("/"));
    assertThat(listing.getObjectSummaries()).isEmpty();
    for (String prefix : listing.getCommonPrefixes()) {
        builder.add(prefix);
    }
    assertThat(builder.build()).containsOnly("prefix/");

    builder = ImmutableList.builder();
    listing = client.listObjects(containerName);
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        builder.add(summary.getKey());
    }
    assertThat(builder.build()).containsOnly("prefix/blob1",
            "prefix/blob2");
    assertThat(listing.getCommonPrefixes()).isEmpty();
}
 

private void addKeyUris(List<URI> uris, ObjectListing list, URI uri, Predicate<URI> uriPredicate) {
    List<S3ObjectSummary> summaries = list.getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        String key = summary.getKey();
        if (!key.endsWith("/")) {
            URI keyUri = uri.resolve("/" + key);
            if (uriPredicate.apply(keyUri)) {
                uris.add(keyUri);
                if (logger.isDebugEnabled()) {
                    logger.debug("{}", keyUri);
                }
            }
        }
    }
}
 

/**
 * Test cases for various object listing functions for S3 bucket.
 */
@Test
public void testListObjects() {
    ObjectListing listing = s3.listObjects("testBucket");

    List<S3ObjectSummary> summaries = listing.getObjectSummaries();

    assertFalse("'testBucket' contains keys", summaries.isEmpty());
    assertTrue("'testBucket' contains more keys to fetch", listing.isTruncated());
    assertTrue(fakeKeyPrefixSet.contains(summaries.get(0).getKey()));

    listing = s3.listNextBatchOfObjects(listing);

    summaries = listing.getObjectSummaries();

    assertFalse("'testBucket' contains keys", summaries.isEmpty());
    assertTrue("'testBucket' contains more keys to fetch", listing.isTruncated());
    assertTrue(fakeKeyPrefixSet.contains(summaries.get(0).getKey()));

    listing = s3.listNextBatchOfObjects(listing);

    summaries = listing.getObjectSummaries();

    assertFalse("'testBucket' contains keys", summaries.isEmpty());
    assertFalse("'testBucket' does not contain anymore keys", listing.isTruncated());
    assertTrue(fakeKeyPrefixSet.contains(summaries.get(0).getKey()));

    try {
        s3.listObjects("nonExistentBucket");
    }
    catch (AmazonS3Exception e) {
        assertTrue(e.getMessage().contains("The specified bucket does not exist"));
    }
}
 

/***
 * Get list of S3 objects within a S3 bucket qualified by prefix path
 *
 * @param bucketName S3 bucket name
 * @param prefix S3 prefix to object
 * @return List of {@link S3ObjectSummary} objects within the bucket qualified by prefix path
 */
public List<S3ObjectSummary> listS3Bucket(String bucketName,
    String prefix) {

  final AmazonS3 amazonS3 = getS3Client();

  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName)
      .withPrefix(prefix);

  final ObjectListing objectListing = amazonS3.listObjects(listObjectsRequest);
  LOGGER.info("S3 bucket listing for bucket: " + bucketName + " with prefix: " + prefix + " is: " + objectListing);

  return objectListing.getObjectSummaries();
}
 

private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
	
	ListObjectsRequest listRequest = new ListObjectsRequest();
	listRequest.setBucketName(s3BucketName);
	// listRequest.setGeneralProgressListener(this);
	listRequest.setMaxKeys(1000);
	
	String nextMarker = null;
	ObjectListing objectListing = null;
	
	while(true) {
		
		objectListing = s3Client.listObjects(listRequest);
		
		List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
		
		for (S3ObjectSummary objSummary : objectSummaries) {
			String key = objSummary.getKey();
			
			TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
			
			// is it a "dir/" ?
			if (key.lastIndexOf("/") == (key.length() - 1)) {
				tocInfo.isDirectory = true;
			} else {
				tocInfo.isDirectory = false;
			}
			
			toc.add(tocInfo);
			tocQueue.add(tocInfo);
			tocInfosGenerated++; // increment for logging

		}
		
		// for pagination
		nextMarker = objectListing.getNextMarker();
		if (nextMarker == null) {
			break;
		} else {
			listRequest.setMarker(nextMarker);
			logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
		}
	}
	
}
 

private S3FileStatus[] listBucketContent(final Path f, final S3BucketObjectPair bop) throws IOException {

		ObjectListing listing = null;
		final List<S3FileStatus> resultList = new ArrayList<S3FileStatus>();

		final int depth = (bop.hasObject() ? getDepth(bop.getObject()) + 1 : 0);

		while (true) {

			if (listing == null) {
				if (bop.hasObject()) {
					listing = this.s3Client.listObjects(bop.getBucket(), bop.getObject());
				} else {
					listing = this.s3Client.listObjects(bop.getBucket());
				}
			} else {
				listing = this.s3Client.listNextBatchOfObjects(listing);
			}

			final List<S3ObjectSummary> list = listing.getObjectSummaries();
			final Iterator<S3ObjectSummary> it = list.iterator();
			while (it.hasNext()) {

				final S3ObjectSummary os = it.next();
				String key = os.getKey();

				final int childDepth = getDepth(os.getKey());

				if (childDepth != depth) {
					continue;
				}

				// Remove the prefix
				if (bop.hasObject()) {
					if (key.startsWith(bop.getObject())) {
						key = key.substring(bop.getObject().length());
					}

					// This has been the prefix itself
					if (key.isEmpty()) {
						continue;
					}
				}

				final long modificationDate = dateToLong(os.getLastModified());

				S3FileStatus fileStatus;
				if (objectRepresentsDirectory(os)) {
					fileStatus = new S3FileStatus(extendPath(f, key), 0, true, modificationDate, 0L);
				} else {
					fileStatus = new S3FileStatus(extendPath(f, key), os.getSize(), false, modificationDate, 0L);
				}

				resultList.add(fileStatus);
			}

			if (!listing.isTruncated()) {
				break;
			}
		}

		/*
		 * System.out.println("---- RETURN CONTENT ----");
		 * for (final FileStatus entry : resultList) {
		 * System.out.println(entry.getPath());
		 * }
		 * System.out.println("------------------------");
		 */

		return resultList.toArray(new S3FileStatus[0]);

	}
 

public void processDirectory(AmazonS3 s3, ObjectListing listing, String bucket) throws Exception {

		double time = System.currentTimeMillis();
		ExecutorService executor = Executors.newFixedThreadPool(16);

		int count = 0;

		for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
			if ("STANDARD".equalsIgnoreCase(objectSummary.getStorageClass())) {
				long size = objectSummary.getSize();
				logger.debug("*** Processing S3 {}, size: {}", objectSummary.getKey(), size);
				S3Object object = s3.getObject(new GetObjectRequest(bucket, objectSummary.getKey()));

				String bucketName = object.getBucketName();
				String keyName = object.getKey();

				GetObjectTaggingRequest request = new GetObjectTaggingRequest(bucketName, keyName);
				GetObjectTaggingResult result = s3.getObjectTagging(request);
				List<Tag> tags = result.getTagSet();
				String type = null;
				String name = null;

				if (tags.isEmpty()) {
					object.close();
					logger.warn("Error, S3 object: {} has no tags", keyName);
				} else {
					for (Tag tag : tags) {
						String key = tag.getKey();
						String value = tag.getValue();

						if (key.equals("type")) {
							type = value;
						}

						if (key.equals("name")) {
							name = value;
						}
					}

					if (name == null) {
						object.close();
						throw new Exception("Error: " + keyName + " is missing a name tag");
					}
					if (name.contains(" ")) {
						object.close();
						throw new Exception("Error: " + keyName + " has a name attribute with a space in it");
					}
					if (type == null) {
						object.close();
						throw new Exception("Error: " + keyName + " has no type tag");
					}

					if (!name.startsWith("$"))
						name = "$" + name;

					// The runnable will call object.close();
					Runnable w = new AwsWorker(type, name, object, size);
					executor.execute(w);

					count++;
				}
			}
		}
		executor.shutdown();
		executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);

		time = System.currentTimeMillis() - time;
		time = time / 60000;
		logger.info("Initialized all {} S3 objects in {} minutes", count, time);
	}