Java Code Examples for com.amazonaws.services.s3.model.ObjectListing#isTruncated()

The following examples show how to use com.amazonaws.services.s3.model.ObjectListing#isTruncated() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: PrimitiveS3OperationHandler.java    From CloverETL-Engine with GNU Lesser General Public License v2.1 6 votes vote down vote up
private void deleteObjects(AmazonS3 service, ObjectListing listing) throws MultiObjectDeleteException, IOException {
	do {
		if (Thread.currentThread().isInterrupted()) {
			throw new IOException(FileOperationMessages.getString("IOperationHandler.interrupted")); //$NON-NLS-1$
		}
		List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
		if (!objectSummaries.isEmpty()) {
			List<KeyVersion> keys = new ArrayList<KeyVersion>(objectSummaries.size());
			for (S3ObjectSummary object: objectSummaries) {
				keys.add(new KeyVersion(object.getKey()));
			}
			DeleteObjectsRequest request = new DeleteObjectsRequest(listing.getBucketName()).withKeys(keys).withQuiet(true);
			service.deleteObjects(request); // quiet
		}
		listing = service.listNextBatchOfObjects(listing);
	} while (listing.isTruncated());
}
 
Example 2
Source File: S3NotebookRepo.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) throws IOException {
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(rootFolder + folderPath + "/");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith(".zpln")) {
          String noteId = getNoteId(objectSummary.getKey());
          String notePath = getNotePath(rootFolder, objectSummary.getKey());
          String newNotePath = newFolderPath + notePath.substring(folderPath.length());
          move(noteId, notePath, newNotePath, subject);
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Fail to move folder: " + folderPath + " to " + newFolderPath  + " in S3" , ace);
  }
}
 
Example 3
Source File: S3Profile.java    From jobcacher-plugin with MIT License 6 votes vote down vote up
public void delete(String bucketName, String pathPrefix) {
    ObjectListing listing = null;
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, pathPrefix) : helper.client().listNextBatchOfObjects(listing);

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);

        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        }
        req.withKeys(keys);

        helper.client().deleteObjects(req);
    } while (listing.isTruncated());
}
 
Example 4
Source File: S3FileInput.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public List<URI> listUris(URI uri, Predicate<URI> uriPredicate) throws IOException {
    String bucketName = uri.getHost();
    if (client == null) {
        client = clientBuilder.client(uri);
    }
    String prefix = uri.getPath().length() > 1 ? uri.getPath().substring(1) : "";
    List<URI> uris = new ArrayList<>();
    ObjectListing list = client.listObjects(bucketName, prefix);
    addKeyUris(uris, list, uri, uriPredicate);
    while (list.isTruncated()) {
        list = client.listNextBatchOfObjects(list);
        addKeyUris(uris, list, uri, uriPredicate);
    }

    return uris;
}
 
Example 5
Source File: S3Restorer.java    From cassandra-backup with Apache License 2.0 6 votes vote down vote up
@Override
public void consumeFiles(final RemoteObjectReference prefix, final Consumer<RemoteObjectReference> consumer) {

    final Path bucketPath = Paths.get(request.storageLocation.clusterId).resolve(request.storageLocation.datacenterId).resolve(request.storageLocation.nodeId);

    ObjectListing objectListing = amazonS3.listObjects(request.storageLocation.bucket, prefix.canonicalPath);

    boolean hasMoreContent = true;

    while (hasMoreContent) {
        objectListing.getObjectSummaries().stream()
            .filter(objectSummary -> !objectSummary.getKey().endsWith("/"))
            .forEach(objectSummary -> consumer.accept(objectKeyToRemoteReference(bucketPath.relativize(Paths.get(objectSummary.getKey())))));

        if (objectListing.isTruncated()) {
            objectListing = amazonS3.listNextBatchOfObjects(objectListing);
        } else {
            hasMoreContent = false;
        }
    }
}
 
Example 6
Source File: OldS3NotebookRepo.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
Example 7
Source File: S3Service.java    From fullstop with Apache License 2.0 6 votes vote down vote up
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
    final List<String> commonPrefixes = Lists.newArrayList();

    try {
        log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withDelimiter("/")
                .withBucketName(bucketName)
                .withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (final AmazonServiceException e) {
        log.error("Could not list common prefixes in S3", e);
    }

    return commonPrefixes;
}
 
Example 8
Source File: RepositoryS3.java    From github-bucket with ISC License 5 votes vote down vote up
private List<S3ObjectSummary> getS3ObjectSummaries() {
    // Do not include .git repository
    // matches: .git, .git/test...
    final Pattern excludePattern = Pattern.compile(String.format("^(\\.ssh|%s)(\\/.+)*$", Pattern.quote(Constants.DOT_GIT)));

    // S3 limits the size of the response to 1000 entries. Batch the requests.
    ObjectListing listing = s3.listObjects(bucket.getName());
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    while (listing.isTruncated()) {
        listing = s3.listNextBatchOfObjects(listing);
        summaries.addAll(listing.getObjectSummaries());
    }

    return summaries.stream().filter(file -> !excludePattern.matcher(file.getKey()).matches()).collect(Collectors.toList());
}
 
Example 9
Source File: S3UploadAllCallable.java    From jobcacher-plugin with MIT License 5 votes vote down vote up
private Map<String,S3ObjectSummary> lookupExistingCacheEntries(AmazonS3 s3) {
    Map<String,S3ObjectSummary> summaries = new HashMap<>();

    ObjectListing listing = s3.listObjects(bucketName, pathPrefix);
    do {
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            summaries.put(summary.getKey(), summary);
        }
        listing = listing.isTruncated() ? s3.listNextBatchOfObjects(listing) : null;
    } while (listing != null);

    return summaries;
}
 
Example 10
Source File: S3DataManipulator.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private List<KeyVersion> getKeysFromListing(ObjectListing listing) {
  Set<String> keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toSet());
  List<KeyVersion> keysToDelete = keys.stream().map(k -> new KeyVersion(k)).collect(Collectors.toList());

  if (!listing.isTruncated()) {
    return keysToDelete;
  }
  listing.setNextMarker(listing.getNextMarker());
  keysToDelete.addAll(getKeysFromListing(listing));

  return keysToDelete;
}
 
Example 11
Source File: S3BlobContainer.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    final HashMap<String, BlobMetaData> blobsBuilder = new HashMap<>();
    try (AmazonS3Reference clientReference = blobStore.clientReference()) {
        ObjectListing prevListing = null;
        while (true) {
            ObjectListing list;
            if (prevListing != null) {
                final ObjectListing finalPrevListing = prevListing;
                list = clientReference.client().listNextBatchOfObjects(finalPrevListing);
            } else {
                if (blobNamePrefix != null) {
                    list = clientReference.client().listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                } else {
                    list = clientReference.client().listObjects(blobStore.bucket(), keyPath);
                }
            }
            for (final S3ObjectSummary summary : list.getObjectSummaries()) {
                final String name = summary.getKey().substring(keyPath.length());
                blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
            }
            if (list.isTruncated()) {
                prevListing = list;
            } else {
                break;
            }
        }
        return Map.copyOf(blobsBuilder);
    } catch (final AmazonClientException e) {
        throw new IOException("Exception when listing blobs by prefix [" + blobNamePrefix + "]", e);
    }
}
 
Example 12
Source File: S3Utils.java    From micro-server with Apache License 2.0 5 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 13
Source File: AbstractS3IT.java    From nifi with Apache License 2.0 5 votes vote down vote up
@AfterClass
public static void oneTimeTearDown() {
    // Empty the bucket before deleting it.
    try {
        ObjectListing objectListing = client.listObjects(BUCKET_NAME);

        while (true) {
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                client.deleteObject(BUCKET_NAME, objectSummary.getKey());
            }

            if (objectListing.isTruncated()) {
                objectListing = client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }

        DeleteBucketRequest dbr = new DeleteBucketRequest(BUCKET_NAME);
        client.deleteBucket(dbr);
    } catch (final AmazonS3Exception e) {
        System.err.println("Unable to delete bucket " + BUCKET_NAME + e.toString());
    }

    if (client.doesBucketExist(BUCKET_NAME)) {
        Assert.fail("Incomplete teardown, subsequent tests might fail");
    }

}
 
Example 14
Source File: OldS3NotebookRepo.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<OldNoteInfo> infos = new LinkedList<>();
  OldNoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          } else {
            LOG.debug("Unable to get notebook info for key: " + objectSummary.getKey());
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 
Example 15
Source File: CloudFormationClient.java    From herd-mdl with Apache License 2.0 4 votes vote down vote up
/**
 * Delete the stack {@link #stackName}
 */
public void deleteStack() throws Exception {

    CFTStackInfo cftStackInfo = getStackInfo();
    String rootStackId = cftStackInfo.stackId(); // Use the stack id to track the delete operation
    LOGGER.info("rootStackId   =   " + rootStackId);

    // Go through the stack and pick up resources that we want
    // to finalize before deleting the stack.
    List<String> s3BucketIds = new ArrayList<>();

    DescribeStacksResult describeStacksResult = amazonCloudFormation.describeStacks();
    for (Stack currentStack : describeStacksResult.getStacks()) {
        if (rootStackId.equals(currentStack.getRootId()) || rootStackId
                .equals(currentStack.getStackId())) {
            LOGGER.info("stackId   =   " + currentStack.getStackId());
            DescribeStackResourcesRequest describeStackResourcesRequest = new DescribeStackResourcesRequest();
            describeStackResourcesRequest.setStackName(currentStack.getStackName());
            List<StackResource> stackResources = amazonCloudFormation
                    .describeStackResources(describeStackResourcesRequest).getStackResources();
            for (StackResource stackResource : stackResources) {
                if (!stackResource.getResourceStatus()
                        .equals(ResourceStatus.DELETE_COMPLETE.toString())) {
                    if (stackResource.getResourceType().equals("AWS::S3::Bucket")) {
                        s3BucketIds.add(stackResource.getPhysicalResourceId());
                    }
                }
            }
        }
    }

    // Now empty S3 buckets, clean up will be done when the stack is deleted
    AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard().withRegion(Regions.getCurrentRegion().getName())
            .withCredentials(new InstanceProfileCredentialsProvider(true)).build();
    for (String s3BucketPhysicalId : s3BucketIds) {
        String s3BucketName = s3BucketPhysicalId;
        if(!amazonS3.doesBucketExistV2(s3BucketName)){
            break;
        }
        LOGGER.info("Empyting S3 bucket, " + s3BucketName);
        ObjectListing objectListing = amazonS3.listObjects(s3BucketName);
        while (true) {
            for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator
                    .hasNext(); ) {
                S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
                amazonS3.deleteObject(s3BucketName, summary.getKey());
            }
            if (objectListing.isTruncated()) {
                objectListing = amazonS3.listNextBatchOfObjects(objectListing);
            }
            else {
                break;
            }
        }
    }

    //Proceed with the regular stack deletion operation
    DeleteStackRequest deleteRequest = new DeleteStackRequest();
    deleteRequest.setStackName(stackName);
    amazonCloudFormation.deleteStack(deleteRequest);
    LOGGER.info("Stack deletion initiated");

    CFTStackStatus cftStackStatus = waitForCompletionAndGetStackStatus(amazonCloudFormation,
            rootStackId);
    LOGGER.info(
            "Stack deletion completed, the stack " + stackName + " completed with " + cftStackStatus);

    // Throw exception if failed
    if (!cftStackStatus.getStackStatus().equals(StackStatus.DELETE_COMPLETE.toString())) {
        throw new Exception(
                "deleteStack operation failed for stack " + stackName + " - " + cftStackStatus);
    }
}
 
Example 16
Source File: S3Operations.java    From ats-framework with Apache License 2.0 4 votes vote down vote up
/**
 * Delete all objects matching given prefix. This method is preferred for efficient deletion of many files
 * 
 * @param folderPrefix empty path is expected for objects in the "root" of the bucket 
 * @param searchString what pattern to be matched. This pattern will be matched against "short file name", i.e. 
 *                     the object's ID after last path separator (&quot;/&quot;).<br />
 *                     If null it means all ( string &quot;.*&quot;). 
 * @param recursive if true searches recursively for matching in nested path levels (&quot;/&quot;)
 * 
 * @return list of deleted objects
 * @throws S3OperationException in case of an error from server
 */
@PublicAtsApi
public void deleteObjects( String folderPrefix, String searchString, boolean recursive ) {

    //Alternative but not documented in S3 API: getClient().listObjectsV2(bucket, "prefix")
    ListObjectsRequest request = new ListObjectsRequest(bucketName, folderPrefix, null, recursive
                                                                                                  ? null
                                                                                                  : "/",
                                                        null);
    int totallyDeleted = 0;
    try {
        ObjectListing objectListing = s3Client.listObjects(request);
        int i = 0;
        if (searchString == null) {
            searchString = ".*"; // any string
        }
        List<KeyVersion> keysForDelete = new ArrayList<KeyVersion>(100);
        Pattern searchStringPattern = Pattern.compile(searchString);
        while (true) {
            keysForDelete.clear();
            for (Iterator<?> iterator = objectListing.getObjectSummaries().iterator(); iterator.hasNext();) {
                S3ObjectSummary objectSummary = (S3ObjectSummary) iterator.next();
                if (LOG.isTraceEnabled()) {
                    LOG.trace("listObject[" + (++i) + "]: " + objectSummary.toString());
                }

                String[] fileTokens = objectSummary.getKey().split("/");
                String s3Object = fileTokens[fileTokens.length - 1];

                Matcher matcher = searchStringPattern.matcher(s3Object);
                if (matcher.find()) {
                    keysForDelete.add(new KeyVersion(objectSummary.getKey()));
                    //allListElements.add(new S3ObjectInfo(objectSummary));
                }
            }
            if (keysForDelete.size() > 0) {
                // delete current set / batch size
                DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(bucketName).withKeys(keysForDelete)
                                                                                                    .withQuiet(false);
                DeleteObjectsResult delObjRes = s3Client.deleteObjects(multiObjectDeleteRequest);
                int currentlyDeletedCount = delObjRes.getDeletedObjects().size();
                totallyDeleted = totallyDeleted + currentlyDeletedCount;

                // verify size of deleted objects
                if (keysForDelete.size() != currentlyDeletedCount) {
                    LOG.warn("The number of actually deleted objects " + currentlyDeletedCount +
                             " does not match the expected size of " + keysForDelete.size());
                } else {
                    LOG.debug("Number of deleted S3 objects in current batch is " + currentlyDeletedCount);
                }
            }

            // more objects to retrieve (1K batch size of objects)
            if (objectListing.isTruncated()) {
                objectListing = s3Client.listNextBatchOfObjects(objectListing);
            } else {
                break;
            }
        }
        LOG.info("Successfully deleted " + totallyDeleted + " objects");
    } catch (AmazonClientException e) {
        throw new S3OperationException("Error deleting multiple objects matching pattern " + searchString 
                                       + ". Number of deleted objects is " + totallyDeleted, e);
    } 
}
 
Example 17
Source File: S3DaoImpl.java    From herd with Apache License 2.0 4 votes vote down vote up
@Override
public List<S3ObjectSummary> listDirectory(final S3FileTransferRequestParamsDto params, boolean ignoreZeroByteDirectoryMarkers)
{
    Assert.isTrue(!isRootKeyPrefix(params.getS3KeyPrefix()), "Listing of S3 objects from root directory is not allowed.");

    AmazonS3Client s3Client = getAmazonS3(params);
    List<S3ObjectSummary> s3ObjectSummaries = new ArrayList<>();

    try
    {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(params.getS3BucketName()).withPrefix(params.getS3KeyPrefix());
        ObjectListing objectListing;

        do
        {
            objectListing = s3Operations.listObjects(listObjectsRequest, s3Client);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries())
            {
                // Ignore 0 byte objects that represent S3 directories.
                if (!(ignoreZeroByteDirectoryMarkers && objectSummary.getKey().endsWith("/") && objectSummary.getSize() == 0L))
                {
                    s3ObjectSummaries.add(objectSummary);
                }
            }

            listObjectsRequest.setMarker(objectListing.getNextMarker());
        }
        while (objectListing.isTruncated());
    }
    catch (AmazonS3Exception amazonS3Exception)
    {
        if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(amazonS3Exception.getErrorCode()))
        {
            throw new IllegalArgumentException("The specified bucket '" + params.getS3BucketName() + "' does not exist.", amazonS3Exception);
        }
        throw new IllegalStateException("Error accessing S3", amazonS3Exception);
    }
    catch (AmazonClientException e)
    {
        throw new IllegalStateException(String
            .format("Failed to list keys with prefix \"%s\" from bucket \"%s\". Reason: %s", params.getS3KeyPrefix(), params.getS3BucketName(),
                e.getMessage()), e);
    }
    finally
    {
        // Shutdown the AmazonS3Client instance to release resources.
        s3Client.shutdown();
    }

    return s3ObjectSummaries;
}
 
Example 18
Source File: BucketClass.java    From cloudExplorer with GNU General Public License v3.0 4 votes vote down vote up
String getObjectInfo(String key, String access_key,
        String secret_key, String bucket,
        String endpoint, String process
) {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    objectlist = null;

    try {
        ObjectListing current = s3Client.listObjects((bucket));

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (process.contains("checkmd5")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = objectSummary.getETag();
                        break;
                    }
                }
                if (process.contains("objectsize")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getSize());
                        break;
                    }
                }

                if (process.contains("objectdate")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getLastModified());
                        break;
                    }

                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (Exception listBucket) {
        //  mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
    }

    return objectlist;
}
 
Example 19
Source File: S3ChangeLogStore.java    From athenz with Apache License 2.0 4 votes vote down vote up
/**
 * list the objects in the zts bucket. If the mod time is specified as 0
 * then we want to list all objects otherwise, we only list objects
 * that are newer than the specified timestamp
 * @param s3 AWS S3 client object
 * @param domains collection to be updated to include domain names
 * @param modTime only include domains newer than this timestamp
 */
void listObjects(AmazonS3 s3, Collection<String> domains, long modTime) {
    
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("listObjects: Retrieving domains from {} with mod time > {}",
                s3BucketName, modTime);
    }
    
    ObjectListing objectListing = s3.listObjects(new ListObjectsRequest()
            .withBucketName(s3BucketName));
    
    String objectName;
    while (objectListing != null) {
        
        // process each entry in our result set and add the domain
        // name to our return list

        final List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
        boolean listTruncated = objectListing.isTruncated();
        
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("listObjects: retrieved {} objects, more objects available - {}",
                    objectSummaries.size(), listTruncated);
        }
        
        for (S3ObjectSummary objectSummary : objectSummaries) {
            
            // if mod time is specified then make sure we automatically skip
            // any domains older than the specified value
            
            if (modTime > 0 && objectSummary.getLastModified().getTime() <= modTime) {
                continue;
            }
            
            // for now skip any folders/objects that start with '.'
            
            objectName = objectSummary.getKey();
            if (objectName.charAt(0) == '.') {
                continue;
            }
            domains.add(objectName);
        }
        
        // check if the object listing is truncated or not (break out in this case)
        // technically we can skip this call and just call listNextBatchOfResults
        // since that returns null if the object listing is not truncated but 
        // this direct check here makes the logic easier to follow
        
        if (!listTruncated) {
            break;
        }
        
        objectListing = s3.listNextBatchOfObjects(objectListing);
    }
}
 
Example 20
Source File: AWSCommon.java    From camel-kafka-connector with Apache License 2.0 4 votes vote down vote up
/**
 * Delete an S3 bucket using the provided client. Coming from AWS documentation:
 * https://docs.aws.amazon.com/AmazonS3/latest/dev/delete-or-empty-bucket.html#delete-bucket-sdk-java
 * @param s3Client the AmazonS3 client instance used to delete the bucket
 * @param bucketName a String containing the bucket name
 */
public static void deleteBucket(AmazonS3 s3Client, String bucketName) {
    // Delete all objects from the bucket. This is sufficient
    // for non versioned buckets. For versioned buckets, when you attempt to delete objects, Amazon S3 inserts
    // delete markers for all objects, but doesn't delete the object versions.
    // To delete objects from versioned buckets, delete all of the object versions before deleting
    // the bucket (see below for an example).
    ObjectListing objectListing = s3Client.listObjects(bucketName);
    while (true) {
        Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
        while (objIter.hasNext()) {
            s3Client.deleteObject(bucketName, objIter.next().getKey());
        }

        // If the bucket contains many objects, the listObjects() call
        // might not return all of the objects in the first listing. Check to
        // see whether the listing was truncated. If so, retrieve the next page of objects
        // and delete them.
        if (objectListing.isTruncated()) {
            objectListing = s3Client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }

    // Delete all object versions (required for versioned buckets).
    VersionListing versionList = s3Client.listVersions(new ListVersionsRequest().withBucketName(bucketName));
    while (true) {
        Iterator<S3VersionSummary> versionIter = versionList.getVersionSummaries().iterator();
        while (versionIter.hasNext()) {
            S3VersionSummary vs = versionIter.next();
            s3Client.deleteVersion(bucketName, vs.getKey(), vs.getVersionId());
        }

        if (versionList.isTruncated()) {
            versionList = s3Client.listNextBatchOfVersions(versionList);
        } else {
            break;
        }
    }

    // After all objects and object versions are deleted, delete the bucket.
    s3Client.deleteBucket(bucketName);
}