Java Code Examples for com.amazonaws.services.s3.model.ObjectListing

The following are top voted examples for showing how to use com.amazonaws.services.s3.model.ObjectListing. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ibm-cos-sdk-java   File: AmazonS3Client.java   View source code 7 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listObjectsRequest = beforeClientExecution(listObjectsRequest);
    rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");

    /**
     * This flag shows whether we need to url decode S3 key names. This flag is enabled
     * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)},
     * otherwise, it will be disabled for maintaining backwards compatibility.
     */
    final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null;

    Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET);
    addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix());
    addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker());
    addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter());
    if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString());
    request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType());

    return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null);
}
 
Example 2
Project: cmn-project   File: S3.java   View source code 7 votes vote down vote up
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 
Example 3
Project: Reer   File: S3ResourceResolver.java   View source code 6 votes vote down vote up
private List<String> resolveDirectoryResourceNames(ObjectListing objectListing) {
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    if (objectListing.getCommonPrefixes() != null) {
        for (String prefix : objectListing.getCommonPrefixes()) {
            /**
             * The common prefixes will also include the prefix of the <code>ObjectListing</code>
             */
            String directChild = prefix.split(Pattern.quote(objectListing.getPrefix()))[1];
            if (directChild.endsWith("/")) {
                builder.add(directChild.substring(0, directChild.length() - 1));
            } else {
                builder.add(directChild);
            }
        }
        return builder.build();
    }
    return Collections.emptyList();
}
 
Example 4
Project: Reer   File: S3Client.java   View source code 6 votes vote down vote up
public List<String> listDirectChildren(URI parent) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(parent);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(s3BucketKey)
        .withMaxKeys(1000)
        .withDelimiter("/");
    ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest);
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    builder.addAll(resourceResolver.resolveResourceNames(objectListing));

    while (objectListing.isTruncated()) {
        objectListing = amazonS3Client.listNextBatchOfObjects(objectListing);
        builder.addAll(resourceResolver.resolveResourceNames(objectListing));
    }
    return builder.build();
}
 
Example 5
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 6
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 7
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 8
Project: alfresco-lambda-java-utils   File: S3CleanerIntegrationTest.java   View source code 6 votes vote down vote up
@Test
public void testEmptyWithVersions() throws Exception
{
    // enable versioning on the bucket
    SetBucketVersioningConfigurationRequest req = new SetBucketVersioningConfigurationRequest(
                bucketName, new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED));
    s3.setBucketVersioningConfiguration(req);
    
    // create some versions
    createVersionedObjects();
    
    // empty the bucket of it's contents but leave the bucket itself
    S3Cleaner.emptyBucket(s3, bucketName, false, offlineContext);
    
    // check the bucket is still present but empty
    assertTrue("Bucket '" + bucketName + "' should still be present", s3.doesBucketExistV2(bucketName));
    ObjectListing objects = s3.listObjects(bucketName);
    assertTrue("Bucket '" + bucketName + "' should be empty", objects.getObjectSummaries().isEmpty());
}
 
Example 9
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 10
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 11
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 12
Project: Elasticsearch   File: S3FileInput.java   View source code 6 votes vote down vote up
@Override
public List<URI> listUris(URI uri, Predicate<URI> uriPredicate) throws IOException {
    String bucketName = uri.getHost();
    if (client == null) {
        client = clientBuilder.client(uri);
    }
    String prefix = uri.getPath().length() > 1 ? uri.getPath().substring(1) : "";
    List<URI> uris = new ArrayList<>();
    ObjectListing list = client.listObjects(bucketName, prefix);
    addKeyUris(uris, list, uri, uriPredicate);
    while (list.isTruncated()) {
        list = client.listNextBatchOfObjects(list);
        addKeyUris(uris, list, uri, uriPredicate);
    }

    return uris;
}
 
Example 13
Project: ibm-cos-sdk-java   File: AmazonS3Client.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest);
    rejectNull(listNextBatchOfObjectsRequest,
            "The request object parameter must be specified when listing the next batch of objects in a bucket");
    ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing();

    if (!previousObjectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(previousObjectListing.getBucketName());
        emptyListing.setDelimiter(previousObjectListing.getDelimiter());
        emptyListing.setMarker(previousObjectListing.getNextMarker());
        emptyListing.setMaxKeys(previousObjectListing.getMaxKeys());
        emptyListing.setPrefix(previousObjectListing.getPrefix());
        emptyListing.setEncodingType(previousObjectListing.getEncodingType());
        emptyListing.setTruncated(false);

        return emptyListing;
    }
    return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest());
}
 
Example 14
Project: ibm-cos-sdk-java   File: S3XmlResponseHandler.java   View source code 6 votes vote down vote up
/**
 * @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse)
 */
public AmazonWebServiceResponse<T> handle(HttpResponse response) throws Exception {
    AmazonWebServiceResponse<T> awsResponse = parseResponseMetadata(response);
    responseHeaders = response.getHeaders();

    if (responseUnmarshaller != null) {
        log.trace("Beginning to parse service response XML");
        T result = responseUnmarshaller.unmarshall(response.getContent());
        log.trace("Done parsing service response XML");
        awsResponse.setResult(result);
        
        if (result instanceof ObjectListing) {
        	if (!StringUtils.isNullOrEmpty(responseHeaders.get(Headers.IBM_SSE_KP_ENABLED))){
        		((ObjectListing) result).setIBMSSEKPEnabled(Boolean.parseBoolean(responseHeaders.get(Headers.IBM_SSE_KP_ENABLED)));
        	}
        	if (!StringUtils.isNullOrEmpty(responseHeaders.get(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN))) {
        		((ObjectListing) result).setIBMSSEKPCrk(responseHeaders.get(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN));
        	} 
        }
    }

    return awsResponse;
}
 
Example 15
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   View source code 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK are set in the ObjectLIsting
 * response object
 * @throws Exception 
 * 
 */	
@Test
public void testHeadersAddedToObjectListing() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, "True");
	httpResponse.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, "123456");

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), "123456");
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), true);
}
 
Example 16
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   View source code 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK null headers are handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testNullKPHeadersAreHandled() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_CRK, null);

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}
 
Example 17
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   View source code 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK empty headers are handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testEmptyKPHeadersAreHandled() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}
 
Example 18
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   View source code 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_CRK empty header is handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testOnlyKPEnabledHeaderIsSet() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, "True");

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), true);
}
 
Example 19
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   View source code 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_CRK empty header is handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testOnlyCRKHeaderIsSet() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, "34567");

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), "34567");
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}
 
Example 20
Project: S3Mock   File: AmazonClientUploadIT.java   View source code 6 votes vote down vote up
/**
 * Tests if the list objects can be retrieved.
 */
@Test
public void shouldGetObjectListing() {
  final File uploadFile = new File(UPLOAD_FILE_NAME);
  s3Client.createBucket(BUCKET_NAME);
  s3Client.putObject(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));

  final ObjectListing objectListingResult =
      s3Client.listObjects(BUCKET_NAME, UPLOAD_FILE_NAME);

  assertThat("ObjectListinig has no S3Objects.",
      objectListingResult.getObjectSummaries().size(),
      is(greaterThan(0)));
  assertThat("The Name of the first S3ObjectSummary item has not expected the key name.",
      objectListingResult.getObjectSummaries().get(0).getKey(),
      is(UPLOAD_FILE_NAME));
}
 
Example 21
Project: galaxy-fds-migration-tool   File: S3Source.java   View source code 6 votes vote down vote up
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 
Example 22
Project: RekognitionS3Batch   File: Scanner.java   View source code 6 votes vote down vote up
public void scanBucket() {
    ListObjectsRequest listReq = new ListObjectsRequest()
        .withPrefix(opt.prefix())
        .withBucketName(opt.bucket());

    Logger.Info("Scanning S3 bucket %s %s", opt.bucket(), opt.prefix());
    ObjectListing listing = s3.listObjects(listReq);
    boolean ok = processObjects(listing.getObjectSummaries());

    while (ok && listing.isTruncated()) {
        listing = s3.listNextBatchOfObjects(listing);
        ok = processObjects(listing.getObjectSummaries());
    }

    Logger.Info("Completed scan, added %s images to the processing queue.", numSeen.get());
}
 
Example 23
Project: ismartonline   File: AwsFileManager.java   View source code 6 votes vote down vote up
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 
Example 24
Project: vertx-pairtree   File: AbstractS3IT.java   View source code 6 votes vote down vote up
/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}
 
Example 25
Project: emodb   File: StashReaderTest.java   View source code 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 26
Project: kafka-connect-storage-cloud   File: TestWithMockedS3.java   View source code 6 votes vote down vote up
public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}
 
Example 27
Project: hawkbit-extensions   File: S3Repository.java   View source code 6 votes vote down vote up
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 
Example 28
Project: aws-doc-sdk-examples   File: ListObjects.java   View source code 6 votes vote down vote up
public static void main(String[] args)
{
    final String USAGE = "\n" +
        "To run this example, supply the name of a bucket to list!\n" +
        "\n" +
        "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    ObjectListing ol = s3.listObjects(bucket_name);
    List<S3ObjectSummary> objects = ol.getObjectSummaries();
    for (S3ObjectSummary os: objects) {
        System.out.println("* " + os.getKey());
    }
}
 
Example 29
Project: cerberus-lifecycle-cli   File: S3StoreService.java   View source code 6 votes vote down vote up
public void deleteAllKeysOnPartialPath(String path) {
    ObjectListing objectListing = s3Client.listObjects(s3Bucket, getFullPath(path));

    if (objectListing.getObjectSummaries().isEmpty()) {
        return;
    }

    List<DeleteObjectsRequest.KeyVersion> keys = objectListing
            .getObjectSummaries()
            .stream()
            .map(objectSummary -> new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()))
            .collect(Collectors.toList());

    DeleteObjectsRequest request = new DeleteObjectsRequest(s3Bucket);
    request.setKeys(keys);
    s3Client.deleteObjects(request);
}
 
Example 30
Project: cerberus-lifecycle-cli   File: S3StoreServiceTest.java   View source code 6 votes vote down vote up
@Test
public void testGetKeysInPartialPath() {
    AmazonS3 client = mock(AmazonS3.class);
    S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX);

    String path = "path";
    String key = "my-key";

    S3ObjectSummary summary = new S3ObjectSummary();
    summary.setKey(S3_PREFIX + key);

    ObjectListing listing = mock(ObjectListing.class);

    when(listing.getObjectSummaries()).thenReturn(Lists.newArrayList(summary));
    when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing);

    // invoke method under test
    Set<String> results = service.getKeysInPartialPath(path);

    assertEquals(1, results.size());
    assertEquals(key, results.iterator().next());
}
 
Example 31
Project: Camel   File: S3IncludeBodyTest.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 32
Project: Camel   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 33
Project: Camel   File: S3ConsumerPrefixTest.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 34
Project: exhibitor   File: S3PseudoLock.java   View source code 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 35
Project: micro-server   File: S3Utils.java   View source code 6 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 36
Project: micro-server   File: S3UtilsTest.java   View source code 6 votes vote down vote up
@Test
public void getAllSummaries() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    utils.getAllSummaries(new ListObjectsRequest());
    verify(objectListing, times(2)).getObjectSummaries();
}
 
Example 37
Project: zeppelin   File: S3NotebookRepo.java   View source code 6 votes vote down vote up
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 
Example 38
Project: zeppelin   File: S3NotebookRepo.java   View source code 6 votes vote down vote up
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
Example 39
Project: caboclo   File: AmazonClient.java   View source code 6 votes vote down vote up
public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 40
Project: caboclo   File: AmazonClient.java   View source code 6 votes vote down vote up
public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}