Java Code Examples for com.amazonaws.services.s3.model.ObjectListing

The following are top voted examples for showing how to use com.amazonaws.services.s3.model.ObjectListing. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ibm-cos-sdk-java   File: AmazonS3Client.java   Source Code and License 7 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listObjectsRequest = beforeClientExecution(listObjectsRequest);
    rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");

    /**
     * This flag shows whether we need to url decode S3 key names. This flag is enabled
     * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)},
     * otherwise, it will be disabled for maintaining backwards compatibility.
     */
    final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null;

    Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET);
    addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix());
    addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker());
    addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter());
    if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString());
    request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType());

    return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null);
}
 
Example 2
Project: cmn-project   File: S3.java   Source Code and License 7 votes vote down vote up
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 
Example 3
Project: Reer   File: S3ResourceResolver.java   Source Code and License 6 votes vote down vote up
private List<String> resolveDirectoryResourceNames(ObjectListing objectListing) {
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    if (objectListing.getCommonPrefixes() != null) {
        for (String prefix : objectListing.getCommonPrefixes()) {
            /**
             * The common prefixes will also include the prefix of the <code>ObjectListing</code>
             */
            String directChild = prefix.split(Pattern.quote(objectListing.getPrefix()))[1];
            if (directChild.endsWith("/")) {
                builder.add(directChild.substring(0, directChild.length() - 1));
            } else {
                builder.add(directChild);
            }
        }
        return builder.build();
    }
    return Collections.emptyList();
}
 
Example 4
Project: Reer   File: S3Client.java   Source Code and License 6 votes vote down vote up
public List<String> listDirectChildren(URI parent) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(parent);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(s3BucketKey)
        .withMaxKeys(1000)
        .withDelimiter("/");
    ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest);
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    builder.addAll(resourceResolver.resolveResourceNames(objectListing));

    while (objectListing.isTruncated()) {
        objectListing = amazonS3Client.listNextBatchOfObjects(objectListing);
        builder.addAll(resourceResolver.resolveResourceNames(objectListing));
    }
    return builder.build();
}
 
Example 5
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 6
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 7
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 8
Project: alfresco-lambda-java-utils   File: S3CleanerIntegrationTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testEmptyWithVersions() throws Exception
{
    // enable versioning on the bucket
    SetBucketVersioningConfigurationRequest req = new SetBucketVersioningConfigurationRequest(
                bucketName, new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED));
    s3.setBucketVersioningConfiguration(req);
    
    // create some versions
    createVersionedObjects();
    
    // empty the bucket of it's contents but leave the bucket itself
    S3Cleaner.emptyBucket(s3, bucketName, false, offlineContext);
    
    // check the bucket is still present but empty
    assertTrue("Bucket '" + bucketName + "' should still be present", s3.doesBucketExistV2(bucketName));
    ObjectListing objects = s3.listObjects(bucketName);
    assertTrue("Bucket '" + bucketName + "' should be empty", objects.getObjectSummaries().isEmpty());
}
 
Example 9
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 10
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 11
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 12
Project: Elasticsearch   File: S3FileInput.java   Source Code and License 6 votes vote down vote up
@Override
public List<URI> listUris(URI uri, Predicate<URI> uriPredicate) throws IOException {
    String bucketName = uri.getHost();
    if (client == null) {
        client = clientBuilder.client(uri);
    }
    String prefix = uri.getPath().length() > 1 ? uri.getPath().substring(1) : "";
    List<URI> uris = new ArrayList<>();
    ObjectListing list = client.listObjects(bucketName, prefix);
    addKeyUris(uris, list, uri, uriPredicate);
    while (list.isTruncated()) {
        list = client.listNextBatchOfObjects(list);
        addKeyUris(uris, list, uri, uriPredicate);
    }

    return uris;
}
 
Example 13
Project: ibm-cos-sdk-java   File: AmazonS3Client.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listNextBatchOfObjects(ListNextBatchOfObjectsRequest listNextBatchOfObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listNextBatchOfObjectsRequest = beforeClientExecution(listNextBatchOfObjectsRequest);
    rejectNull(listNextBatchOfObjectsRequest,
            "The request object parameter must be specified when listing the next batch of objects in a bucket");
    ObjectListing previousObjectListing = listNextBatchOfObjectsRequest.getPreviousObjectListing();

    if (!previousObjectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(previousObjectListing.getBucketName());
        emptyListing.setDelimiter(previousObjectListing.getDelimiter());
        emptyListing.setMarker(previousObjectListing.getNextMarker());
        emptyListing.setMaxKeys(previousObjectListing.getMaxKeys());
        emptyListing.setPrefix(previousObjectListing.getPrefix());
        emptyListing.setEncodingType(previousObjectListing.getEncodingType());
        emptyListing.setTruncated(false);

        return emptyListing;
    }
    return listObjects(listNextBatchOfObjectsRequest.toListObjectsRequest());
}
 
Example 14
Project: ibm-cos-sdk-java   File: S3XmlResponseHandler.java   Source Code and License 6 votes vote down vote up
/**
 * @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse)
 */
public AmazonWebServiceResponse<T> handle(HttpResponse response) throws Exception {
    AmazonWebServiceResponse<T> awsResponse = parseResponseMetadata(response);
    responseHeaders = response.getHeaders();

    if (responseUnmarshaller != null) {
        log.trace("Beginning to parse service response XML");
        T result = responseUnmarshaller.unmarshall(response.getContent());
        log.trace("Done parsing service response XML");
        awsResponse.setResult(result);
        
        if (result instanceof ObjectListing) {
        	if (!StringUtils.isNullOrEmpty(responseHeaders.get(Headers.IBM_SSE_KP_ENABLED))){
        		((ObjectListing) result).setIBMSSEKPEnabled(Boolean.parseBoolean(responseHeaders.get(Headers.IBM_SSE_KP_ENABLED)));
        	}
        	if (!StringUtils.isNullOrEmpty(responseHeaders.get(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN))) {
        		((ObjectListing) result).setIBMSSEKPCrk(responseHeaders.get(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN));
        	} 
        }
    }

    return awsResponse;
}
 
Example 15
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   Source Code and License 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK are set in the ObjectLIsting
 * response object
 * @throws Exception 
 * 
 */	
@Test
public void testHeadersAddedToObjectListing() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, "True");
	httpResponse.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, "123456");

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), "123456");
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), true);
}
 
Example 16
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   Source Code and License 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK null headers are handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testNullKPHeadersAreHandled() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_CRK, null);

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}
 
Example 17
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   Source Code and License 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_ENABLED & IBM_SSE_KP_CRK empty headers are handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testEmptyKPHeadersAreHandled() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}
 
Example 18
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   Source Code and License 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_CRK empty header is handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testOnlyKPEnabledHeaderIsSet() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_ENABLED, "True");

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), null);
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), true);
}
 
Example 19
Project: ibm-cos-sdk-java   File: S3XmlResponseHandlerTest.java   Source Code and License 6 votes vote down vote up
/**
 * Test the IBM_SSE_KP_CRK empty header is handled
 * 
 * @throws Exception 
 * 
 */	
@Test
public void testOnlyCRKHeaderIsSet() throws Exception {
	
	Unmarshaller<ObjectListing, InputStream> unmarshaller = new Unmarshallers.ListObjectsUnmarshaller(false);
	S3XmlResponseHandler xmlResponseHandler = new S3XmlResponseHandler<ObjectListing>(unmarshaller);
	HttpResponse httpResponse = new HttpResponse(null, null);
	httpResponse.addHeader(Headers.IBM_SSE_KP_CUSTOMER_ROOT_KEY_CRN, "34567");

	InputStream is = new ByteArrayInputStream(getXmlContent().getBytes());;
	httpResponse.setContent(is);

	AmazonWebServiceResponse<ObjectListing> objectListing = xmlResponseHandler.handle(httpResponse);
	
	assertEquals(objectListing.getResult().getIBMSSEKPCrk(), "34567");
	assertEquals(objectListing.getResult().getIBMSSEKPEnabled(), false);
}
 
Example 20
Project: S3Mock   File: AmazonClientUploadIT.java   Source Code and License 6 votes vote down vote up
/**
 * Tests if the list objects can be retrieved.
 */
@Test
public void shouldGetObjectListing() {
  final File uploadFile = new File(UPLOAD_FILE_NAME);
  s3Client.createBucket(BUCKET_NAME);
  s3Client.putObject(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));

  final ObjectListing objectListingResult =
      s3Client.listObjects(BUCKET_NAME, UPLOAD_FILE_NAME);

  assertThat("ObjectListinig has no S3Objects.",
      objectListingResult.getObjectSummaries().size(),
      is(greaterThan(0)));
  assertThat("The Name of the first S3ObjectSummary item has not expected the key name.",
      objectListingResult.getObjectSummaries().get(0).getKey(),
      is(UPLOAD_FILE_NAME));
}
 
Example 21
Project: galaxy-fds-migration-tool   File: S3Source.java   Source Code and License 6 votes vote down vote up
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 
Example 22
Project: RekognitionS3Batch   File: Scanner.java   Source Code and License 6 votes vote down vote up
public void scanBucket() {
    ListObjectsRequest listReq = new ListObjectsRequest()
        .withPrefix(opt.prefix())
        .withBucketName(opt.bucket());

    Logger.Info("Scanning S3 bucket %s %s", opt.bucket(), opt.prefix());
    ObjectListing listing = s3.listObjects(listReq);
    boolean ok = processObjects(listing.getObjectSummaries());

    while (ok && listing.isTruncated()) {
        listing = s3.listNextBatchOfObjects(listing);
        ok = processObjects(listing.getObjectSummaries());
    }

    Logger.Info("Completed scan, added %s images to the processing queue.", numSeen.get());
}
 
Example 23
Project: ismartonline   File: AwsFileManager.java   Source Code and License 6 votes vote down vote up
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 
Example 24
Project: vertx-pairtree   File: AbstractS3IT.java   Source Code and License 6 votes vote down vote up
/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}
 
Example 25
Project: emodb   File: StashReaderTest.java   Source Code and License 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 26
Project: kafka-connect-storage-cloud   File: TestWithMockedS3.java   Source Code and License 6 votes vote down vote up
public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}
 
Example 27
Project: hawkbit-extensions   File: S3Repository.java   Source Code and License 6 votes vote down vote up
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 
Example 28
Project: aws-doc-sdk-examples   File: ListObjects.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args)
{
    final String USAGE = "\n" +
        "To run this example, supply the name of a bucket to list!\n" +
        "\n" +
        "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    ObjectListing ol = s3.listObjects(bucket_name);
    List<S3ObjectSummary> objects = ol.getObjectSummaries();
    for (S3ObjectSummary os: objects) {
        System.out.println("* " + os.getKey());
    }
}
 
Example 29
Project: cerberus-lifecycle-cli   File: S3StoreService.java   Source Code and License 6 votes vote down vote up
public void deleteAllKeysOnPartialPath(String path) {
    ObjectListing objectListing = s3Client.listObjects(s3Bucket, getFullPath(path));

    if (objectListing.getObjectSummaries().isEmpty()) {
        return;
    }

    List<DeleteObjectsRequest.KeyVersion> keys = objectListing
            .getObjectSummaries()
            .stream()
            .map(objectSummary -> new DeleteObjectsRequest.KeyVersion(objectSummary.getKey()))
            .collect(Collectors.toList());

    DeleteObjectsRequest request = new DeleteObjectsRequest(s3Bucket);
    request.setKeys(keys);
    s3Client.deleteObjects(request);
}
 
Example 30
Project: cerberus-lifecycle-cli   File: S3StoreServiceTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testGetKeysInPartialPath() {
    AmazonS3 client = mock(AmazonS3.class);
    S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX);

    String path = "path";
    String key = "my-key";

    S3ObjectSummary summary = new S3ObjectSummary();
    summary.setKey(S3_PREFIX + key);

    ObjectListing listing = mock(ObjectListing.class);

    when(listing.getObjectSummaries()).thenReturn(Lists.newArrayList(summary));
    when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing);

    // invoke method under test
    Set<String> results = service.getKeysInPartialPath(path);

    assertEquals(1, results.size());
    assertEquals(key, results.iterator().next());
}
 
Example 31
Project: Camel   File: S3IncludeBodyTest.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 32
Project: Camel   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 33
Project: Camel   File: S3ConsumerPrefixTest.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 34
Project: exhibitor   File: S3PseudoLock.java   Source Code and License 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 35
Project: micro-server   File: S3Utils.java   Source Code and License 6 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 36
Project: micro-server   File: S3UtilsTest.java   Source Code and License 6 votes vote down vote up
@Test
public void getAllSummaries() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    utils.getAllSummaries(new ListObjectsRequest());
    verify(objectListing, times(2)).getObjectSummaries();
}
 
Example 37
Project: zeppelin   File: S3NotebookRepo.java   Source Code and License 6 votes vote down vote up
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 
Example 38
Project: zeppelin   File: S3NotebookRepo.java   Source Code and License 6 votes vote down vote up
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
Example 39
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 40
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 41
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public List<String> listBucket(String bkt, String prefix, String delimiter) throws IOException {

        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(bkt);
        listRequest.setDelimiter(delimiter);
        listRequest.setPrefix(prefix);

        ObjectListing listing = s3.listObjects(listRequest);

        ArrayList<String> list = new ArrayList<String>();

        for (S3ObjectSummary summ : listing.getObjectSummaries()) {
            list.add(summ.getKey());
        }

        return list;
    }
 
Example 42
Project: digdag   File: TestUtils.java   Source Code and License 6 votes vote down vote up
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 
Example 43
Project: ignite   File: S3CheckpointSpiSelfTest.java   Source Code and License 6 votes vote down vote up
/**
 * @throws Exception If error.
 */
@Override protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(),
        IgniteS3TestSuite.getSecretKey());

    AmazonS3 s3 = new AmazonS3Client(cred);

    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                s3.deleteObject(bucketName, sum.getKey());

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    }
    catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}
 
Example 44
Project: datacollector   File: TestAmazonS3Target.java   Source Code and License 6 votes vote down vote up
@Test
public void testWriteTextData() throws Exception {

  String prefix = "testWriteTextData";
  String suffix = "txt";
  AmazonS3Target amazonS3Target = createS3targetWithTextData(prefix, false, suffix);
  TargetRunner targetRunner = new TargetRunner.Builder(AmazonS3DTarget.class, amazonS3Target).build();
  targetRunner.runInit();

  List<Record> logRecords = TestUtil.createStringRecords(BUCKET_NAME);

  //Make sure the prefix is empty
  ObjectListing objectListing = s3client.listObjects(BUCKET_NAME, prefix);
  Assert.assertTrue(objectListing.getObjectSummaries().isEmpty());

  targetRunner.runWrite(logRecords);

  TestUtil.assertStringRecords(s3client, BUCKET_NAME, prefix);

  List<Record> events = targetRunner.getEventRecords();
  Assert.assertNotNull(events);
  Assert.assertEquals(1, events.size());
  Assert.assertEquals(9, events.get(0).get("/recordCount").getValueAsLong());

  targetRunner.runDestroy();
}
 
Example 45
Project: datacollector   File: TestAmazonS3Target.java   Source Code and License 6 votes vote down vote up
@Test
public void testWriteToTwoBuckets() throws Exception {
  String prefix = "testWriteToTwoBuckets";
  String suffix = "txt";
  AmazonS3Target amazonS3Target = createS3targetWithTextData(prefix, false, suffix);
  TargetRunner targetRunner = new TargetRunner.Builder(AmazonS3DTarget.class, amazonS3Target).build();
  targetRunner.runInit();

  List<Record> logRecords = TestUtil.createStringRecords(BUCKET_NAME);
  logRecords.addAll(TestUtil.createStringRecords(SECOND_BUCKET_NAME));

  //Make sure the prefix is empty
  ObjectListing objectListing = s3client.listObjects(BUCKET_NAME, prefix);
  Assert.assertTrue(objectListing.getObjectSummaries().isEmpty());

  objectListing = s3client.listObjects(SECOND_BUCKET_NAME, prefix);
  Assert.assertTrue(objectListing.getObjectSummaries().isEmpty());

  targetRunner.runWrite(logRecords);
  targetRunner.runDestroy();

  TestUtil.assertStringRecords(s3client, BUCKET_NAME, prefix);
  TestUtil.assertStringRecords(s3client, SECOND_BUCKET_NAME, prefix);
}
 
Example 46
Project: datacollector   File: TestAmazonS3Target.java   Source Code and License 6 votes vote down vote up
@Test
public void testWriteEmptyBatch() throws Exception {

  String prefix = "testWriteEmptyBatch";
  String suffix = "";
  AmazonS3Target amazonS3Target = createS3targetWithTextData(prefix, false, suffix);
  TargetRunner targetRunner = new TargetRunner.Builder(AmazonS3DTarget.class, amazonS3Target).build();
  targetRunner.runInit();

  List<Record> logRecords = new ArrayList<>();

  //Make sure the prefix is empty
  ObjectListing objectListing = s3client.listObjects(BUCKET_NAME, prefix);
  Assert.assertTrue(objectListing.getObjectSummaries().isEmpty());

  targetRunner.runWrite(logRecords);
  targetRunner.runDestroy();

  //Make sure the prefix is empty as no records were written
  objectListing = s3client.listObjects(BUCKET_NAME, prefix);
  Assert.assertTrue(objectListing.getObjectSummaries().isEmpty());

}
 
Example 47
Project: datacollector   File: TestAmazonS3Target.java   Source Code and License 6 votes vote down vote up
@Test
public void testEventRecords() throws Exception {
  String prefix = "testEventRecords";
  AmazonS3Target amazonS3Target = createS3targetWithTextData(prefix, false, "");
  TargetRunner targetRunner = new TargetRunner.Builder(AmazonS3DTarget.class, amazonS3Target).build();
  targetRunner.runInit();

  List<Record> logRecords = TestUtil.createStringRecords(BUCKET_NAME);

  //Make sure the prefix is empty
  ObjectListing objectListing = s3client.listObjects(BUCKET_NAME, prefix);
  Assert.assertTrue(objectListing.getObjectSummaries().isEmpty());

  targetRunner.runWrite(logRecords);

  Assert.assertEquals(1, targetRunner.getEventRecords().size());
  Record eventRecord = targetRunner.getEventRecords().get(0);

  Assert.assertTrue(eventRecord.has("/bucket"));
  Assert.assertTrue(eventRecord.has("/objectKey"));
  Assert.assertEquals(BUCKET_NAME, eventRecord.get("/bucket").getValueAsString());
  targetRunner.runDestroy();
}
 
Example 48
Project: gocd-s3-poller   File: PackageRepositoryPoller.java   Source Code and License 6 votes vote down vote up
public CheckConnectionResultMessage checkConnectionToPackage(PackageMaterialProperties packageConfiguration, PackageMaterialProperties repositoryConfiguration) {
    String bucketName = repositoryConfiguration.getProperty(Constants.S3_BUCKET).value();
    String path = packageConfiguration.getProperty(Constants.S3_PATH).value();
    ObjectListing listing;
    try {
        listing = client.listObjects(bucketName, path);
    } catch (Exception ex) {
        return new CheckConnectionResultMessage(
            CheckConnectionResultMessage.STATUS.FAILURE,
            asList("Could not find path '" + path + "' in bucket '" + bucketName + "'. [" + ex.getMessage() + "]"));
    }
    if (!listing.getObjectSummaries().isEmpty()) {
        return new CheckConnectionResultMessage(CheckConnectionResultMessage.STATUS.SUCCESS, asList("Objects found on path"));
    }
    return new CheckConnectionResultMessage(
        CheckConnectionResultMessage.STATUS.FAILURE,
        asList("Could not find objects in path. Folder can't be empty."));
}
 
Example 49
Project: flow   File: S3Filer.java   Source Code and License 6 votes vote down vote up
@Override
@Nonnull
public Record getRecord(@Nonnull String path) throws IOException {
	ObjectListing listing = s3.listObjects(
			new ListObjectsRequest().withBucketName(bucket).withPrefix(path.substring(1)));

	S3ObjectSummary summary = listing.getObjectSummaries().stream().findFirst().orElse(null);
	if (summary == null) {
		return Record.noFile(uri, path);
	}

	long time = summary.getLastModified().getTime();
	long size = summary.getSize();
	boolean directory = summary.getKey().endsWith("/");
	return new Record(uri, RecordPath.from("/" + summary.getKey()), time, size, directory);
}
 
Example 50
Project: qds-sdk-java   File: TestResultStreamer.java   Source Code and License 6 votes vote down vote up
private ResultStreamer getResultStreamer(final ObjectListing objectListing, final Map<String, S3Object> objects)
{
    return new ResultStreamer(new MockClient())
    {
        @Override
        protected Account getAccount() throws Exception
        {
            return new Account();
        }
        
        @Override
        protected AccountCredentials getAccountCredentials() throws Exception
        {
            return new AccountCredentials();
        }

        @Override
        protected S3Client newS3Client()
        {
            return new TestS3Client(objectListing, objects);
        }
    };
}
 
Example 51
Project: aws-ant-tasks   File: AWSTestUtils.java   Source Code and License 6 votes vote down vote up
public static void emptyAndDeleteBucket(AmazonS3Client client,
        String bucketName) {
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries()
                .iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator
                    .next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    client.deleteBucket(bucketName);
}
 
Example 52
Project: VStriker   File: VStrikerAPIUnitTest.java   Source Code and License 6 votes vote down vote up
public static void TestS3ReadBucket() throws Exception
{
	
	String username="[email protected]";
	String password="BRuu4J/pKJP9pAXCDvKNF+WozM+/4bJsMOuVC83F";
	String proxy="http://10.243.188.178:10101";
	String bucket="test";
	ObjectListing list =s3api.ReadBucket(username,password,proxy,null,bucket);
	   for (S3ObjectSummary objectSummary : 
		   list.getObjectSummaries()) {
		   vLogger.LogInfo("Unit Test S3 API Bucket Read - " + objectSummary.getKey() + "  " +
                      "(size = " + objectSummary.getSize() + 
                      ")");
		   
		   System.out.println("Unit Test S3 API Bucket Read - " + objectSummary.getKey() + "  " +
                      "(size = " + objectSummary.getSize() + 
                      ")");
	   }
}
 
Example 53
Project: StormCV   File: S3Connector.java   Source Code and License 6 votes vote down vote up
@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 
Example 54
Project: s3proxy   File: AwsSdkTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testUnicodeObject() throws Exception {
    String blobName = "ŪņЇЌœđЗ/☺ unicode € rocks ™";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);

    metadata = client.getObjectMetadata(containerName, blobName);
    assertThat(metadata).isNotNull();

    ObjectListing listing = client.listObjects(containerName);
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    assertThat(summaries).hasSize(1);
    S3ObjectSummary summary = summaries.iterator().next();
    assertThat(summary.getKey()).isEqualTo(blobName);
}
 
Example 55
Project: detective   File: S3ListTask.java   Source Code and License 6 votes vote down vote up
@Override
protected void doExecute(Parameters config, Parameters output) {
  String bucketName = this.readAsString(config, "aws.s3.bucketName", null, false, "aws.s3.bucketName not present in config");
  String prefix = this.readAsString(config, "aws.s3.list.prefix", null, false, "aws.s3.list.prefix not present");
  
  AmazonS3 client = new AmazonS3Client(AwsUtils.getCredentialProviderC(config), AwsUtils.getClientConfig(config));
  client.setRegion(AwsUtils.getRegion(config));
  
  ListObjectsRequest listObjectsRequest =
      new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);

  List<S3ObjectSummary> summarys = new ArrayList<S3ObjectSummary>();
  ObjectListing objectListing;
  do {
    objectListing = client.listObjects(listObjectsRequest);
    summarys.addAll(objectListing.getObjectSummaries());
    String marker = objectListing.getNextMarker();
    listObjectsRequest.setMarker(marker);
    
    Detective.info("Reading from S3...current marker:" + marker + " Continue..." );
  } while (objectListing.isTruncated());
  
  output.put("s3ObjectSummaries", summarys);
}
 
Example 56
Project: webdav-handler   File: S3Store.java   Source Code and License 6 votes vote down vote up
StoredObject readFolderObject(final String uri) {
  ObjectListing objects = this.s3client.listObjects(this.s3Properties.getBucketName(), this.s3Properties.getKeyspacePrefix() + S3Properties.normalizeFolderUri(uri));

  if (objects.getCommonPrefixes().isEmpty() && objects.getObjectSummaries().isEmpty()) {
    return null; //looks like there's no directory there.
  }

  StoredObject result = new StoredObject();

  long length = 0;
  long earliest = System.currentTimeMillis();
  for (S3ObjectSummary summary : objects.getObjectSummaries()) {
    length = Math.max(length, summary.getSize());
    earliest = Math.min(earliest, summary.getLastModified().getTime());
  }

  result.setFolder(true);
  result.setResourceLength(length);
  result.setLastModified(new Date(earliest));
  result.setCreationDate(result.getLastModified());
  return result;
}
 
Example 57
Project: xproclet   File: S3MediaStorage.java   Source Code and License 6 votes vote down vote up
@Override
public void deleteCollection(Map<String, Object> attributes, String collectionName) 
   throws IOException 
{
   if (s3Client==null) {
      throw new IOException("S3 Client is not available due to configuration error.");
   }
   String bucketName = getBucketName(attributes);
   if (bucketName==null) {
      throw new IOException("S3 Bucket not configured for "+attributes.get("app.database"));
   }
   try {
      ObjectListing listing = null;
      do {
         listing = listing==null ? s3Client.listObjects(bucketName, collectionName+"/") : s3Client.listNextBatchOfObjects(listing);
         for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            s3Client.deleteObject(bucketName, summary.getKey());
         }
      } while (listing.isTruncated());
   } catch (AmazonClientException ex) {
      throw new IOException("Cannot delete collection objects due to S3 error.",ex);
   }
}
 
Example 58
Project: esthree   File: Ls.java   Source Code and License 6 votes vote down vote up
/**
 * Return the ObjectListing starting at the given marker. The number of
 * ObjectSummaries is bounded to be 1000 (this is the Amazon S3 default,
 * which we're also setting here explicitly to minimize future SDK update
 * implications).
 *
 * @param marker the last marker from a previous call or null
 * @param limit  return no more than this many
 */
public ObjectListing list(final String marker, final int limit) throws ExecutionException, RetryException {
    return (ObjectListing) RetryUtils.AWS_RETRYER.call(new Callable<Object>() {
        public ObjectListing call() throws Exception {

            ListObjectsRequest lor = new ListObjectsRequest()
                    .withBucketName(bucket)
                    .withMarker(marker)
                    .withDelimiter("/")
                    .withMaxKeys(limit);
            if (prefix != null) {
                lor.withPrefix(prefix);
            }
            return amazonS3Client.listObjects(lor);
        }
    });
}
 
Example 59
Project: imhotep   File: S3RemoteFileSystem.java   Source Code and License 6 votes vote down vote up
private List<String> getFilenamesFromListing(ObjectListing listing, String prefix) {
    List<String> results = new ArrayList<String>(100);

    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        final String key = summary.getKey();
        final String filename;
        
        filename = key.substring(prefix.length());
        if (filename.length() == 0 || filename.contains(DELIMITER)) {
            log.error("Error parsing S3 object Key.  Key: " + key);
            continue;
        }
        results.add(filename);
    }
    
    return results;
}
 
Example 60
Project: imhotep   File: S3RemoteFileSystem.java   Source Code and License 6 votes vote down vote up
private List<String> getCommonPrefixFromListing(ObjectListing listing, String prefix) {
    List<String> results = new ArrayList<String>(100);

    for (String commonPrefix : listing.getCommonPrefixes()) {
        final String dirname;
        
        /* remove prefix and trailing delimiter */
        dirname = commonPrefix.substring(prefix.length(), 
                                         commonPrefix.length() - DELIMITER.length());
        if (dirname.length() == 0 || dirname.contains(DELIMITER)) {
            log.error("Error parsing S3 object prefix.  Prefix: " + commonPrefix);
            continue;
        }
        results.add(dirname);
    }
    
    return results;
}