Java Code Examples for com.amazonaws.services.s3.model.ListObjectsRequest

The following are top voted examples for showing how to use com.amazonaws.services.s3.model.ListObjectsRequest. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ibm-cos-sdk-java   File: AmazonS3Client.java   View source code 7 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listObjectsRequest = beforeClientExecution(listObjectsRequest);
    rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");

    /**
     * This flag shows whether we need to url decode S3 key names. This flag is enabled
     * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)},
     * otherwise, it will be disabled for maintaining backwards compatibility.
     */
    final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null;

    Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET);
    addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix());
    addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker());
    addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter());
    if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString());
    request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType());

    return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null);
}
 
Example 2
Project: cmn-project   File: S3.java   View source code 7 votes vote down vote up
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 
Example 3
Project: Reer   File: S3Client.java   View source code 6 votes vote down vote up
public List<String> listDirectChildren(URI parent) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(parent);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(s3BucketKey)
        .withMaxKeys(1000)
        .withDelimiter("/");
    ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest);
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    builder.addAll(resourceResolver.resolveResourceNames(objectListing));

    while (objectListing.isTruncated()) {
        objectListing = amazonS3Client.listNextBatchOfObjects(objectListing);
        builder.addAll(resourceResolver.resolveResourceNames(objectListing));
    }
    return builder.build();
}
 
Example 4
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 5
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 6
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 7
Project: circus-train   File: S3S3CopierTest.java   View source code 6 votes vote down vote up
@Test
public void copyMultipleObjects() throws Exception {
  // Making sure we only request 1 file at the time so we need to loop
  ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
  when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));

  client.putObject("source", "bar/data1", inputData);
  client.putObject("source", "bar/data2", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(14L));

  S3Object object1 = client.getObject("target", "foo/data1");
  String data1 = IOUtils.toString(object1.getObjectContent());
  assertThat(data1, is("bar foo"));
  S3Object object2 = client.getObject("target", "foo/data2");
  String data2 = IOUtils.toString(object2.getObjectContent());
  assertThat(data2, is("bar foo"));
}
 
Example 8
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 9
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 10
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 11
Project: ibm-cos-sdk-java   File: S3Objects.java   View source code 6 votes vote down vote up
private void prepareCurrentListing() {
    while ( currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated()) ) {

        if ( currentListing == null ) {
            ListObjectsRequest req = new ListObjectsRequest();
            req.setBucketName(getBucketName());
            req.setPrefix(getPrefix());
            req.setMaxKeys(getBatchSize());
            currentListing = getS3().listObjects(req);
        } else {
            currentListing = getS3().listNextBatchOfObjects(currentListing);
        }

        currentIterator = currentListing.getObjectSummaries().iterator();
    }
}
 
Example 12
Project: galaxy-fds-migration-tool   File: S3Source.java   View source code 6 votes vote down vote up
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 
Example 13
Project: RekognitionS3Batch   File: Scanner.java   View source code 6 votes vote down vote up
public void scanBucket() {
    ListObjectsRequest listReq = new ListObjectsRequest()
        .withPrefix(opt.prefix())
        .withBucketName(opt.bucket());

    Logger.Info("Scanning S3 bucket %s %s", opt.bucket(), opt.prefix());
    ObjectListing listing = s3.listObjects(listReq);
    boolean ok = processObjects(listing.getObjectSummaries());

    while (ok && listing.isTruncated()) {
        listing = s3.listNextBatchOfObjects(listing);
        ok = processObjects(listing.getObjectSummaries());
    }

    Logger.Info("Completed scan, added %s images to the processing queue.", numSeen.get());
}
 
Example 14
Project: ismartonline   File: AwsFileManager.java   View source code 6 votes vote down vote up
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 
Example 15
Project: emodb   File: StashReaderTest.java   View source code 6 votes vote down vote up
private Matcher<ListObjectsRequest> listObjectRequest(final String bucket, final String prefix, @Nullable final String marker) {
    return new BaseMatcher<ListObjectsRequest>() {
        @Override
        public boolean matches(Object item) {
            ListObjectsRequest request = (ListObjectsRequest) item;
            return request != null &&
                    request.getBucketName().equals(bucket) &&
                    request.getPrefix().equals(prefix) &&
                    Objects.equals(request.getMarker(), marker);
        }

        @Override
        public void describeTo(Description description) {
            description.appendText("ListObjectRequest[s3://").appendText(bucket).appendText("/").appendText(prefix);
            if (marker != null) {
                description.appendText(", marker=").appendText(marker);
            }
            description.appendText("]");
        }
    };
}
 
Example 16
Project: emodb   File: StashReaderTest.java   View source code 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 17
Project: Camel   File: S3IncludeBodyTest.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 18
Project: Camel   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 19
Project: Camel   File: S3ConsumerPrefixTest.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 20
Project: exhibitor   File: S3PseudoLock.java   View source code 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 21
Project: s3_video   File: AWSAdapter.java   View source code 6 votes vote down vote up
public List<String> listKeysInOutputBucket(String bucketName) {
	List<String> videoKeys = new ArrayList<>();
	
	ListObjectsRequest listFirstLevelKeyRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withDelimiter("/");
	
	for (String commonPrefix : s3client.listObjects(listFirstLevelKeyRequest).getCommonPrefixes()) {
		
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withPrefix(commonPrefix)
		.withDelimiter("/");
		
		for (S3ObjectSummary file : s3client.listObjects(listObjectsRequest).getObjectSummaries()) {
			videoKeys.add(file.getKey());
		}
	}
	return videoKeys;		
}
 
Example 22
Project: micro-server   File: S3Utils.java   View source code 6 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 23
Project: micro-server   File: S3UtilsTest.java   View source code 6 votes vote down vote up
@Test
public void getAllSummaries() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    utils.getAllSummaries(new ListObjectsRequest());
    verify(objectListing, times(2)).getObjectSummaries();
}
 
Example 24
Project: zeppelin   File: S3NotebookRepo.java   View source code 6 votes vote down vote up
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 
Example 25
Project: zeppelin   File: S3NotebookRepo.java   View source code 6 votes vote down vote up
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
Example 26
Project: caboclo   File: AmazonClient.java   View source code 6 votes vote down vote up
public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 27
Project: caboclo   File: AmazonClient.java   View source code 6 votes vote down vote up
public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 28
Project: caboclo   File: AmazonClient.java   View source code 6 votes vote down vote up
public List<String> listBucket(String bkt, String prefix, String delimiter) throws IOException {

        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(bkt);
        listRequest.setDelimiter(delimiter);
        listRequest.setPrefix(prefix);

        ObjectListing listing = s3.listObjects(listRequest);

        ArrayList<String> list = new ArrayList<String>();

        for (S3ObjectSummary summ : listing.getObjectSummaries()) {
            list.add(summ.getKey());
        }

        return list;
    }
 
Example 29
Project: digdag   File: TestUtils.java   View source code 6 votes vote down vote up
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 
Example 30
Project: datacollector   File: S3ConnectionSourceConfig.java   View source code 6 votes vote down vote up
private void validateConnection(
    Stage.Context context,
    String configPrefix,
    List<Stage.ConfigIssue> issues
) {
  try {
    //check if the credentials are right by trying to list an object in the common prefix
    getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
  } catch (AmazonS3Exception e) {
    LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
    issues.add(
        context.createConfigIssue(
            Groups.S3.name(),
            configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
            Errors.S3_SPOOLDIR_20,
            e.toString()
        )
    );
  }
}
 
Example 31
Project: flow   File: S3Filer.java   View source code 6 votes vote down vote up
@Override
@Nonnull
public Record getRecord(@Nonnull String path) throws IOException {
	ObjectListing listing = s3.listObjects(
			new ListObjectsRequest().withBucketName(bucket).withPrefix(path.substring(1)));

	S3ObjectSummary summary = listing.getObjectSummaries().stream().findFirst().orElse(null);
	if (summary == null) {
		return Record.noFile(uri, path);
	}

	long time = summary.getLastModified().getTime();
	long size = summary.getSize();
	boolean directory = summary.getKey().endsWith("/");
	return new Record(uri, RecordPath.from("/" + summary.getKey()), time, size, directory);
}
 
Example 32
Project: StormCV   File: S3Connector.java   View source code 6 votes vote down vote up
@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 
Example 33
Project: s3proxy   File: AwsSdkTest.java   View source code 6 votes vote down vote up
@Test
public void testSpecialCharacters() throws Exception {
    String prefix = "special !\"#$%&'()*+,-./:;<=>[email protected][\\]^_`{|}~";
    if (blobStoreType.equals("azureblob") || blobStoreType.equals("b2")) {
        prefix = prefix.replace("\\", "");
    }
    if (blobStoreType.equals("azureblob")) {
        // Avoid blob names that end with a dot (.), a forward slash (/), or
        // a sequence or combination of the two.
        prefix = prefix.replace("./", "/") + ".";
    }
    String blobName = prefix + "foo";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);

    ObjectListing listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withPrefix(prefix));
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    assertThat(summaries).hasSize(1);
    S3ObjectSummary summary = summaries.iterator().next();
    assertThat(summary.getKey()).isEqualTo(blobName);
}
 
Example 34
Project: detective   File: S3ListTask.java   View source code 6 votes vote down vote up
@Override
protected void doExecute(Parameters config, Parameters output) {
  String bucketName = this.readAsString(config, "aws.s3.bucketName", null, false, "aws.s3.bucketName not present in config");
  String prefix = this.readAsString(config, "aws.s3.list.prefix", null, false, "aws.s3.list.prefix not present");
  
  AmazonS3 client = new AmazonS3Client(AwsUtils.getCredentialProviderC(config), AwsUtils.getClientConfig(config));
  client.setRegion(AwsUtils.getRegion(config));
  
  ListObjectsRequest listObjectsRequest =
      new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);

  List<S3ObjectSummary> summarys = new ArrayList<S3ObjectSummary>();
  ObjectListing objectListing;
  do {
    objectListing = client.listObjects(listObjectsRequest);
    summarys.addAll(objectListing.getObjectSummaries());
    String marker = objectListing.getNextMarker();
    listObjectsRequest.setMarker(marker);
    
    Detective.info("Reading from S3...current marker:" + marker + " Continue..." );
  } while (objectListing.isTruncated());
  
  output.put("s3ObjectSummaries", summarys);
}
 
Example 35
Project: BucketSyncer   File: S3KeyLister.java   View source code 6 votes vote down vote up
public S3KeyLister(Object client,
                   String bucket,
                   String prefix,
                   MirrorContext context,
                   Integer maxQueueCapacity) {
    super(bucket, prefix, context, maxQueueCapacity);
    this.s3Client = (AmazonS3Client) client;

    final MirrorOptions options = context.getOptions();
    int fetchSize = options.getMaxThreads();
    this.summaries = new ArrayList<S3ObjectSummary>(10 * fetchSize);

    final ListObjectsRequest request = new ListObjectsRequest(bucket, prefix, null, null, fetchSize);
    listing = s3getFirstBatch(s3Client, request);
    synchronized (summaries) {
        final List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
        summaries.addAll(objectSummaries);
        context.getStats().objectsRead.addAndGet(objectSummaries.size());
        if (options.isVerbose()) {
            log.info("added initial set of {} keys", objectSummaries.size());
        }
    }
}
 
Example 36
Project: esthree   File: Ls.java   View source code 6 votes vote down vote up
/**
 * Return the ObjectListing starting at the given marker. The number of
 * ObjectSummaries is bounded to be 1000 (this is the Amazon S3 default,
 * which we're also setting here explicitly to minimize future SDK update
 * implications).
 *
 * @param marker the last marker from a previous call or null
 * @param limit  return no more than this many
 */
public ObjectListing list(final String marker, final int limit) throws ExecutionException, RetryException {
    return (ObjectListing) RetryUtils.AWS_RETRYER.call(new Callable<Object>() {
        public ObjectListing call() throws Exception {

            ListObjectsRequest lor = new ListObjectsRequest()
                    .withBucketName(bucket)
                    .withMarker(marker)
                    .withDelimiter("/")
                    .withMaxKeys(limit);
            if (prefix != null) {
                lor.withPrefix(prefix);
            }
            return amazonS3Client.listObjects(lor);
        }
    });
}
 
Example 37
Project: DeployMan   File: FileMonitor.java   View source code 6 votes vote down vote up
private Map<String, String> getRemoteFiles(AmazonS3 client, String bucketName) {

    this.console.write("Get remote files..."); //$NON-NLS-1$

    ObjectListing listing;
    Map<String, String> files = new TreeMap<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);

    do {
      listing = client.listObjects(listObjectsRequest);
      for (S3ObjectSummary object : listing.getObjectSummaries())
        files.put(object.getKey(), object.getETag());
    } while (listing.isTruncated());

    this.console.write("Found " + files.size() + " remote files"); //$NON-NLS-1$ //$NON-NLS-2$
    return files;
  }
 
Example 38
Project: RxS3   File: AsyncS3Client.java   View source code 6 votes vote down vote up
public void listNextBatchOfObjects(ObjectListing objectListing, Subscriber<ObjectListing> observable) {
    if (!objectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(objectListing.getBucketName());
        emptyListing.setDelimiter(objectListing.getDelimiter());
        emptyListing.setMarker(objectListing.getNextMarker());
        emptyListing.setMaxKeys(objectListing.getMaxKeys());
        emptyListing.setPrefix(objectListing.getPrefix());
        emptyListing.setTruncated(false);

        observable.onNext(objectListing);
        observable.onCompleted();
    }

    listObjects(new ListObjectsRequest(
            objectListing.getBucketName(),
            objectListing.getPrefix(),
            objectListing.getNextMarker(),
            objectListing.getDelimiter(),
            objectListing.getMaxKeys()), observable);
}
 
Example 39
Project: RxS3   File: AsyncS3Client.java   View source code 6 votes vote down vote up
public Observable<ObjectListing> listNextBatchOfObjects(ObjectListing objectListing) {
    if (!objectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(objectListing.getBucketName());
        emptyListing.setDelimiter(objectListing.getDelimiter());
        emptyListing.setMarker(objectListing.getNextMarker());
        emptyListing.setMaxKeys(objectListing.getMaxKeys());
        emptyListing.setPrefix(objectListing.getPrefix());
        emptyListing.setTruncated(false);

        return Observable.just(emptyListing);
    }

    return listObjects(new ListObjectsRequest(
            objectListing.getBucketName(),
            objectListing.getPrefix(),
            objectListing.getNextMarker(),
            objectListing.getDelimiter(),
            objectListing.getMaxKeys()));
}
 
Example 40
Project: RxS3   File: AsyncS3ClientTest.java   View source code 6 votes vote down vote up
@Test
public void shouldListObjectsWhenUsingRequest() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/05/");

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing);
}