Java Code Examples for com.amazonaws.services.s3.model.ListObjectsRequest

The following are top voted examples for showing how to use com.amazonaws.services.s3.model.ListObjectsRequest. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ibm-cos-sdk-java   File: AmazonS3Client.java   Source Code and License 7 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws SdkClientException, AmazonServiceException {
    listObjectsRequest = beforeClientExecution(listObjectsRequest);
    rejectNull(listObjectsRequest.getBucketName(), "The bucket name parameter must be specified when listing objects in a bucket");

    /**
     * This flag shows whether we need to url decode S3 key names. This flag is enabled
     * only when the customers don't explicitly call {@link ListObjectsRequest#setEncodingType(String)},
     * otherwise, it will be disabled for maintaining backwards compatibility.
     */
    final boolean shouldSDKDecodeResponse = listObjectsRequest.getEncodingType() == null;

    Request<ListObjectsRequest> request = createRequest(listObjectsRequest.getBucketName(), null, listObjectsRequest, HttpMethodName.GET);
    addParameterIfNotNull(request, "prefix", listObjectsRequest.getPrefix());
    addParameterIfNotNull(request, "marker", listObjectsRequest.getMarker());
    addParameterIfNotNull(request, "delimiter", listObjectsRequest.getDelimiter());
    if (listObjectsRequest.getMaxKeys() != null && listObjectsRequest.getMaxKeys().intValue() >= 0) request.addParameter("max-keys", listObjectsRequest.getMaxKeys().toString());
    request.addParameter("encoding-type", shouldSDKDecodeResponse ? Constants.URL_ENCODING : listObjectsRequest.getEncodingType());

    return invoke(request, new Unmarshallers.ListObjectsUnmarshaller(shouldSDKDecodeResponse), listObjectsRequest.getBucketName(), null);
}
 
Example 2
Project: cmn-project   File: S3.java   Source Code and License 7 votes vote down vote up
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 
Example 3
Project: Reer   File: S3Client.java   Source Code and License 6 votes vote down vote up
public List<String> listDirectChildren(URI parent) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(parent);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(s3BucketKey)
        .withMaxKeys(1000)
        .withDelimiter("/");
    ObjectListing objectListing = amazonS3Client.listObjects(listObjectsRequest);
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    builder.addAll(resourceResolver.resolveResourceNames(objectListing));

    while (objectListing.isTruncated()) {
        objectListing = amazonS3Client.listNextBatchOfObjects(objectListing);
        builder.addAll(resourceResolver.resolveResourceNames(objectListing));
    }
    return builder.build();
}
 
Example 4
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 5
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 6
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 7
Project: circus-train   File: S3S3CopierTest.java   Source Code and License 6 votes vote down vote up
@Test
public void copyMultipleObjects() throws Exception {
  // Making sure we only request 1 file at the time so we need to loop
  ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
  when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));

  client.putObject("source", "bar/data1", inputData);
  client.putObject("source", "bar/data2", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(14L));

  S3Object object1 = client.getObject("target", "foo/data1");
  String data1 = IOUtils.toString(object1.getObjectContent());
  assertThat(data1, is("bar foo"));
  S3Object object2 = client.getObject("target", "foo/data2");
  String data2 = IOUtils.toString(object2.getObjectContent());
  assertThat(data2, is("bar foo"));
}
 
Example 8
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 9
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 10
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 11
Project: ibm-cos-sdk-java   File: S3Objects.java   Source Code and License 6 votes vote down vote up
private void prepareCurrentListing() {
    while ( currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated()) ) {

        if ( currentListing == null ) {
            ListObjectsRequest req = new ListObjectsRequest();
            req.setBucketName(getBucketName());
            req.setPrefix(getPrefix());
            req.setMaxKeys(getBatchSize());
            currentListing = getS3().listObjects(req);
        } else {
            currentListing = getS3().listNextBatchOfObjects(currentListing);
        }

        currentIterator = currentListing.getObjectSummaries().iterator();
    }
}
 
Example 12
Project: galaxy-fds-migration-tool   File: S3Source.java   Source Code and License 6 votes vote down vote up
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 
Example 13
Project: RekognitionS3Batch   File: Scanner.java   Source Code and License 6 votes vote down vote up
public void scanBucket() {
    ListObjectsRequest listReq = new ListObjectsRequest()
        .withPrefix(opt.prefix())
        .withBucketName(opt.bucket());

    Logger.Info("Scanning S3 bucket %s %s", opt.bucket(), opt.prefix());
    ObjectListing listing = s3.listObjects(listReq);
    boolean ok = processObjects(listing.getObjectSummaries());

    while (ok && listing.isTruncated()) {
        listing = s3.listNextBatchOfObjects(listing);
        ok = processObjects(listing.getObjectSummaries());
    }

    Logger.Info("Completed scan, added %s images to the processing queue.", numSeen.get());
}
 
Example 14
Project: ismartonline   File: AwsFileManager.java   Source Code and License 6 votes vote down vote up
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 
Example 15
Project: emodb   File: StashReaderTest.java   Source Code and License 6 votes vote down vote up
private Matcher<ListObjectsRequest> listObjectRequest(final String bucket, final String prefix, @Nullable final String marker) {
    return new BaseMatcher<ListObjectsRequest>() {
        @Override
        public boolean matches(Object item) {
            ListObjectsRequest request = (ListObjectsRequest) item;
            return request != null &&
                    request.getBucketName().equals(bucket) &&
                    request.getPrefix().equals(prefix) &&
                    Objects.equals(request.getMarker(), marker);
        }

        @Override
        public void describeTo(Description description) {
            description.appendText("ListObjectRequest[s3://").appendText(bucket).appendText("/").appendText(prefix);
            if (marker != null) {
                description.appendText(", marker=").appendText(marker);
            }
            description.appendText("]");
        }
    };
}
 
Example 16
Project: emodb   File: StashReaderTest.java   Source Code and License 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 17
Project: Camel   File: S3IncludeBodyTest.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 18
Project: Camel   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 19
Project: Camel   File: S3ConsumerPrefixTest.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 20
Project: exhibitor   File: S3PseudoLock.java   Source Code and License 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 21
Project: s3_video   File: AWSAdapter.java   Source Code and License 6 votes vote down vote up
public List<String> listKeysInOutputBucket(String bucketName) {
	List<String> videoKeys = new ArrayList<>();
	
	ListObjectsRequest listFirstLevelKeyRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withDelimiter("/");
	
	for (String commonPrefix : s3client.listObjects(listFirstLevelKeyRequest).getCommonPrefixes()) {
		
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withPrefix(commonPrefix)
		.withDelimiter("/");
		
		for (S3ObjectSummary file : s3client.listObjects(listObjectsRequest).getObjectSummaries()) {
			videoKeys.add(file.getKey());
		}
	}
	return videoKeys;		
}
 
Example 22
Project: micro-server   File: S3Utils.java   Source Code and License 6 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 23
Project: micro-server   File: S3UtilsTest.java   Source Code and License 6 votes vote down vote up
@Test
public void getAllSummaries() {
    answer = true;
    AmazonS3Client client = mock(AmazonS3Client.class);
    ObjectListing objectListing = mock(ObjectListing.class);
    when(client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
    when(objectListing.isTruncated()).thenAnswer(__ -> {
        try {
            return answer;
        } finally {
            answer = false;
        }
    });
    S3Utils utils = new S3Utils(
                                client, null, null, false, null);
    utils.getAllSummaries(new ListObjectsRequest());
    verify(objectListing, times(2)).getObjectSummaries();
}
 
Example 24
Project: zeppelin   File: S3NotebookRepo.java   Source Code and License 6 votes vote down vote up
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 
Example 25
Project: zeppelin   File: S3NotebookRepo.java   Source Code and License 6 votes vote down vote up
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
Example 26
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 27
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 28
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public List<String> listBucket(String bkt, String prefix, String delimiter) throws IOException {

        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(bkt);
        listRequest.setDelimiter(delimiter);
        listRequest.setPrefix(prefix);

        ObjectListing listing = s3.listObjects(listRequest);

        ArrayList<String> list = new ArrayList<String>();

        for (S3ObjectSummary summ : listing.getObjectSummaries()) {
            list.add(summ.getKey());
        }

        return list;
    }
 
Example 29
Project: digdag   File: TestUtils.java   Source Code and License 6 votes vote down vote up
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 
Example 30
Project: datacollector   File: S3ConnectionSourceConfig.java   Source Code and License 6 votes vote down vote up
private void validateConnection(
    Stage.Context context,
    String configPrefix,
    List<Stage.ConfigIssue> issues
) {
  try {
    //check if the credentials are right by trying to list an object in the common prefix
    getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
  } catch (AmazonS3Exception e) {
    LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
    issues.add(
        context.createConfigIssue(
            Groups.S3.name(),
            configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
            Errors.S3_SPOOLDIR_20,
            e.toString()
        )
    );
  }
}
 
Example 31
Project: flow   File: S3Filer.java   Source Code and License 6 votes vote down vote up
@Override
@Nonnull
public Record getRecord(@Nonnull String path) throws IOException {
	ObjectListing listing = s3.listObjects(
			new ListObjectsRequest().withBucketName(bucket).withPrefix(path.substring(1)));

	S3ObjectSummary summary = listing.getObjectSummaries().stream().findFirst().orElse(null);
	if (summary == null) {
		return Record.noFile(uri, path);
	}

	long time = summary.getLastModified().getTime();
	long size = summary.getSize();
	boolean directory = summary.getKey().endsWith("/");
	return new Record(uri, RecordPath.from("/" + summary.getKey()), time, size, directory);
}
 
Example 32
Project: StormCV   File: S3Connector.java   Source Code and License 6 votes vote down vote up
@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 
Example 33
Project: s3proxy   File: AwsSdkTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testSpecialCharacters() throws Exception {
    String prefix = "special !\"#$%&'()*+,-./:;<=>[email protected][\\]^_`{|}~";
    if (blobStoreType.equals("azureblob") || blobStoreType.equals("b2")) {
        prefix = prefix.replace("\\", "");
    }
    if (blobStoreType.equals("azureblob")) {
        // Avoid blob names that end with a dot (.), a forward slash (/), or
        // a sequence or combination of the two.
        prefix = prefix.replace("./", "/") + ".";
    }
    String blobName = prefix + "foo";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);

    ObjectListing listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withPrefix(prefix));
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    assertThat(summaries).hasSize(1);
    S3ObjectSummary summary = summaries.iterator().next();
    assertThat(summary.getKey()).isEqualTo(blobName);
}
 
Example 34
Project: detective   File: S3ListTask.java   Source Code and License 6 votes vote down vote up
@Override
protected void doExecute(Parameters config, Parameters output) {
  String bucketName = this.readAsString(config, "aws.s3.bucketName", null, false, "aws.s3.bucketName not present in config");
  String prefix = this.readAsString(config, "aws.s3.list.prefix", null, false, "aws.s3.list.prefix not present");
  
  AmazonS3 client = new AmazonS3Client(AwsUtils.getCredentialProviderC(config), AwsUtils.getClientConfig(config));
  client.setRegion(AwsUtils.getRegion(config));
  
  ListObjectsRequest listObjectsRequest =
      new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);

  List<S3ObjectSummary> summarys = new ArrayList<S3ObjectSummary>();
  ObjectListing objectListing;
  do {
    objectListing = client.listObjects(listObjectsRequest);
    summarys.addAll(objectListing.getObjectSummaries());
    String marker = objectListing.getNextMarker();
    listObjectsRequest.setMarker(marker);
    
    Detective.info("Reading from S3...current marker:" + marker + " Continue..." );
  } while (objectListing.isTruncated());
  
  output.put("s3ObjectSummaries", summarys);
}
 
Example 35
Project: BucketSyncer   File: S3KeyLister.java   Source Code and License 6 votes vote down vote up
public S3KeyLister(Object client,
                   String bucket,
                   String prefix,
                   MirrorContext context,
                   Integer maxQueueCapacity) {
    super(bucket, prefix, context, maxQueueCapacity);
    this.s3Client = (AmazonS3Client) client;

    final MirrorOptions options = context.getOptions();
    int fetchSize = options.getMaxThreads();
    this.summaries = new ArrayList<S3ObjectSummary>(10 * fetchSize);

    final ListObjectsRequest request = new ListObjectsRequest(bucket, prefix, null, null, fetchSize);
    listing = s3getFirstBatch(s3Client, request);
    synchronized (summaries) {
        final List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
        summaries.addAll(objectSummaries);
        context.getStats().objectsRead.addAndGet(objectSummaries.size());
        if (options.isVerbose()) {
            log.info("added initial set of {} keys", objectSummaries.size());
        }
    }
}
 
Example 36
Project: esthree   File: Ls.java   Source Code and License 6 votes vote down vote up
/**
 * Return the ObjectListing starting at the given marker. The number of
 * ObjectSummaries is bounded to be 1000 (this is the Amazon S3 default,
 * which we're also setting here explicitly to minimize future SDK update
 * implications).
 *
 * @param marker the last marker from a previous call or null
 * @param limit  return no more than this many
 */
public ObjectListing list(final String marker, final int limit) throws ExecutionException, RetryException {
    return (ObjectListing) RetryUtils.AWS_RETRYER.call(new Callable<Object>() {
        public ObjectListing call() throws Exception {

            ListObjectsRequest lor = new ListObjectsRequest()
                    .withBucketName(bucket)
                    .withMarker(marker)
                    .withDelimiter("/")
                    .withMaxKeys(limit);
            if (prefix != null) {
                lor.withPrefix(prefix);
            }
            return amazonS3Client.listObjects(lor);
        }
    });
}
 
Example 37
Project: DeployMan   File: FileMonitor.java   Source Code and License 6 votes vote down vote up
private Map<String, String> getRemoteFiles(AmazonS3 client, String bucketName) {

    this.console.write("Get remote files..."); //$NON-NLS-1$

    ObjectListing listing;
    Map<String, String> files = new TreeMap<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);

    do {
      listing = client.listObjects(listObjectsRequest);
      for (S3ObjectSummary object : listing.getObjectSummaries())
        files.put(object.getKey(), object.getETag());
    } while (listing.isTruncated());

    this.console.write("Found " + files.size() + " remote files"); //$NON-NLS-1$ //$NON-NLS-2$
    return files;
  }
 
Example 38
Project: RxS3   File: AsyncS3Client.java   Source Code and License 6 votes vote down vote up
public void listNextBatchOfObjects(ObjectListing objectListing, Subscriber<ObjectListing> observable) {
    if (!objectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(objectListing.getBucketName());
        emptyListing.setDelimiter(objectListing.getDelimiter());
        emptyListing.setMarker(objectListing.getNextMarker());
        emptyListing.setMaxKeys(objectListing.getMaxKeys());
        emptyListing.setPrefix(objectListing.getPrefix());
        emptyListing.setTruncated(false);

        observable.onNext(objectListing);
        observable.onCompleted();
    }

    listObjects(new ListObjectsRequest(
            objectListing.getBucketName(),
            objectListing.getPrefix(),
            objectListing.getNextMarker(),
            objectListing.getDelimiter(),
            objectListing.getMaxKeys()), observable);
}
 
Example 39
Project: RxS3   File: AsyncS3Client.java   Source Code and License 6 votes vote down vote up
public Observable<ObjectListing> listNextBatchOfObjects(ObjectListing objectListing) {
    if (!objectListing.isTruncated()) {
        ObjectListing emptyListing = new ObjectListing();
        emptyListing.setBucketName(objectListing.getBucketName());
        emptyListing.setDelimiter(objectListing.getDelimiter());
        emptyListing.setMarker(objectListing.getNextMarker());
        emptyListing.setMaxKeys(objectListing.getMaxKeys());
        emptyListing.setPrefix(objectListing.getPrefix());
        emptyListing.setTruncated(false);

        return Observable.just(emptyListing);
    }

    return listObjects(new ListObjectsRequest(
            objectListing.getBucketName(),
            objectListing.getPrefix(),
            objectListing.getNextMarker(),
            objectListing.getDelimiter(),
            objectListing.getMaxKeys()));
}
 
Example 40
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldListObjectsWhenUsingRequest() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/05/");

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing);
}
 
Example 41
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldListObjectBatchesWhenStartingWithARequest() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/06/");

    // When & Then
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    while (amazonListing.isTruncated()) {
        assertThat(listing).isEqualTo(amazonListing);
        listing = client.listNextBatchOfObjects(listing.toBlocking().single());
        amazonListing = amazonS3Client.listNextBatchOfObjects(amazonListing);
    }

    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing).isNotTruncated();
}
 
Example 42
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldListObjectWithMaxKeysLimit() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/");
    request.setMaxKeys(2);

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isTruncated()
            .hasSize(2);
}
 
Example 43
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldListObjectBatchesWhenUsingRequest() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setMaxKeys(2);
    request.setPrefix("COUNTRY_BY_DATE/2014/");

    // When & Then
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    while (amazonListing.isTruncated()) {
        assertThat(listing)
                .ignoreFields(fieldsToIgnore)
                .isEqualTo(amazonListing);

        listing = client.listNextBatchOfObjects(listing.toBlocking().single());
        amazonListing = amazonS3Client.listNextBatchOfObjects(amazonListing);
    }

    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();
}
 
Example 44
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldReturnEmptyListingWhenNotTruncated() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/");

    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();

    // When
    listing = client.listNextBatchOfObjects(listing.toBlocking().single());
    amazonListing = amazonS3Client.listNextBatchOfObjects(amazonListing);

    // Then
    assertThat(listing).isEqualTo(amazonListing).isNotNull();
}
 
Example 45
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldListCommonPrefixes_ContainingFiles() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/05/");
    request.setDelimiter("/");

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();
}
 
Example 46
Project: RxS3   File: AsyncS3ClientTest.java   Source Code and License 6 votes vote down vote up
@Test
public void shouldListCommonPrefixes_ContainingDirectories() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/");
    request.setDelimiter("/");

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();
}
 
Example 47
Project: sequenceiq-samples   File: S3Source.java   Source Code and License 6 votes vote down vote up
@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 
Example 48
Project: elasticsearch-lambda   File: S3SnapshotTransport.java   Source Code and License 6 votes vote down vote up
@Override
protected boolean checkExists(String destination, Integer shard) throws IOException {
	// Break that s3 path into bucket & key 
	String[] pieces = StringUtils.split(destination, "/");
	String bucket = pieces[0];
	String key = destination.substring(bucket.length() + 1);
	
	// AWS SDK doesn't have an "exists" method so you have to list and check if the key is there. Thanks Obama
	ObjectListing objects = tx.getAmazonS3Client().listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));

	for (S3ObjectSummary objectSummary: objects.getObjectSummaries()) {
		if (objectSummary.getKey().startsWith(key + shard)) {
			return true;
		}
	}
	return false;
}
 
Example 49
Project: spring-cloud-aws   File: PathMatchingSimpleStorageResourcePatternResolver.java   Source Code and License 6 votes vote down vote up
private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources, String prefix, String keyPattern) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
    ObjectListing objectListing = null;

    do {
        try {
            if (objectListing == null) {
                objectListing = this.amazonS3.listObjects(listObjectsRequest);
            } else {
                objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
            }
            Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern, objectListing.getObjectSummaries());
            if (!newResources.isEmpty()) {
                resources.addAll(newResources);
            }
        } catch (AmazonS3Exception e) {
            if (301 != e.getStatusCode()) {
                throw e;
            }
        }
    } while (objectListing != null && objectListing.isTruncated());
}
 
Example 50
Project: spring-cloud-aws   File: PathMatchingSimpleStorageResourcePatternResolver.java   Source Code and License 6 votes vote down vote up
/**
 * Searches for matching keys progressively. This means that instead of retrieving all keys given a prefix, it goes
 * down one level at a time and filters out all non-matching results. This avoids a lot of unused requests results.
 * WARNING: This method does not truncate results. Therefore all matching resources will be returned regardless of
 * the truncation.
 */
private void findProgressivelyWithPartialMatch(String bucketName, Set<Resource> resources, String prefix, String keyPattern) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withDelimiter("/").withPrefix(prefix);
    ObjectListing objectListing = null;

    do {
        if (objectListing == null) {
            objectListing = this.amazonS3.listObjects(listObjectsRequest);
        } else {
            objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
        }

        Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName, keyPattern, objectListing.getObjectSummaries());
        if (!newResources.isEmpty()) {
            resources.addAll(newResources);
        }

        for (String commonPrefix : objectListing.getCommonPrefixes()) {
            if (isKeyPathMatchesPartially(keyPattern, commonPrefix)) {
                findPathMatchingKeyInBucket(bucketName, resources, commonPrefix, keyPattern);
            }
        }
    } while (objectListing.isTruncated());
}
 
Example 51
Project: extension_amazon_s3_repository   File: AmazonS3RepositoryTest.java   Source Code and License 6 votes vote down vote up
private void runListing(AmazonS3Client client, ListObjectsRequest req) {
	ObjectListing listObjects = client.listObjects(req);
	while (listObjects != null) {
		List<S3ObjectSummary> objectSummaries = listObjects
				.getObjectSummaries();
		for (S3ObjectSummary objectSummary : objectSummaries) {
			String key = objectSummary.getKey();
			System.out.println("key: " + key + " - "
					+ objectSummary.getStorageClass());
		}
		if (listObjects.isTruncated()) {
			listObjects = client.listNextBatchOfObjects(listObjects);
		} else {
			listObjects = null;
		}
	}
}
 
Example 52
Project: s3tool   File: Transporter.java   Source Code and License 6 votes vote down vote up
/**
     * list
     * @param region
     * @param bucketName
     * @param uri
     * @return
     * @throws AmazonServiceException
     * @throws AmazonClientException
     * @throws ConfigException
     */
    public static List<FileInfo> list(String region, String bucketName, String uri, boolean recursive)
            throws AmazonServiceException, AmazonClientException, ConfigException{
//        ObjectListing objects = getClient(region).listObjects(bucketName, uri);
        ListObjectsRequest request = new ListObjectsRequest()
        .withBucketName(bucketName).withPrefix(uri).withDelimiter("/");
        ObjectListing objects = getClient(region).listObjects(request);
        List<FileInfo> files = new ArrayList<FileInfo>();
        for(S3ObjectSummary summary : objects.getObjectSummaries()){
            String name = summary.getKey();
//            if(null!=name && (recursive || !name.contains("/"))){
                FileInfo finfo = new FileInfo();
                finfo.name = summary.getKey();
                finfo.type =  summary.getStorageClass();
                finfo.size =  summary.getSize();
                finfo.date = summary.getLastModified();
                files.add(finfo);
//            }
        }
        return files;
    }
 
Example 53
Project: jcabi-beanstalk-maven-plugin   File: OverridingBundleTest.java   Source Code and License 6 votes vote down vote up
/**
 * OverridingBundle can override a file in AWS S3.
 * @throws Exception If something is wrong
 */
@Test
public void overridesFileInAws() throws Exception {
    final String bucket = "some-bucket";
    final String key = "some-key";
    final File war = this.temp.newFile("temp.war");
    FileUtils.writeStringToFile(war, "broken JAR file content");
    final AmazonS3 client = Mockito.mock(AmazonS3.class);
    Mockito.doReturn(new PutObjectResult())
        .when(client).putObject(bucket, key, war);
    Mockito.doReturn(new ObjectListing())
        .when(client).listObjects(Mockito.any(ListObjectsRequest.class));
    final Bundle bundle = new OverridingBundle(client, bucket, key, war);
    MatcherAssert.assertThat(
        bundle.name(),
        Matchers.equalTo(key)
    );
    MatcherAssert.assertThat(
        bundle.location().getS3Key(),
        Matchers.equalTo(key)
    );
}
 
Example 54
Project: jcabi-beanstalk-maven-plugin   File: OverridingBundleTest.java   Source Code and License 6 votes vote down vote up
/**
 * OverridingBundle caches result of location() method.
 * @throws Exception If something is wrong
 */
@Test
public void cachesResultOfLocation() throws Exception {
    final AmazonS3 client = Mockito.mock(AmazonS3.class);
    final File war = this.temp.newFile("temp1.war");
    FileUtils.writeStringToFile(war, "some JAR file content");
    final String bucket = "some-bucket-for-cache";
    final String key = "some-key-for-cache";
    Mockito.doReturn(new PutObjectResult())
        .when(client).putObject(bucket, key, war);
    Mockito.doReturn(new ObjectListing())
        .when(client).listObjects(Mockito.any(ListObjectsRequest.class));
    final Bundle bundle = new OverridingBundle(client, bucket, key, war);
    bundle.location();
    bundle.location();
    Mockito.verify(client, Mockito.times(1)).putObject(bucket, key, war);
}
 
Example 55
Project: Scribengin   File: S3FeaturesDemoTest.java   Source Code and License 6 votes vote down vote up
public void listingObjects(String bucketName, String key) throws AmazonServiceException {
  /**
   * List objects in your bucket by prefix - There are many options for
   * listing the objects in your bucket. Keep in mind that buckets with many
   * objects might truncate their results when listing their objects, so be
   * sure to check if the returned object listing is truncated, and use the
   * AmazonS3.listNextBatchOfObjects(...) operation to retrieve additional
   * results.
   */
  System.out.println("Listing objects");
  ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucketName).withPrefix("My");
  ObjectListing objectListing = s3Client.listObjects(request);
  for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
    System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")");
  }
  System.out.println();
}
 
Example 56
Project: snomed-release-service   File: BuildDAOImplTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testFindAllDesc() {
	final ObjectListing objectListing = new ObjectListing();
	addObjectSummary(objectListing, "international/" + product.getBusinessKey() + "/2014-02-04T10:30:01/configuration.json");
	addObjectSummary(objectListing, "international/" + product.getBusinessKey() + "/2014-02-04T10:30:01/status:BEFORE_TRIGGER");
	addObjectSummary(objectListing, "international/" + product.getBusinessKey() + "/2014-03-04T10:30:01/configuration.json");
	addObjectSummary(objectListing, "international/" + product.getBusinessKey() + "/2014-03-04T10:30:01/status:BEFORE_TRIGGER");
	EasyMock.expect(mockS3Client.listObjects(EasyMock.isA(ListObjectsRequest.class))).andReturn(objectListing);

	mocksControl.replay();
	final List<Build> all = buildDAO.findAllDesc(product);
	mocksControl.verify();

	Assert.assertEquals(2, all.size());

	Assert.assertEquals("2014-03-04T10:30:01", all.get(0).getCreationTime());
	Assert.assertEquals(Build.Status.BEFORE_TRIGGER, all.get(0).getStatus());

	Assert.assertEquals("2014-02-04T10:30:01", all.get(1).getCreationTime());
	Assert.assertEquals(Build.Status.BEFORE_TRIGGER, all.get(1).getStatus());
}
 
Example 57
Project: snomed-release-service   File: BuildDAOImplTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testFindOne() {
	final String buildId = "2014-02-04T10:30:01";
	final ObjectListing objectListing = new ObjectListing();
	addObjectSummary(objectListing, "international/" + product.getBusinessKey() + "/" + buildId + "/configuration.json");
	addObjectSummary(objectListing, "international/" + product.getBusinessKey() + "/" + buildId + "/status:BEFORE_TRIGGER");
	final Capture<ListObjectsRequest> listObjectsRequestCapture = new Capture<>();
	EasyMock.expect(mockS3Client.listObjects(EasyMock.capture(listObjectsRequestCapture))).andReturn(objectListing);

	mocksControl.replay();
	final Build foundBuild = buildDAO.find(product, buildId);
	mocksControl.verify();

	Assert.assertNotNull(foundBuild);

	Assert.assertEquals("2014-02-04T10:30:01", foundBuild.getCreationTime());
	Assert.assertEquals(Build.Status.BEFORE_TRIGGER, foundBuild.getStatus());
}
 
Example 58
Project: elasticsearch_my   File: MockAmazonS3.java   Source Code and License 5 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    MockObjectListing list = new MockObjectListing();
    list.setTruncated(false);

    String blobName;
    String prefix = listObjectsRequest.getPrefix();

    ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();

    for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
        blobName = blob.getKey();
        S3ObjectSummary objectSummary = new S3ObjectSummary();

        if (prefix.isEmpty() || blobName.startsWith(prefix)) {
            objectSummary.setKey(blobName);

            try {
                objectSummary.setSize(getSize(blob.getValue()));
            } catch (IOException e) {
                throw new AmazonS3Exception("Object listing " +
                        "failed for blob [" + blob.getKey() + "]");
            }

            mockObjectSummaries.add(objectSummary);
        }
    }

    list.setObjectSummaries(mockObjectSummaries);
    return list;
}
 
Example 59
Project: circus-train   File: S3S3Copier.java   Source Code and License 5 votes vote down vote up
private void copy(AmazonS3URI source, AmazonS3URI target) {
  ListObjectsRequest request = listObjectsRequestFactory.newInstance().withBucketName(source.getBucket()).withPrefix(
      source.getKey());
  ObjectListing listing = srcClient.listObjects(request);
  submitCopyJobsFromListing(source, target, request, listing);
  while (listing.isTruncated()) {
    listing = srcClient.listNextBatchOfObjects(listing);
    submitCopyJobsFromListing(source, target, request, listing);
  }
}
 
Example 60
Project: circus-train   File: S3S3Copier.java   Source Code and License 5 votes vote down vote up
private void submitCopyJobsFromListing(
    AmazonS3URI sourceS3Uri,
    final AmazonS3URI targetS3Uri,
    ListObjectsRequest request,
    ObjectListing listing) {
  LOG.debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
      request.getPrefix());
  List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
  for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
    String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
    final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
    LOG.info("copying object from '{}/{}' to '{}/{}'", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(),
        targetS3Uri.getBucket(), targetKey);

    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
        s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);

    TransferStateChangeListener stateChangeListener = new TransferStateChangeListener() {

      @Override
      public void transferStateChanged(Transfer transfer, TransferState state) {
        if (state == TransferState.Completed) {
          // NOTE: running progress doesn't seem to be reported correctly.
          // transfer.getProgress().getBytesTransferred() is always 0. Unsure what is the cause of this at this moment
          // so just printing total bytes when completed.
          LOG.debug("copied object from '{}/{}' to '{}/{}': {} bytes transferred", s3ObjectSummary.getBucketName(),
              s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey,
              transfer.getProgress().getTotalBytesToTransfer());
        }
      }
    };
    Copy copy = transferManager.copy(copyObjectRequest, srcClient, stateChangeListener);
    totalBytesToReplicate += copy.getProgress().getTotalBytesToTransfer();
    copyJobs.add(copy);
  }
}