Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary

The following are top voted examples for showing how to use com.amazonaws.services.s3.model.S3ObjectSummary. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: cmn-project   File: S3.java   Source Code and License 7 votes vote down vote up
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 
Example 2
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 3
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 4
Project: syndesis   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 5
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 6
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 7
Project: connectors   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 8
Project: github-bucket   File: RepositoryS3.java   Source Code and License 6 votes vote down vote up
private boolean isUploadFile(Iterator<S3ObjectSummary> iter, String path, String hash) {
    while (iter.hasNext()) {
        S3ObjectSummary fileS3 = iter.next();
        // Filename should look like this:
        // a/b
        if (!fileS3.getKey().equals(path)) {
            // If this is another file, then continue!
            continue;
        }
        // Remove the file from the S3 list as it does not need to be processed further
        iter.remove();
        // Upload if the hashes differ
        return StringUtils.isNullOrEmpty(hash) || !fileS3.getETag().equals(hash);
    }
    return true;
}
 
Example 9
Project: nifi-minifi   File: S3CacheFileInfoImpl.java   Source Code and License 6 votes vote down vote up
@Override
public Stream<WriteableConfiguration> getCachedConfigurations() throws IOException {

  Iterable<S3ObjectSummary> objectSummaries = S3Objects.withPrefix(s3, bucket, prefix);
  Stream<S3ObjectSummary> objectStream = StreamSupport.stream(objectSummaries.spliterator(), false);

  return objectStream.map(p -> {
    Integer version = getVersionIfMatch(p.getKey());
    if (version == null) {
      return null;
    }
    return new Pair<>(version, p);
  }).filter(Objects::nonNull)
      .sorted(Comparator.comparing(pair -> ((Pair<Integer, S3ObjectSummary>) pair).getFirst())
            .reversed()).map(pair -> new S3WritableConfiguration(s3, pair.getSecond(), Integer.toString(pair.getFirst())));

}
 
Example 10
Project: galaxy-fds-migration-tool   File: S3Source.java   Source Code and License 6 votes vote down vote up
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 
Example 11
Project: RekognitionS3Batch   File: Scanner.java   Source Code and License 6 votes vote down vote up
private boolean processObjects(List<S3ObjectSummary> objects) {
    Logger.Debug("Scanning next batch of %s ", objects.size());
    objects
        .parallelStream()
        .filter(this::shouldEnqueue)
        .forEach(object -> {
            numSeen.incrementAndGet();
            String path = object.getBucketName() + "/" + object.getKey();
            Logger.Info("Posting: %s", path);

            SendMessageRequest msg = new SendMessageRequest()
                .withQueueUrl(queueUrl)
                .withMessageBody(path);
            sqs.sendMessage(msg);
        });
    if (max > -1L && numSeen.incrementAndGet() > max) {
        Logger.Info("Added max jobs, quitting");
        return false;
    }
    return true;
}
 
Example 12
Project: ismartonline   File: AwsFileManager.java   Source Code and License 6 votes vote down vote up
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 
Example 13
Project: vertx-pairtree   File: AbstractS3IT.java   Source Code and License 6 votes vote down vote up
/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}
 
Example 14
Project: cloudkeeper   File: S3Utilities.java   Source Code and License 6 votes vote down vote up
static void deletePrefix(AmazonS3 s3Client, String bucketName, String prefix) {
    List<DeleteObjectsRequest.KeyVersion> trashKeys = new ArrayList<>();
    DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName);
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, prefix)) {
        trashKeys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        if (trashKeys.size() == BULK_DELETE_SIZE) {
            deleteObjectsRequest.setKeys(trashKeys);
            s3Client.deleteObjects(deleteObjectsRequest);
            trashKeys.clear();
        }
    }
    if (!trashKeys.isEmpty()) {
        deleteObjectsRequest.setKeys(trashKeys);
        s3Client.deleteObjects(deleteObjectsRequest);
    }
}
 
Example 15
Project: cloudkeeper   File: S3StagingArea.java   Source Code and License 6 votes vote down vote up
@Override
protected void copy(RuntimeExecutionTrace source, RuntimeExecutionTrace target,
        RuntimeAnnotatedExecutionTrace absoluteSource, RuntimeAnnotatedExecutionTrace absoluteTarget)
        throws IOException {
    String sourcePrefix = toS3Path(source).getPrefixForChildren();
    String targetPrefix = toS3Path(target).getPrefixForChildren();
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, sourcePrefix)) {
        if (summary.getKey().startsWith(sourcePrefix)) {
            String relativeKey = summary.getKey().substring(sourcePrefix.length());
            s3Client.copyObject(bucketName, summary.getKey(), bucketName, targetPrefix + relativeKey);
        } else {
            log.error(String.format(
                "S3Objects.withPrefix() returned unexpected key '%s' when asked for prefix '%s'.",
                summary.getKey(), sourcePrefix
            ));
        }
    }
}
 
Example 16
Project: javapasswordsafe   File: S3FileTest.java   Source Code and License 6 votes vote down vote up
@Override
protected void tearDown() throws Exception {
	final File theFile = new File(TEST_FILE_NAME);
	theFile.delete();

	if (s3Account != null) {
		final AmazonS3Client s3 = new AmazonS3Client(
				new BasicAWSCredentials(s3Account.keyId, s3Account.secretKey));
		String hash = s3Account.getHashedName();
		if (s3.doesBucketExist(hash)) {
			for (S3ObjectSummary summary : s3.listObjects(hash).getObjectSummaries()) {
				s3.deleteObject(s3Account.getHashedName(), summary.getKey());
			}

		}
	}

	super.tearDown();
}
 
Example 17
Project: etcetera   File: S3ConfigBackend.java   Source Code and License 6 votes vote down vote up
@Override public Collection<String> getFilenames() {
    Collection<String> names = new ArrayList<>();
    final ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket);
    ListObjectsV2Result listing;
    do {
      listing = client.listObjectsV2(req);
      names.addAll(listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(Collectors.toList()));
    } while (listing.isTruncated());

    return names;
}
 
Example 18
Project: emodb   File: StashReader.java   Source Code and License 6 votes vote down vote up
/**
 * Gets the metadata for a single table in this stash.  This is similar to getting the splits for the table
 * except that it exposes lower level information about the underlying S3 files.  For clients who will use
 * their own system for reading the files from S3, such as source files for a map-reduce job, this method provides
 * the necessary information.  For simply iterating over the stash contents using either {@link #scan(String)}
 * or {@link #getSplits(String)} in conjunction with {@link #getSplit(StashSplit)} is preferred.
 */
public StashTableMetadata getTableMetadata(String table)
        throws StashNotAvailableException, TableNotStashedException {
    ImmutableList.Builder<StashFileMetadata> filesBuilder = ImmutableList.builder();

    Iterator<S3ObjectSummary> objectSummaries = getS3ObjectSummariesForTable(table);
    while (objectSummaries.hasNext()) {
        S3ObjectSummary objectSummary = objectSummaries.next();
        filesBuilder.add(new StashFileMetadata(_bucket, objectSummary.getKey(), objectSummary.getSize()));
    }

    List<StashFileMetadata> files = filesBuilder.build();

    // Get the prefix arbitrarily from the first file.
    String prefix = files.get(0).getKey();
    prefix = prefix.substring(0, prefix.lastIndexOf('/') + 1);

    return new StashTableMetadata(_bucket, prefix, table, files);
}
 
Example 19
Project: emodb   File: StashReaderTest.java   Source Code and License 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 20
Project: kafka-connect-storage-cloud   File: TestWithMockedS3.java   Source Code and License 6 votes vote down vote up
public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}
 
Example 21
Project: hawkbit-extensions   File: S3Repository.java   Source Code and License 6 votes vote down vote up
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 
Example 22
Project: modeshape-s3-binary-store   File: S3BinaryStore.java   Source Code and License 6 votes vote down vote up
@Override
public Iterable<BinaryKey> getAllBinaryKeys() throws BinaryStoreException {
    try {
        final Iterator<S3ObjectSummary> objectsIterator =
            S3Objects.inBucket(s3Client, bucketName).iterator();
        // Lambda to hand back BinaryKeys rather than S3ObjectSummaries
        return () -> {
            return new Iterator<BinaryKey>() {
                @Override
                public boolean hasNext() {
                    return objectsIterator.hasNext();
                }

                @Override
                public BinaryKey next() {
                    S3ObjectSummary object = objectsIterator.next();
                    return new BinaryKey(object.getKey());
                }
            };
        };
    } catch (AmazonClientException e) {
        throw new BinaryStoreException(e);
    }
}
 
Example 23
Project: ecs-samples   File: _99_DeleteBucket.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  	// create the AWS S3 Client
AmazonS3 s3 = AWSS3Factory.getS3Client();

      // delete the demo bucket and all its content
for (S3ObjectSummary summary : s3.listObjects(AWSS3Factory.S3_BUCKET).getObjectSummaries())
{
	s3.deleteObject(AWSS3Factory.S3_BUCKET, summary.getKey());
	System.out.println(String.format("Deleted [%s/%s]", AWSS3Factory.S3_BUCKET, summary.getKey()));
}
s3.deleteBucket(AWSS3Factory.S3_BUCKET);

  	            // print bucket key/value and content for validation
  	System.out.println( String.format("deleted bucket [%s]",
  			AWSS3Factory.S3_BUCKET));
  }
 
Example 24
Project: aws-doc-sdk-examples   File: ListObjects.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args)
{
    final String USAGE = "\n" +
        "To run this example, supply the name of a bucket to list!\n" +
        "\n" +
        "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    ObjectListing ol = s3.listObjects(bucket_name);
    List<S3ObjectSummary> objects = ol.getObjectSummaries();
    for (S3ObjectSummary os: objects) {
        System.out.println("* " + os.getKey());
    }
}
 
Example 25
Project: cerberus-lifecycle-cli   File: S3StoreServiceTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testGetKeysInPartialPath() {
    AmazonS3 client = mock(AmazonS3.class);
    S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX);

    String path = "path";
    String key = "my-key";

    S3ObjectSummary summary = new S3ObjectSummary();
    summary.setKey(S3_PREFIX + key);

    ObjectListing listing = mock(ObjectListing.class);

    when(listing.getObjectSummaries()).thenReturn(Lists.newArrayList(summary));
    when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing);

    // invoke method under test
    Set<String> results = service.getKeysInPartialPath(path);

    assertEquals(1, results.size());
    assertEquals(key, results.iterator().next());
}
 
Example 26
Project: lumber-mill   File: S3ScheduledPoll.java   Source Code and License 6 votes vote down vote up
private void subscribe(S3ObjectSummary os, Observable<? extends Object> result) {

        result.subscribe(new Subscriber<Object>() {
            @Override
            public void onCompleted() {
                if (s3ObjectsInPipeline.remove(os)) {
                    processingState.completed(os);
                    release();
                }
            }

            @Override
            public void onError(Throwable e) {
                if (s3ObjectsInPipeline.remove(os)) {
                    release();
                }

                if (LOGGER.isTraceEnabled()) e.printStackTrace();
            }

            @Override
            public void onNext(Object event) { }
        });

    }
 
Example 27
Project: Camel   File: S3IncludeBodyTest.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 28
Project: Camel   File: AmazonS3ClientMock.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 29
Project: Camel   File: S3ConsumerPrefixTest.java   Source Code and License 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 30
Project: exhibitor   File: S3PseudoLock.java   Source Code and License 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 31
Project: s3_video   File: AWSAdapter.java   Source Code and License 6 votes vote down vote up
public List<String> listKeysInOutputBucket(String bucketName) {
	List<String> videoKeys = new ArrayList<>();
	
	ListObjectsRequest listFirstLevelKeyRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withDelimiter("/");
	
	for (String commonPrefix : s3client.listObjects(listFirstLevelKeyRequest).getCommonPrefixes()) {
		
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withPrefix(commonPrefix)
		.withDelimiter("/");
		
		for (S3ObjectSummary file : s3client.listObjects(listObjectsRequest).getObjectSummaries()) {
			videoKeys.add(file.getKey());
		}
	}
	return videoKeys;		
}
 
Example 32
Project: WeatherPipe   File: AWSInterfaceTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testListBucket() {
	final String dataBucket = "noaa-nexrad-level2";
	String[] key = {"2010/01/01", "2010/07/14"};
	String jobID = null; 
	AWSInterface awsInterface = new AWSInterface(jobID, null); 
	AWSAnonInterface awsAnonInterface = new AWSAnonInterface();
	List<S3ObjectSummary> summaries; 
	int[] output = new int[2];
	for(int i = 0 ; i < 2; i++) {
		summaries = awsAnonInterface.ListBucket(dataBucket, key[i]);
		output[i] = summaries.size();
		summaries.clear();
	}
	int[] answer = {14104, 33468};
	assertArrayEquals(answer, output);
	System.out.println("ListBucket() is ok");
}
 
Example 33
Project: herd   File: StorageFileHelperTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testValidateCopiedS3Files() throws IOException
{
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES)
    {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }

    // Validate the files.
    storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION));
}
 
Example 34
Project: herd   File: StorageFileHelperTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testValidateCopiedS3FilesActualFileSizeMismatch() throws IOException
{
    // Create two lists of expected and actual storage files, with expected file size not matching actual file size.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, NO_ROW_COUNT));
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_2));

    // Try to validate S3 files when expected file size does not match actual file size.
    try
    {
        storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
            new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE,
                SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an IllegalStateException when expected file size does not match actual file size.");
    }
    catch (IllegalStateException e)
    {
        assertEquals(String
            .format("Specified file size of %d bytes for copied \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.",
                FILE_SIZE, TARGET_S3_KEY, STORAGE_NAME, FILE_SIZE_2), e.getMessage());
    }
}
 
Example 35
Project: herd   File: StorageFileHelperTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testValidateRegisteredS3Files() throws IOException
{
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES)
    {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }

    // Validate the files.
    storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION));
}
 
Example 36
Project: herd   File: StorageFileHelperTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testValidateRegisteredS3FilesUnexpectedS3FileFound() throws IOException
{
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));

    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION);

    // Try to validate S3 files when unexpected S3 file exists.
    try
    {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail("Should throw an IllegalStateException when S3 contains unexpected S3 file.");
    }
    catch (IllegalStateException e)
    {
        assertEquals(String
            .format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating registered S3 files. Business object data {%s}", TARGET_S3_KEY,
                STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}
 
Example 37
Project: herd   File: S3ServiceTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testListDirectoryIgnoreZeroByteDirectoryMarkers()
{
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create a list of S3 object summaries.
    List<S3ObjectSummary> s3ObjectSummaries = Arrays.asList(new S3ObjectSummary());

    // Mock the external calls.
    when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries);

    // Call the method under test.
    List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);

    // Verify the external calls.
    verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true);
    verifyNoMoreInteractions(s3Dao);

    // Validate the returned object.
    assertEquals(s3ObjectSummaries, result);
}
 
Example 38
Project: flow   File: S3Filer.java   Source Code and License 6 votes vote down vote up
@Override
@Nonnull
public Record getRecord(@Nonnull String path) throws IOException {
	ObjectListing listing = s3.listObjects(
			new ListObjectsRequest().withBucketName(bucket).withPrefix(path.substring(1)));

	S3ObjectSummary summary = listing.getObjectSummaries().stream().findFirst().orElse(null);
	if (summary == null) {
		return Record.noFile(uri, path);
	}

	long time = summary.getLastModified().getTime();
	long size = summary.getSize();
	boolean directory = summary.getKey().endsWith("/");
	return new Record(uri, RecordPath.from("/" + summary.getKey()), time, size, directory);
}
 
Example 39
Project: micro-server   File: S3Utils.java   Source Code and License 6 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 40
Project: zeppelin   File: S3NotebookRepo.java   Source Code and License 6 votes vote down vote up
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 
Example 41
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public ArrayList<String> getAllChildren(String folderName) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    System.out.println(listing.getObjectSummaries().size());

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 42
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public List<String> getAllChildren(String folderName, String bucket) throws IOException {
    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(bucket);

    if (!(folderName == null || folderName.equals(""))) {
        listRequest.setPrefix(folderName);
    }

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<String> list = new ArrayList<String>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        list.add(summ.getKey());
    }

    return list;
}
 
Example 43
Project: caboclo   File: AmazonClient.java   Source Code and License 6 votes vote down vote up
public List<String> listBucket(String bkt, String prefix, String delimiter) throws IOException {

        ListObjectsRequest listRequest = new ListObjectsRequest();
        listRequest.setBucketName(bkt);
        listRequest.setDelimiter(delimiter);
        listRequest.setPrefix(prefix);

        ObjectListing listing = s3.listObjects(listRequest);

        ArrayList<String> list = new ArrayList<String>();

        for (S3ObjectSummary summ : listing.getObjectSummaries()) {
            list.add(summ.getKey());
        }

        return list;
    }
 
Example 44
Project: presto   File: PrestoS3FileSystem.java   Source Code and License 6 votes vote down vote up
private Iterator<LocatedFileStatus> statusFromObjects(List<S3ObjectSummary> objects)
{
    // NOTE: for encrypted objects, S3ObjectSummary.size() used below is NOT correct,
    // however, to get the correct size we'd need to make an additional request to get
    // user metadata, and in this case it doesn't matter.
    return objects.stream()
            .filter(object -> !object.getKey().endsWith("/"))
            .map(object -> new FileStatus(
                    object.getSize(),
                    false,
                    1,
                    BLOCK_SIZE.toBytes(),
                    object.getLastModified().getTime(),
                    qualifiedPath(new Path("/" + object.getKey()))))
            .map(this::createLocatedFileStatus)
            .iterator();
}
 
Example 45
Project: digdag   File: RedshiftUnloadOperatorFactory.java   Source Code and License 6 votes vote down vote up
private void clearDest(AWSCredentials credentials, RedshiftConnection.UnloadConfig unloadConfig)
{
    try {
        RetryExecutor.retryExecutor() .run(() -> {
            AmazonS3Client s3Client = new AmazonS3Client(credentials);
            ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(unloadConfig.s3Bucket).withPrefix(unloadConfig.s3Prefix);
            ListObjectsV2Result result;
            // This operation shouldn't be skipped since remaining files created by other operation can cause duplicated data
            do {
                result = s3Client.listObjectsV2(req);

                for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                    s3Client.deleteObject(unloadConfig.s3Bucket, objectSummary.getKey());
                }
                req.setContinuationToken(result.getNextContinuationToken());
            } while (result.isTruncated());
        });
    }
    catch (RetryExecutor.RetryGiveupException e) {
        throw Throwables.propagate(e);
    }
}
 
Example 46
Project: digdag   File: TestUtils.java   Source Code and License 6 votes vote down vote up
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 
Example 47
Project: ignite   File: S3CheckpointSpiSelfTest.java   Source Code and License 6 votes vote down vote up
/**
 * @throws Exception If error.
 */
@Override protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(),
        IgniteS3TestSuite.getSecretKey());

    AmazonS3 s3 = new AmazonS3Client(cred);

    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                s3.deleteObject(bucketName, sum.getKey());

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    }
    catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}
 
Example 48
Project: datacollector   File: S3Spooler.java   Source Code and License 6 votes vote down vote up
public void postProcessOlderObjectIfNeeded(AmazonS3Source.S3Offset s3Offset) {
  //If sdc was shutdown after reading an object but before post processing it, handle it now.

  //The scenario is detected as follows:
  //  1. the current key must not be null
  //  2. offset must be -1
  //  3. An object with same key must exist in s3
  //  4. The timestamp of the object ins3 must be same as that of the timestamp in offset [It is possible that one
  //    uploads another object with the same name. We can avoid post processing it without producing records by
  //    comparing the timestamp on that object

  if(s3Offset.getKey() != null &&
    "-1".equals(s3Offset.getOffset())) {
    //conditions 1, 2 are met. Check for 3 and 4.
    S3ObjectSummary objectSummary = AmazonS3Util.getObjectSummary(s3Client, s3ConfigBean.s3Config.bucket, s3Offset.getKey());
    if(objectSummary != null &&
      objectSummary.getLastModified().compareTo(new Date(Long.parseLong(s3Offset.getTimestamp()))) == 0) {
      postProcessOrErrorHandle(s3Offset.getKey(), s3ConfigBean.postProcessingConfig.postProcessing,
        s3ConfigBean.postProcessingConfig.postProcessBucket, s3ConfigBean.postProcessingConfig.postProcessPrefix,
        s3ConfigBean.postProcessingConfig.archivingOption);
    }
  }
  currentObject = null;
}
 
Example 49
Project: datacollector   File: TestAmazonS3TargetForWholeFile.java   Source Code and License 6 votes vote down vote up
private static List<Record> createRecordsForWholeFileFromS3() throws Exception {
  Iterator<S3ObjectSummary> s3ObjectSummaryIterator = S3Objects.inBucket(s3client, SOURCE_BUCKET_NAME).iterator();
  List<Record> records = new ArrayList<>();
  while (s3ObjectSummaryIterator.hasNext()) {
    S3ObjectSummary s3ObjectSummary = s3ObjectSummaryIterator.next();
    Map<String, Object> metadata = getS3Metadata(s3client.getObject(SOURCE_BUCKET_NAME, s3ObjectSummary.getKey()));
    Record record = RecordCreator.create();
    record.getHeader().setAttribute("bucket", TARGET_BUCKET_NAME);
    record.set(
        FileRefUtil.getWholeFileRecordRootField(
            new S3FileRef.Builder()
                .s3Client(s3client)
                .s3ObjectSummary(s3ObjectSummary)
                .useSSE(false)
                .verifyChecksum(false)
                .bufferSize(1024)
                .build(),
            metadata
        )
    );
    records.add(record);
  }
  return records;
}
 
Example 50
Project: Reer   File: S3ResourceResolver.java   Source Code and License 5 votes vote down vote up
@Override
public String apply(S3ObjectSummary input) {
    Matcher matcher = FILENAME_PATTERN.matcher(input.getKey());
    if (matcher.find()) {
        String group = matcher.group(0);
        return group.contains(".") ? group : null;
    }
    return null;
}
 
Example 51
Project: Reer   File: S3ResourceResolver.java   Source Code and License 5 votes vote down vote up
private List<String> resolveFileResourceNames(ObjectListing objectListing) {
    List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
    if (null != objectSummaries) {
        return ImmutableList.copyOf(Iterables.filter(
            Iterables.transform(objectSummaries, EXTRACT_FILE_NAME),
            Predicates.notNull()
        ));
    }
    return Collections.emptyList();

}
 
Example 52
Project: pprxmtr   File: S3.java   Source Code and License 5 votes vote down vote up
public static boolean fileExistsInBucket(String filenamePrefix) {
	try {
		ObjectListing ol = S3.listObjects(properties.getProperty(S3_BUCKET_NAME));
		List<S3ObjectSummary> objects = ol.getObjectSummaries();
		for (S3ObjectSummary os : objects) {
			if (StringUtils.startsWith(os.getKey(), filenamePrefix)) {
				LOG.info("Found file {} from S3 bucket.", filenamePrefix);
				return true;
			}
		}
	} catch (AmazonServiceException e) {
		LOG.error("Exception occured when checking if file exists in S3.", e);
	}
	return false;
}
 
Example 53
Project: pprxmtr   File: S3.java   Source Code and License 5 votes vote down vote up
public static String getS3KeyFromBucket(String filenamePrefix) {
	try {
		ObjectListing ol = S3.listObjects(properties.getProperty(S3_BUCKET_NAME));
		List<S3ObjectSummary> objects = ol.getObjectSummaries();
		for (S3ObjectSummary os : objects) {
			if (StringUtils.startsWith(os.getKey(), filenamePrefix)) {
				LOG.info("Found file {} from S3 bucket.", filenamePrefix);
				return os.getKey();
			}
		}
	} catch (AmazonServiceException e) {
		LOG.error("Exception occured when checking if file exists in S3.", e);
	}
	return StringUtils.EMPTY;
}
 
Example 54
Project: elasticsearch_my   File: S3BlobContainer.java   Source Code and License 5 votes vote down vote up
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}
 
Example 55
Project: elasticsearch_my   File: MockAmazonS3.java   Source Code and License 5 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
        throws AmazonClientException, AmazonServiceException {
    MockObjectListing list = new MockObjectListing();
    list.setTruncated(false);

    String blobName;
    String prefix = listObjectsRequest.getPrefix();

    ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();

    for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
        blobName = blob.getKey();
        S3ObjectSummary objectSummary = new S3ObjectSummary();

        if (prefix.isEmpty() || blobName.startsWith(prefix)) {
            objectSummary.setKey(blobName);

            try {
                objectSummary.setSize(getSize(blob.getValue()));
            } catch (IOException e) {
                throw new AmazonS3Exception("Object listing " +
                        "failed for blob [" + blob.getKey() + "]");
            }

            mockObjectSummaries.add(objectSummary);
        }
    }

    list.setObjectSummaries(mockObjectSummaries);
    return list;
}
 
Example 56
Project: ooso   File: CommonUtilitiesTest.java   Source Code and License 5 votes vote down vote up
@Test
public void getBucketObjectSummariesTest() throws Exception {
    List<S3ObjectSummary> bucketObjectSummariesWithPrefix = Commons.getBucketObjectSummaries(DUMMY_BUCKET_NAME, "pref1/");
    Set<String> storedPrefixedKeys = bucketObjectSummariesWithPrefix.stream().map(S3ObjectSummary::getKey).collect(toSet());
    Set<String> expectedPrefixedKeys = KEY_CONTENT_MAPPING.keySet().stream().filter(o -> o.startsWith("pref1/")).collect(toSet());
    assertEquals(storedPrefixedKeys, expectedPrefixedKeys);
}
 
Example 57
Project: syndesis-qe   File: S3Utils.java   Source Code and License 5 votes vote down vote up
public void deleteS3Bucket(String bucketName) {
	try {
		final ObjectListing bucketObjects = s3client.listObjects(bucketName);
		for (Iterator<?> iterator = bucketObjects.getObjectSummaries().iterator(); iterator.hasNext();) {
			final S3ObjectSummary summary = (S3ObjectSummary) iterator.next();
			s3client.deleteObject(bucketName, summary.getKey());
		}
		s3client.deleteBucket(bucketName);
	} catch (AmazonServiceException e) {
		log.error("Could not delete the S3 bucket: {}", e.getErrorMessage());
	}
}
 
Example 58
Project: syndesis-qe   File: S3Utils.java   Source Code and License 5 votes vote down vote up
/**
 * Checks if the specified text file exists in specified S3 bucket.
 *
 * @param bucketName
 * @param fileName
 * @return
 */
public boolean checkFileExistsInBucket(String bucketName, String fileName) {

	boolean fileExists = false;
	final ObjectListing objectListing = s3client.listObjects(bucketName);
	for (S3ObjectSummary os : objectListing.getObjectSummaries()) {
		log.debug(os.getKey());
		if (os.getKey().matches(fileName)) {
			fileExists = true;
			break;
		}
	}
	return fileExists;
}
 
Example 59
Project: ats-framework   File: S3Operations.java   Source Code and License 5 votes vote down vote up
/**
 * Get MD5, size, owner, storage class and last modification time for a desired file in the pointed bucket
 *
 * @param fileName the file name
 */
@PublicAtsApi
public S3ObjectInfo getFileMetadata( String fileName ) {

    S3ObjectSummary element = getBucketElement("", ".*", true, fileName);
    if (element != null) {
        S3ObjectInfo s3Info = new S3ObjectInfo(element);
        return s3Info;
    } else {
        throw new NoSuchElementException("File with name '" + fileName + "' does not exist!");
    }
}
 
Example 60
Project: ats-framework   File: S3Operations.java   Source Code and License 5 votes vote down vote up
/**
 * Get size of the specified object/file
 *
 * @param objectName the object name
 * @return the size of the object in bytes
 */
@PublicAtsApi
public long getFileSize( String objectName ) {

    S3ObjectSummary element = getBucketElement("", ".*", true, objectName);
    if (element != null) {
        return element.getSize();
    } else {
        throw new NoSuchElementException("Object with name '" + objectName + "' does not exist!");
    }
}