Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary

The following are top voted examples for showing how to use com.amazonaws.services.s3.model.S3ObjectSummary. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: cmn-project   File: S3.java   View source code 7 votes vote down vote up
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 
Example 2
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 3
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 4
Project: syndesis   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 5
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 6
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 7
Project: connectors   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 8
Project: github-bucket   File: RepositoryS3.java   View source code 6 votes vote down vote up
private boolean isUploadFile(Iterator<S3ObjectSummary> iter, String path, String hash) {
    while (iter.hasNext()) {
        S3ObjectSummary fileS3 = iter.next();
        // Filename should look like this:
        // a/b
        if (!fileS3.getKey().equals(path)) {
            // If this is another file, then continue!
            continue;
        }
        // Remove the file from the S3 list as it does not need to be processed further
        iter.remove();
        // Upload if the hashes differ
        return StringUtils.isNullOrEmpty(hash) || !fileS3.getETag().equals(hash);
    }
    return true;
}
 
Example 9
Project: nifi-minifi   File: S3CacheFileInfoImpl.java   View source code 6 votes vote down vote up
@Override
public Stream<WriteableConfiguration> getCachedConfigurations() throws IOException {

  Iterable<S3ObjectSummary> objectSummaries = S3Objects.withPrefix(s3, bucket, prefix);
  Stream<S3ObjectSummary> objectStream = StreamSupport.stream(objectSummaries.spliterator(), false);

  return objectStream.map(p -> {
    Integer version = getVersionIfMatch(p.getKey());
    if (version == null) {
      return null;
    }
    return new Pair<>(version, p);
  }).filter(Objects::nonNull)
      .sorted(Comparator.comparing(pair -> ((Pair<Integer, S3ObjectSummary>) pair).getFirst())
            .reversed()).map(pair -> new S3WritableConfiguration(s3, pair.getSecond(), Integer.toString(pair.getFirst())));

}
 
Example 10
Project: galaxy-fds-migration-tool   File: S3Source.java   View source code 6 votes vote down vote up
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 
Example 11
Project: RekognitionS3Batch   File: Scanner.java   View source code 6 votes vote down vote up
private boolean processObjects(List<S3ObjectSummary> objects) {
    Logger.Debug("Scanning next batch of %s ", objects.size());
    objects
        .parallelStream()
        .filter(this::shouldEnqueue)
        .forEach(object -> {
            numSeen.incrementAndGet();
            String path = object.getBucketName() + "/" + object.getKey();
            Logger.Info("Posting: %s", path);

            SendMessageRequest msg = new SendMessageRequest()
                .withQueueUrl(queueUrl)
                .withMessageBody(path);
            sqs.sendMessage(msg);
        });
    if (max > -1L && numSeen.incrementAndGet() > max) {
        Logger.Info("Added max jobs, quitting");
        return false;
    }
    return true;
}
 
Example 12
Project: ismartonline   File: AwsFileManager.java   View source code 6 votes vote down vote up
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 
Example 13
Project: vertx-pairtree   File: AbstractS3IT.java   View source code 6 votes vote down vote up
/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}
 
Example 14
Project: cloudkeeper   File: S3Utilities.java   View source code 6 votes vote down vote up
static void deletePrefix(AmazonS3 s3Client, String bucketName, String prefix) {
    List<DeleteObjectsRequest.KeyVersion> trashKeys = new ArrayList<>();
    DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName);
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, prefix)) {
        trashKeys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        if (trashKeys.size() == BULK_DELETE_SIZE) {
            deleteObjectsRequest.setKeys(trashKeys);
            s3Client.deleteObjects(deleteObjectsRequest);
            trashKeys.clear();
        }
    }
    if (!trashKeys.isEmpty()) {
        deleteObjectsRequest.setKeys(trashKeys);
        s3Client.deleteObjects(deleteObjectsRequest);
    }
}
 
Example 15
Project: cloudkeeper   File: S3StagingArea.java   View source code 6 votes vote down vote up
@Override
protected void copy(RuntimeExecutionTrace source, RuntimeExecutionTrace target,
        RuntimeAnnotatedExecutionTrace absoluteSource, RuntimeAnnotatedExecutionTrace absoluteTarget)
        throws IOException {
    String sourcePrefix = toS3Path(source).getPrefixForChildren();
    String targetPrefix = toS3Path(target).getPrefixForChildren();
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, sourcePrefix)) {
        if (summary.getKey().startsWith(sourcePrefix)) {
            String relativeKey = summary.getKey().substring(sourcePrefix.length());
            s3Client.copyObject(bucketName, summary.getKey(), bucketName, targetPrefix + relativeKey);
        } else {
            log.error(String.format(
                "S3Objects.withPrefix() returned unexpected key '%s' when asked for prefix '%s'.",
                summary.getKey(), sourcePrefix
            ));
        }
    }
}
 
Example 16
Project: javapasswordsafe   File: S3FileTest.java   View source code 6 votes vote down vote up
@Override
protected void tearDown() throws Exception {
	final File theFile = new File(TEST_FILE_NAME);
	theFile.delete();

	if (s3Account != null) {
		final AmazonS3Client s3 = new AmazonS3Client(
				new BasicAWSCredentials(s3Account.keyId, s3Account.secretKey));
		String hash = s3Account.getHashedName();
		if (s3.doesBucketExist(hash)) {
			for (S3ObjectSummary summary : s3.listObjects(hash).getObjectSummaries()) {
				s3.deleteObject(s3Account.getHashedName(), summary.getKey());
			}

		}
	}

	super.tearDown();
}
 
Example 17
Project: emodb   File: StashReader.java   View source code 6 votes vote down vote up
/**
 * Gets the metadata for a single table in this stash.  This is similar to getting the splits for the table
 * except that it exposes lower level information about the underlying S3 files.  For clients who will use
 * their own system for reading the files from S3, such as source files for a map-reduce job, this method provides
 * the necessary information.  For simply iterating over the stash contents using either {@link #scan(String)}
 * or {@link #getSplits(String)} in conjunction with {@link #getSplit(StashSplit)} is preferred.
 */
public StashTableMetadata getTableMetadata(String table)
        throws StashNotAvailableException, TableNotStashedException {
    ImmutableList.Builder<StashFileMetadata> filesBuilder = ImmutableList.builder();

    Iterator<S3ObjectSummary> objectSummaries = getS3ObjectSummariesForTable(table);
    while (objectSummaries.hasNext()) {
        S3ObjectSummary objectSummary = objectSummaries.next();
        filesBuilder.add(new StashFileMetadata(_bucket, objectSummary.getKey(), objectSummary.getSize()));
    }

    List<StashFileMetadata> files = filesBuilder.build();

    // Get the prefix arbitrarily from the first file.
    String prefix = files.get(0).getKey();
    prefix = prefix.substring(0, prefix.lastIndexOf('/') + 1);

    return new StashTableMetadata(_bucket, prefix, table, files);
}
 
Example 18
Project: emodb   File: StashReaderTest.java   View source code 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 19
Project: kafka-connect-storage-cloud   File: DataWriterAvroTest.java   View source code 6 votes vote down vote up
@Test
public void testCompressFile() throws Exception {
  String avroCodec = "snappy";
  localProps.put(StorageSinkConnectorConfig.AVRO_CODEC_CONFIG, avroCodec);
  setUp();
  task = new S3SinkTask(connectorConfig, context, storage, partitioner, format, SYSTEM_TIME);

  List<SinkRecord> sinkRecords = createRecords(7);
  // Perform write
  task.put(sinkRecords);
  task.close(context.assignment());
  task.stop();

  List<S3ObjectSummary> summaries = listObjects(S3_TEST_BUCKET_NAME, "/", s3);
  for(S3ObjectSummary summary: summaries){
    InputStream in = s3.getObject(summary.getBucketName(), summary.getKey()).getObjectContent();
    DatumReader<Object> reader = new GenericDatumReader<>();
    DataFileStream<Object> streamReader = new DataFileStream<>(in, reader);
    // make sure that produced Avro file has proper codec set
    Assert.assertEquals(avroCodec, streamReader.getMetaString(StorageSinkConnectorConfig.AVRO_CODEC_CONFIG));
    streamReader.close();
  }

  long[] validOffsets = {0, 3, 6};
  verify(sinkRecords, validOffsets);
}
 
Example 20
Project: kafka-connect-storage-cloud   File: TestWithMockedS3.java   View source code 6 votes vote down vote up
public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}
 
Example 21
Project: hawkbit-extensions   File: S3Repository.java   View source code 6 votes vote down vote up
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 
Example 22
Project: modeshape-s3-binary-store   File: S3BinaryStore.java   View source code 6 votes vote down vote up
@Override
public Iterable<BinaryKey> getAllBinaryKeys() throws BinaryStoreException {
    try {
        final Iterator<S3ObjectSummary> objectsIterator =
            S3Objects.inBucket(s3Client, bucketName).iterator();
        // Lambda to hand back BinaryKeys rather than S3ObjectSummaries
        return () -> {
            return new Iterator<BinaryKey>() {
                @Override
                public boolean hasNext() {
                    return objectsIterator.hasNext();
                }

                @Override
                public BinaryKey next() {
                    S3ObjectSummary object = objectsIterator.next();
                    return new BinaryKey(object.getKey());
                }
            };
        };
    } catch (AmazonClientException e) {
        throw new BinaryStoreException(e);
    }
}
 
Example 23
Project: ecs-samples   File: _99_DeleteBucket.java   View source code 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  	// create the AWS S3 Client
AmazonS3 s3 = AWSS3Factory.getS3Client();

      // delete the demo bucket and all its content
for (S3ObjectSummary summary : s3.listObjects(AWSS3Factory.S3_BUCKET).getObjectSummaries())
{
	s3.deleteObject(AWSS3Factory.S3_BUCKET, summary.getKey());
	System.out.println(String.format("Deleted [%s/%s]", AWSS3Factory.S3_BUCKET, summary.getKey()));
}
s3.deleteBucket(AWSS3Factory.S3_BUCKET);

  	            // print bucket key/value and content for validation
  	System.out.println( String.format("deleted bucket [%s]",
  			AWSS3Factory.S3_BUCKET));
  }
 
Example 24
Project: aws-doc-sdk-examples   File: ListObjects.java   View source code 6 votes vote down vote up
public static void main(String[] args)
{
    final String USAGE = "\n" +
        "To run this example, supply the name of a bucket to list!\n" +
        "\n" +
        "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient();
    ObjectListing ol = s3.listObjects(bucket_name);
    List<S3ObjectSummary> objects = ol.getObjectSummaries();
    for (S3ObjectSummary os: objects) {
        System.out.println("* " + os.getKey());
    }
}
 
Example 25
Project: cerberus-lifecycle-cli   File: S3StoreServiceTest.java   View source code 6 votes vote down vote up
@Test
public void testGetKeysInPartialPath() {
    AmazonS3 client = mock(AmazonS3.class);
    S3StoreService service = new S3StoreService(client, S3_BUCKET, S3_PREFIX);

    String path = "path";
    String key = "my-key";

    S3ObjectSummary summary = new S3ObjectSummary();
    summary.setKey(S3_PREFIX + key);

    ObjectListing listing = mock(ObjectListing.class);

    when(listing.getObjectSummaries()).thenReturn(Lists.newArrayList(summary));
    when(client.listObjects(S3_BUCKET, S3_PREFIX + "/" + path)).thenReturn(listing);

    // invoke method under test
    Set<String> results = service.getKeysInPartialPath(path);

    assertEquals(1, results.size());
    assertEquals(key, results.iterator().next());
}
 
Example 26
Project: lumber-mill   File: S3ScheduledPoll.java   View source code 6 votes vote down vote up
private void subscribe(S3ObjectSummary os, Observable<? extends Object> result) {

        result.subscribe(new Subscriber<Object>() {
            @Override
            public void onCompleted() {
                if (s3ObjectsInPipeline.remove(os)) {
                    processingState.completed(os);
                    release();
                }
            }

            @Override
            public void onError(Throwable e) {
                if (s3ObjectsInPipeline.remove(os)) {
                    release();
                }

                if (LOGGER.isTraceEnabled()) e.printStackTrace();
            }

            @Override
            public void onNext(Object event) { }
        });

    }
 
Example 27
Project: Camel   File: S3IncludeBodyTest.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 28
Project: Camel   File: AmazonS3ClientMock.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 
Example 29
Project: Camel   File: S3ConsumerPrefixTest.java   View source code 6 votes vote down vote up
@Override
public ObjectListing listObjects(ListObjectsRequest request) throws AmazonClientException, AmazonServiceException {
    int currentRequestCount = requestCount.incrementAndGet();
    
    assertEquals("mycamelbucket", request.getBucketName());
    if (currentRequestCount == 2) {
        assertEquals("confidential", request.getPrefix());
    }
    
    ObjectListing response = new ObjectListing();
    response.setBucketName(request.getBucketName());
    response.setPrefix(request.getPrefix());

    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setBucketName(request.getBucketName());
    s3ObjectSummary.setKey("key");
    response.getObjectSummaries().add(s3ObjectSummary);
    
    return response;
}
 
Example 30
Project: exhibitor   File: S3PseudoLock.java   View source code 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 31
Project: s3_video   File: AWSAdapter.java   View source code 6 votes vote down vote up
public List<String> listKeysInOutputBucket(String bucketName) {
	List<String> videoKeys = new ArrayList<>();
	
	ListObjectsRequest listFirstLevelKeyRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withDelimiter("/");
	
	for (String commonPrefix : s3client.listObjects(listFirstLevelKeyRequest).getCommonPrefixes()) {
		
		ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withPrefix(commonPrefix)
		.withDelimiter("/");
		
		for (S3ObjectSummary file : s3client.listObjects(listObjectsRequest).getObjectSummaries()) {
			videoKeys.add(file.getKey());
		}
	}
	return videoKeys;		
}
 
Example 32
Project: WeatherPipe   File: AWSInterfaceTest.java   View source code 6 votes vote down vote up
@Test
public void testListBucket() {
	final String dataBucket = "noaa-nexrad-level2";
	String[] key = {"2010/01/01", "2010/07/14"};
	String jobID = null; 
	AWSInterface awsInterface = new AWSInterface(jobID, null); 
	AWSAnonInterface awsAnonInterface = new AWSAnonInterface();
	List<S3ObjectSummary> summaries; 
	int[] output = new int[2];
	for(int i = 0 ; i < 2; i++) {
		summaries = awsAnonInterface.ListBucket(dataBucket, key[i]);
		output[i] = summaries.size();
		summaries.clear();
	}
	int[] answer = {14104, 33468};
	assertArrayEquals(answer, output);
	System.out.println("ListBucket() is ok");
}
 
Example 33
Project: herd   File: StorageFileHelperTest.java   View source code 6 votes vote down vote up
@Test
public void testValidateCopiedS3Files() throws IOException
{
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES)
    {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }

    // Validate the files.
    storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION));
}
 
Example 34
Project: herd   File: StorageFileHelperTest.java   View source code 6 votes vote down vote up
@Test
public void testValidateCopiedS3FilesActualFileSizeMismatch() throws IOException
{
    // Create two lists of expected and actual storage files, with expected file size not matching actual file size.
    List<StorageFile> testExpectedFiles = Arrays.asList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, NO_ROW_COUNT));
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_2));

    // Try to validate S3 files when expected file size does not match actual file size.
    try
    {
        storageFileHelper.validateCopiedS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
            new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE,
                SUBPARTITION_VALUES, DATA_VERSION));
        fail("Should throw an IllegalStateException when expected file size does not match actual file size.");
    }
    catch (IllegalStateException e)
    {
        assertEquals(String
            .format("Specified file size of %d bytes for copied \"%s\" S3 file in \"%s\" storage does not match file size of %d bytes reported by S3.",
                FILE_SIZE, TARGET_S3_KEY, STORAGE_NAME, FILE_SIZE_2), e.getMessage());
    }
}
 
Example 35
Project: herd   File: StorageFileHelperTest.java   View source code 6 votes vote down vote up
@Test
public void testValidateRegisteredS3Files() throws IOException
{
    // Create two lists of expected and actual storage files.
    // Please note we use different row count values to confirm that row count match is not validated.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = new ArrayList<>();
    for (String file : LOCAL_FILES)
    {
        String filePath = String.format(String.format("%s/%s", TEST_S3_KEY_PREFIX, file));
        testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT));
        testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE));
    }

    // Validate the files.
    storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME,
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION));
}
 
Example 36
Project: herd   File: StorageFileHelperTest.java   View source code 6 votes vote down vote up
@Test
public void testValidateRegisteredS3FilesUnexpectedS3FileFound() throws IOException
{
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Arrays.asList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));

    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION);

    // Try to validate S3 files when unexpected S3 file exists.
    try
    {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail("Should throw an IllegalStateException when S3 contains unexpected S3 file.");
    }
    catch (IllegalStateException e)
    {
        assertEquals(String
            .format("Found unexpected S3 file \"%s\" in \"%s\" storage while validating registered S3 files. Business object data {%s}", TARGET_S3_KEY,
                STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}
 
Example 37
Project: herd   File: S3ServiceTest.java   View source code 6 votes vote down vote up
@Test
public void testListDirectoryIgnoreZeroByteDirectoryMarkers()
{
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create a list of S3 object summaries.
    List<S3ObjectSummary> s3ObjectSummaries = Arrays.asList(new S3ObjectSummary());

    // Mock the external calls.
    when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries);

    // Call the method under test.
    List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);

    // Verify the external calls.
    verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true);
    verifyNoMoreInteractions(s3Dao);

    // Validate the returned object.
    assertEquals(s3ObjectSummaries, result);
}
 
Example 38
Project: flow   File: S3Filer.java   View source code 6 votes vote down vote up
@Override
@Nonnull
public Record getRecord(@Nonnull String path) throws IOException {
	ObjectListing listing = s3.listObjects(
			new ListObjectsRequest().withBucketName(bucket).withPrefix(path.substring(1)));

	S3ObjectSummary summary = listing.getObjectSummaries().stream().findFirst().orElse(null);
	if (summary == null) {
		return Record.noFile(uri, path);
	}

	long time = summary.getLastModified().getTime();
	long size = summary.getSize();
	boolean directory = summary.getKey().endsWith("/");
	return new Record(uri, RecordPath.from("/" + summary.getKey()), time, size, directory);
}
 
Example 39
Project: micro-server   File: S3Utils.java   View source code 6 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 
Example 40
Project: zeppelin   File: S3NotebookRepo.java   View source code 6 votes vote down vote up
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}