Java Code Examples for com.amazonaws.services.s3.model.ListObjectsRequest

The following examples show how to use com.amazonaws.services.s3.model.ListObjectsRequest. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: sequenceiq-samples   Source File: S3Source.java    License: Apache License 2.0 7 votes vote down vote up
@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 
Example 2
Source Project: hop   Source File: S3CommonFileObject.java    License: Apache License 2.0 6 votes vote down vote up
private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
  throws FileSystemException {
  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
    .withBucketName( bucket )
    .withPrefix( keyWithDelimiter )
    .withDelimiter( DELIMITER );
  ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

  if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
    injectType( FileType.FOLDER );
  } else {
    //Folders don't really exist - they will generate a "NoSuchKey" exception
    // confirms key doesn't exist but connection okay
    String errorCode = exception.getErrorCode();
    if ( !errorCode.equals( "NoSuchKey" ) ) {
      // bubbling up other connection errors
      logger.error( "Could not get information on " + getQualifiedName(),
        exception ); // make sure this gets printed for the user
      throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
    }
  }
}
 
Example 3
Source Project: hop   Source File: S3NFileObjectTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 
Example 4
Source Project: hop   Source File: S3NFileObjectTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleAttachExceptionFileNotFound() throws FileSystemException {
  AmazonS3Exception notFoundException = new AmazonS3Exception( "404 Not Found" );
  notFoundException.setErrorCode( "404 Not Found" );
  AmazonS3Exception noSuchKeyException = new AmazonS3Exception( "NoSuchKey" );
  noSuchKeyException.setErrorCode( "NoSuchKey" );

  //test the case where the file is not found; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( notFoundException );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( noSuchKeyException );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 
Example 5
Source Project: circus-train   Source File: S3S3CopierTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void copyMultipleObjects() throws Exception {
  // Making sure we only request 1 file at the time so we need to loop
  ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
  when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));

  client.putObject("source", "bar/data1", inputData);
  client.putObject("source", "bar/data2", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(14L));

  S3Object object1 = client.getObject("target", "foo/data1");
  String data1 = IOUtils.toString(object1.getObjectContent());
  assertThat(data1, is("bar foo"));
  S3Object object2 = client.getObject("target", "foo/data2");
  String data2 = IOUtils.toString(object2.getObjectContent());
  assertThat(data2, is("bar foo"));
}
 
Example 6
Source Project: emodb   Source File: StashReaderTest.java    License: Apache License 2.0 6 votes vote down vote up
private Matcher<ListObjectsRequest> listObjectRequest(final String bucket, final String prefix, @Nullable final String marker) {
    return new BaseMatcher<ListObjectsRequest>() {
        @Override
        public boolean matches(Object item) {
            ListObjectsRequest request = (ListObjectsRequest) item;
            return request != null &&
                    request.getBucketName().equals(bucket) &&
                    request.getPrefix().equals(prefix) &&
                    Objects.equals(request.getMarker(), marker);
        }

        @Override
        public void describeTo(Description description) {
            description.appendText("ListObjectRequest[s3://").appendText(bucket).appendText("/").appendText(prefix);
            if (marker != null) {
                description.appendText(", marker=").appendText(marker);
            }
            description.appendText("]");
        }
    };
}
 
Example 7
Source Project: emodb   Source File: StashReaderTest.java    License: Apache License 2.0 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 8
Source Project: nifi   Source File: TestListS3.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteObjectTags() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_OBJECT_TAGS, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectTaggingRequest> captureRequest = ArgumentCaptor.forClass(GetObjectTaggingRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectTagging(captureRequest.capture());
    GetObjectTaggingRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 
Example 9
Source Project: exhibitor   Source File: S3PseudoLock.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected List<String> getFileNames(String lockPrefix) throws Exception
{
    ListObjectsRequest  request = new ListObjectsRequest();
    request.setBucketName(bucket);
    request.setPrefix(lockPrefix);
    ObjectListing objectListing = client.listObjects(request);

    return Lists.transform
    (
        objectListing.getObjectSummaries(),
        new Function<S3ObjectSummary, String>()
        {
            @Override
            public String apply(S3ObjectSummary summary)
            {
                return summary.getKey();
            }
        }
    );
}
 
Example 10
Source Project: nexus-public   Source File: BucketManager.java    License: Eclipse Public License 1.0 6 votes vote down vote up
@Override
public void deleteStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
  String bucket = getConfiguredBucket(blobStoreConfiguration);
  ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket).withMaxKeys(1));
  if (listing.getObjectSummaries().isEmpty()) {
    s3.deleteBucket(bucket);
  }
  else {
    log.info("Not removing S3 bucket {} because it is not empty", bucket);
    BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
    List<Rule> nonBlobstoreRules = nonBlobstoreRules(lifecycleConfiguration, blobStoreConfiguration.getName());
    if(!isEmpty(nonBlobstoreRules)) {
      lifecycleConfiguration.setRules(nonBlobstoreRules);
      s3.setBucketLifecycleConfiguration(bucket, lifecycleConfiguration);
    } else {
      s3.deleteBucketLifecycleConfiguration(bucket);
    }
  }
}
 
Example 11
Source Project: zeppelin   Source File: OldS3NotebookRepo.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
  String key = user + "/" + "notebook" + "/" + noteId;
  final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
      .withBucketName(bucketName).withPrefix(key);

  try {
    ObjectListing objects = s3client.listObjects(listObjectsRequest);
    do {
      for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
        s3client.deleteObject(bucketName, objectSummary.getKey());
      }
      objects = s3client.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to remove note in S3: " + ace, ace);
  }
}
 
Example 12
Source Project: fullstop   Source File: S3Service.java    License: Apache License 2.0 6 votes vote down vote up
public List<String> listCommonPrefixesS3Objects(final String bucketName, final String prefix) {
    final List<String> commonPrefixes = Lists.newArrayList();

    try {
        log.debug("Listing objects in bucket '{}' with prefix '{}'", bucketName, prefix);

        final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
                .withDelimiter("/")
                .withBucketName(bucketName)
                .withPrefix(prefix);

        ObjectListing objectListing;

        do {
            objectListing = s3client.listObjects(listObjectsRequest);
            objectListing.getCommonPrefixes().stream().map(S3Service::urlDecode).forEach(commonPrefixes::add);
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (final AmazonServiceException e) {
        log.error("Could not list common prefixes in S3", e);
    }

    return commonPrefixes;
}
 
Example 13
Source Project: digdag   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 
Example 14
Source Project: digdag   Source File: EmrIT.java    License: Apache License 2.0 6 votes vote down vote up
protected void validateTdSparkQueryOutput()
{
    AmazonS3URI resultUri = new AmazonS3URI(tmpS3FolderUri.toString() + "/result/");
    ObjectListing resultListing = s3.listObjects(new ListObjectsRequest().withBucketName(resultUri.getBucket()).withPrefix(resultUri.getKey()));
    List<String> resultLines = resultListing.getObjectSummaries().stream().flatMap(o -> {
        try (S3Object object = s3.getObject(o.getBucketName(), o.getKey())) {
            return CharStreams.readLines(new InputStreamReader(object.getObjectContent())).stream();
        }
        catch (IOException e) {
            throw Throwables.propagate(e);
        }
    }).collect(toList());
    // FIXME
    // we need to specify the version of td-spark that we can use for acceptance tests.
    // In the meantime the assertion is commented out.
    //assertThat(resultLines, Matchers.hasItem(",164.54.104.106,/item/games/4663,/category/electronics,404,Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0),121,GET,1412383598"));
}
 
Example 15
Source Project: datacollector   Source File: S3ConnectionSourceConfig.java    License: Apache License 2.0 6 votes vote down vote up
private void validateConnection(
    Stage.Context context,
    String configPrefix,
    List<Stage.ConfigIssue> issues
) {
  try {
    //check if the credentials are right by trying to list an object in the common prefix
    getS3Client().listObjects(new ListObjectsRequest(bucket, commonPrefix, null, delimiter, 1).withEncodingType("url"));
  } catch (AmazonS3Exception e) {
    LOG.debug(Errors.S3_SPOOLDIR_20.getMessage(), e.toString(), e);
    issues.add(
        context.createConfigIssue(
            Groups.S3.name(),
            configPrefix + S3ConnectionBaseConfig.AWS_CONFIG_PREFIX + "awsAccessKeyId",
            Errors.S3_SPOOLDIR_20,
            e.toString()
        )
    );
  }
}
 
Example 16
Source Project: crate   Source File: MockAmazonS3.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final ObjectListing listing = new ObjectListing();
    listing.setBucketName(request.getBucketName());
    listing.setPrefix(request.getPrefix());

    for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
        if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setBucketName(request.getBucketName());
            summary.setKey(blob.getKey());
            summary.setSize(blob.getValue().length);
            listing.getObjectSummaries().add(summary);
        }
    }
    return listing;
}
 
Example 17
Source Project: StormCV   Source File: S3Connector.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> list() {
	List<String> files = new ArrayList<String>();
	if(s3URI == null) return files;
	String[] buckAndKey = this.getBucketAndKey();
	String bucket = buckAndKey[0];
	String key = buckAndKey[1];
	ObjectListing objectListing = s3. listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(key));
	s3list: for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
		// check for valid extensions
		if(extensions == null){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
		} else for(String ext : extensions) if(objectSummary.getKey().endsWith(ext)){
			files.add("s3://"+bucket+"/"+objectSummary.getKey());
			continue s3list;
		}
	 }
	return files;
}
 
Example 18
Source Project: presto   Source File: S3TableConfigClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Call S3 to get the most recent object list.
 * <p>
 * This is an object list request to AWS in the given "directory".
 */
private List<S3ObjectSummary> getObjectSummaries()
{
    AmazonS3Client s3client = clientManager.getS3Client();
    AmazonS3URI directoryURI = new AmazonS3URI(bucketUrl.get());

    List<S3ObjectSummary> result = new ArrayList<>();
    try {
        log.info("Getting the listing of objects in the S3 table config directory: bucket %s prefix %s :", directoryURI.getBucket(), directoryURI.getKey());
        ListObjectsRequest request = new ListObjectsRequest()
                .withBucketName(directoryURI.getBucket())
                .withPrefix(directoryURI.getKey() + "/")
                .withDelimiter("/")
                .withMaxKeys(25);
        ObjectListing response;

        do {
            response = s3client.listObjects(request);

            result.addAll(response.getObjectSummaries());
            request.setMarker(response.getNextMarker());
        }
        while (response.isTruncated());

        log.info("Completed getting S3 object listing.");
    }
    catch (AmazonClientException e) {
        log.error("Skipping update as faced error fetching table descriptions from S3 " + e.toString());
    }
    return result;
}
 
Example 19
Source Project: hop   Source File: S3FileObject.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handleAttachException( String key, String bucket ) throws FileSystemException {
  SimpleEntry<String, String> newPath = fixFilePath( key, bucket );
  String keyWithDelimiter = newPath.getKey() + DELIMITER;
  try {
    s3Object = getS3Object( keyWithDelimiter, newPath.getValue() );
    s3ObjectMetadata = s3Object.getObjectMetadata();
    injectType( FileType.FOLDER );
  } catch ( AmazonS3Exception e2 ) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName( newPath.getValue() )
            .withPrefix( keyWithDelimiter )
            .withDelimiter( DELIMITER );
    ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

    if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
      injectType( FileType.FOLDER );
    } else {
      //Folders don't really exist - they will generate a "NoSuchKey" exception
      String errorCode = e2.getErrorCode();
      // confirms key doesn't exist but connection okay
      if ( !errorCode.equals( "NoSuchKey" ) ) {
        // bubbling up other connection errors
        logger.error( "Could not get information on " + getQualifiedName(),
                e2 ); // make sure this gets printed for the user
        throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), e2 );
      }
    }
  }
}
 
Example 20
private void findAllResourcesThatMatches(String bucketName, Set<Resource> resources,
		String prefix, String keyPattern) {
	ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
			.withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing = null;

	do {
		try {
			if (objectListing == null) {
				objectListing = this.amazonS3.listObjects(listObjectsRequest);
			}
			else {
				objectListing = this.amazonS3.listNextBatchOfObjects(objectListing);
			}
			Set<Resource> newResources = getResourcesFromObjectSummaries(bucketName,
					keyPattern, objectListing.getObjectSummaries());
			if (!newResources.isEmpty()) {
				resources.addAll(newResources);
			}
		}
		catch (AmazonS3Exception e) {
			if (301 != e.getStatusCode()) {
				throw e;
			}
		}
	}
	while (objectListing != null && objectListing.isTruncated());
}
 
Example 21
Source Project: circus-train   Source File: S3S3Copier.java    License: Apache License 2.0 5 votes vote down vote up
private void initialiseCopyJobsFromListing(
    AmazonS3URI sourceS3Uri,
    final AmazonS3URI targetS3Uri,
    ListObjectsRequest request,
    ObjectListing listing) {
  LOG
      .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
          request.getPrefix());
  List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
  for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
    totalBytesToReplicate += s3ObjectSummary.getSize();
    String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
    final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
        s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);

    if (s3s3CopierOptions.getCannedAcl() != null) {
      copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl());
    }

    applyObjectMetadata(copyObjectRequest);

    TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary,
        targetS3Uri, targetKey);
    copyJobRequests.add(new CopyJobRequest(copyObjectRequest, stateChangeListener));
  }
}
 
Example 22
Source Project: nifi   Source File: TestListS3.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteUserMetadata() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_USER_METADATA, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectMetadataRequest> captureRequest = ArgumentCaptor.forClass(GetObjectMetadataRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectMetadata(captureRequest.capture());
    GetObjectMetadataRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());

    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 
Example 23
Source Project: jobcacher-plugin   Source File: S3DownloadAllCallable.java    License: MIT License 5 votes vote down vote up
/**
 * Download to executor
 */
@Override
public Integer invoke(TransferManager transferManager, File base, VirtualChannel channel) throws IOException, InterruptedException {
    if(!base.exists()) {
        if (!base.mkdirs()) {
            throw new IOException("Failed to create directory : " + base);
        }
    }

    int totalCount;
    Downloads downloads = new Downloads();
    ObjectListing objectListing = null;

    do {
        objectListing = transferManager.getAmazonS3Client().listObjects(new ListObjectsRequest()
                .withBucketName(bucketName)
                .withPrefix(pathPrefix)
                .withMarker(objectListing != null ? objectListing.getNextMarker() : null));

        for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
            downloads.startDownload(transferManager, base, pathPrefix, summary);
        }

    } while (objectListing.getNextMarker() != null);

    // Grab # of files copied
    totalCount = downloads.count();

    // Finish the asynchronous downloading process
    downloads.finishDownloading();

    return totalCount;
}
 
Example 24
Source Project: emodb   Source File: StashReader.java    License: Apache License 2.0 5 votes vote down vote up
private static String determineEndpointForBucket(String bucket, AWSCredentialsProvider credentialsProvider,
                                                 @Nullable ClientConfiguration s3Config, String rootPath) {

    // Guess us-east-1.  If wrong AWS will return a redirect with the correct endpoint
    AmazonS3 s3 = createS3ClientForRegion(Regions.US_EAST_1.getName(), credentialsProvider, s3Config);
    if (rootPath.startsWith("/")) {
        rootPath = rootPath.substring(1);
    }
    if (!rootPath.endsWith("/")) {
        rootPath = rootPath + "/";
    }

    try {
        // Any request will work but presumably the client has list access for stash so perform a list.
        s3.listObjects(new ListObjectsRequest()
                .withBucketName(bucket)
                .withPrefix(rootPath)
                .withDelimiter("/")
                .withMaxKeys(1));

        // If this didn't error out then the presumed us-east-1 region was correct
        return  "s3.us-east-1.amazonaws.com";
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == 301 /* MOVED_PERMANENTLY */) {
            String endPoint = e.getAdditionalDetails().get("Endpoint");
            // The end point is prefixed with the bucket name, so strip it
            return endPoint.substring(bucket.length() + 1);
        }

        throw e;
    }
}
 
Example 25
Source Project: emodb   Source File: StashReader.java    License: Apache License 2.0 5 votes vote down vote up
private Iterator<S3ObjectSummary> getS3ObjectSummaries(final String prefix) {
    final int prefixLength = prefix.length();

    Iterator<S3ObjectSummary> allSummaries = Iterators.concat(new AbstractIterator<Iterator<S3ObjectSummary>>() {
        String marker = null;
        ObjectListing response;
        protected Iterator<S3ObjectSummary> computeNext() {
            if (response == null || response.isTruncated()) {
                response = _s3.listObjects(new ListObjectsRequest()
                        .withBucketName(_bucket)
                        .withPrefix(prefix)
                        .withDelimiter("/")
                        .withMarker(marker)
                        .withMaxKeys(1000));
                marker = response.getNextMarker();
                return response.getObjectSummaries().iterator();
            }
            return endOfData();
        }
    });

    // Sometimes the prefix itself can come back as a result.  Filter that entry out.
    return Iterators.filter(allSummaries, new Predicate<S3ObjectSummary>() {
        @Override
        public boolean apply(S3ObjectSummary summary) {
            return summary.getKey().length() > prefixLength;
        }
    });
}
 
Example 26
Source Project: alexa-skills-kit-tester-java   Source File: Lambda.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handleRequest(final InputStream input, final OutputStream output, final Context context) throws IOException {
    final String inputS = IOUtils.toString(Optional.ofNullable(input).orElse(new ByteArrayInputStream("{}".getBytes())));
    final JsonNode root = om.readTree(inputS);

    final String bucket = Optional.ofNullable(root.get(S3_BUCKET_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank).orElse(System.getenv(S3_BUCKET_PROPERTY));
    Validate.notBlank(bucket, S3_BUCKET_PROPERTY + " hasn't been set in the request payload nor as an environment variable.");

    final String key = Optional.ofNullable(root.get(S3_KEY_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
            .orElse(System.getenv(S3_KEY_PROPERTY));
    final String region = Optional.ofNullable(root.get(S3_REGION_PROPERTY)).map(JsonNode::textValue).filter(StringUtils::isNotBlank)
            .orElse(System.getenv(S3_REGION_PROPERTY));

    final AmazonS3 s3client = StringUtils.isNotBlank(region) ? AmazonS3ClientBuilder.standard().withRegion(region).build() : AmazonS3ClientBuilder.defaultClient();

    final ListObjectsRequest listRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(Optional.ofNullable(key).map(k -> k + (k.endsWith("/") ? "" : "/")).orElse(""));

    log.info("[INFO] Reading out *.yml conversation script files in folder '" + listRequest.getPrefix() + "' in bucket '" + listRequest.getBucketName() + "'");

    final List<S3ObjectSummary> conversationScripts = s3client.listObjects(listRequest).getObjectSummaries().stream()
            .filter(os -> os.getKey().toLowerCase().endsWith(".yml")).collect(Collectors.toList());

    log.info("[INFO] Found " + conversationScripts.size() + " conversation script files in bucket '" + bucket + "'");

    for (final S3ObjectSummary conversationScript : conversationScripts) {
        log.info("[INFO] Load conversation script file " + conversationScript.getKey() + " from S3 bucket " + bucket);

        AlexaClient.create(s3client.getObject(bucket, conversationScript.getKey()).getObjectContent())
                .build()
                .startScript();
    }
    output.write("{ \"OK\" }".getBytes());
}
 
Example 27
Source Project: athenz   Source File: S3ChangeLogStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetLocalDomainList() throws FileNotFoundException {

    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null, 0);

    InputStream is1 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is1 = new MockS3ObjectInputStream(is1, null);

    InputStream is2 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is2 = new MockS3ObjectInputStream(is2, null);

    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is1).thenReturn(s3Is2);

    when(store.awsS3Client.getObject("s3-unit-test-bucket-name", "iaas")).thenReturn(object);
    ObjectListing mockObjectListing = mock(ObjectListing.class);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(mockObjectListing);
    List<S3ObjectSummary> tempList = new ArrayList<>();
    S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
    when(s3ObjectSummary.getKey()).thenReturn("iaas");
    tempList.add(s3ObjectSummary);
    when(mockObjectListing.getObjectSummaries()).thenReturn(tempList);

    List<String> temp = store.getLocalDomainList();
    assertNotNull(temp);
    store.getLocalSignedDomain("iaas");
}
 
Example 28
Source Project: herd   Source File: MockS3OperationsImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
            {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}
 
Example 29
Source Project: terrapin   Source File: TerrapinUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
  AmazonS3Client s3Client = mock(AmazonS3Client.class);
  ObjectListing objectListing = mock(ObjectListing.class);
  S3ObjectSummary summary1 = new S3ObjectSummary();
  S3ObjectSummary summary2 = new S3ObjectSummary();
  S3ObjectSummary summary3 = new S3ObjectSummary();
  summary1.setKey("/abc/123");
  summary2.setKey("/abc/456");
  summary3.setKey("/def/123");
  summary1.setSize(32432);
  summary2.setSize(213423);
  summary3.setSize(2334);
  List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
  whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
  when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
  when(objectListing.getObjectSummaries()).thenReturn(summaries);

  List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
      "bucket", "/abc");

  assertEquals(2, results.size());
  assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
  assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
  assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
  assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
 
Example 30
Source Project: micro-server   Source File: S3Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}