Java Code Examples for com.amazonaws.services.s3.model.S3Object#setBucketName()

The following examples show how to use com.amazonaws.services.s3.model.S3Object#setBucketName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AmazonS3SourceMockTests.java From spring-cloud-stream-app-starters with Apache License 2.0

6 votes

@BeforeClass
public static void setup() throws IOException {
	File remoteFolder = TEMPORARY_FOLDER.newFolder("remote");

	File aFile = new File(remoteFolder, "1.test");
	FileCopyUtils.copy("Hello".getBytes(), aFile);
	File bFile = new File(remoteFolder, "2.test");
	FileCopyUtils.copy("Bye".getBytes(), bFile);
	File otherFile = new File(remoteFolder, "otherFile");
	FileCopyUtils.copy("Other\nOther2".getBytes(), otherFile);

	S3_OBJECTS = new ArrayList<>();

	for (File file : remoteFolder.listFiles()) {
		S3Object s3Object = new S3Object();
		s3Object.setBucketName(S3_BUCKET);
		s3Object.setKey(file.getName());
		s3Object.setObjectContent(new FileInputStream(file));
		S3_OBJECTS.add(s3Object);
	}

	String localFolder = TEMPORARY_FOLDER.newFolder("local").getAbsolutePath();

	System.setProperty("s3.localDir", localFolder);
}

Example 2

Source File: MockS3OperationsImpl.java From herd with Apache License 2.0

5 votes

@Override
public S3Object getS3Object(GetObjectRequest getObjectRequest, AmazonS3 s3)
{
    MockS3Object mockS3Object = getMockS3Object(getObjectRequest.getBucketName(), getObjectRequest.getKey());

    S3Object s3Object = new S3Object();
    s3Object.setBucketName(getObjectRequest.getBucketName());
    s3Object.setKey(getObjectRequest.getKey());
    s3Object.setObjectContent(new ByteArrayInputStream(mockS3Object.getData()));
    s3Object.setObjectMetadata(mockS3Object.getObjectMetadata());

    return s3Object;
}

Example 3

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

5 votes

@Test
public void testGetObjectVersion() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.VERSION_ID, "${s3.version}");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    attrs.put("s3.version", "request-version");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    Mockito.when(metadata.getVersionId()).thenReturn("response-version");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertEquals("request-version", request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals("s3.version", "response-version");
    ff.assertContentEquals("Some Content");
}

Example 4

Source File: AmazonS3FileSystem.java From iaf with Apache License 2.0

5 votes

@Override
public Iterator<S3Object> listFiles(String folder) throws FileSystemException {
	List<S3ObjectSummary> summaries = null;
	String prefix = folder != null ? folder + "/" : "";
	try {
		ObjectListing listing = s3Client.listObjects(bucketName, prefix);
		summaries = listing.getObjectSummaries();
		while (listing.isTruncated()) {
			listing = s3Client.listNextBatchOfObjects(listing);
			summaries.addAll(listing.getObjectSummaries());
		}
	} catch (AmazonServiceException e) {
		throw new FileSystemException("Cannot process requested action", e);
	}

	List<S3Object> list = new ArrayList<S3Object>();
	for (S3ObjectSummary summary : summaries) {
		S3Object object = new S3Object();
		ObjectMetadata metadata = new ObjectMetadata();
		metadata.setContentLength(summary.getSize());

		object.setBucketName(summary.getBucketName());
		object.setKey(summary.getKey());
		object.setObjectMetadata(metadata);
		if(!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) {
			list.add(object);
		} 
	}

	return list.iterator();
}

Example 5

Source File: S3ObjectsProviderTest.java From pentaho-kettle with Apache License 2.0

5 votes

private static S3Object buildS3Object( Bucket bucket, String key, String dataString ) {
  S3Object s3Object = new S3Object();
  s3Object.setKey( key );
  s3Object.setBucketName( bucket.getName() );
  s3Object.setObjectContent( new ByteArrayInputStream( dataString.getBytes() ) );
  return s3Object;
}

Example 6

Source File: S3FileObjectTest.java From hop with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {

  s3ServiceMock = mock( AmazonS3.class );
  S3Object s3Object = new S3Object();
  s3Object.setKey( OBJECT_NAME );
  s3Object.setBucketName( BUCKET_NAME );

  filename = new S3FileName( SCHEME, BUCKET_NAME, BUCKET_NAME, FileType.FOLDER );
  S3FileName rootFileName = new S3FileName( SCHEME, "", "", FileType.FOLDER );
  S3HopProperty s3HopProperty = mock( S3HopProperty.class );
  when( s3HopProperty.getPartSize() ).thenReturn( "5MB" );
  S3FileSystem fileSystem = new S3FileSystem( rootFileName, new FileSystemOptions(), new StorageUnitConverter(), s3HopProperty );
  fileSystemSpy = spy( fileSystem );
  VfsComponentContext context = mock( VfsComponentContext.class );
  final DefaultFileSystemManager fsm = new DefaultFileSystemManager();
  FilesCache cache = mock( FilesCache.class );
  fsm.setFilesCache( cache );
  fsm.setCacheStrategy( CacheStrategy.ON_RESOLVE );
  when( context.getFileSystemManager() ).thenReturn( fsm );
  fileSystemSpy.setContext( context );

  S3FileObject s3FileObject = new S3FileObject( filename, fileSystemSpy );
  s3FileObjectBucketSpy = spy( s3FileObject );

  s3FileObjectFileSpy = spy( new S3FileObject(
    new S3FileName( SCHEME, BUCKET_NAME, BUCKET_NAME + "/" + origKey, FileType.IMAGINARY ), fileSystemSpy ) );

  S3FileObject s3FileObjectRoot = new S3FileObject( rootFileName, fileSystemSpy );
  s3FileObjectSpyRoot = spy( s3FileObjectRoot );

  // specify the behaviour of S3 Service
  //when( s3ServiceMock.getBucket( BUCKET_NAME ) ).thenReturn( testBucket );
  when( s3ServiceMock.getObject( BUCKET_NAME, OBJECT_NAME ) ).thenReturn( s3Object );
  when( s3ServiceMock.getObject( BUCKET_NAME, OBJECT_NAME ) ).thenReturn( s3Object );
  when( s3ServiceMock.listBuckets() ).thenReturn( createBuckets() );

  when( s3ServiceMock.doesBucketExistV2( BUCKET_NAME ) ).thenReturn( true );

  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( createObjectSummaries( 0 ) ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() ).thenReturn( createCommonPrefixes( 3 ) );
  when( childObjectListing.isTruncated() ).thenReturn( true ).thenReturn( false );

  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  when( s3ServiceMock.listObjects( anyString(), anyString() ) ).thenReturn( childObjectListing );
  when( s3ServiceMock.listNextBatchOfObjects( any( ObjectListing.class ) ) ).thenReturn( childObjectListing );

  s3ObjectMock = mock( S3Object.class );
  s3ObjectInputStream = mock( S3ObjectInputStream.class );
  s3ObjectMetadata = mock( ObjectMetadata.class );
  when( s3ObjectMock.getObjectContent() ).thenReturn( s3ObjectInputStream );
  when( s3ServiceMock.getObjectMetadata( anyString(), anyString() ) ).thenReturn( s3ObjectMetadata );
  when( s3ObjectMetadata.getContentLength() ).thenReturn( contentLength );
  when( s3ObjectMetadata.getLastModified() ).thenReturn( testDate );
  when( s3ServiceMock.getObject( anyString(), anyString() ) ).thenReturn( s3ObjectMock );

  when( fileSystemSpy.getS3Client() ).thenReturn( s3ServiceMock );
}

Example 7

Source File: S3NFileObjectTest.java From hop with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {

  s3ServiceMock = mock( AmazonS3.class );
  S3Object s3Object = new S3Object();
  s3Object.setKey( OBJECT_NAME );
  s3Object.setBucketName( BUCKET_NAME );

  filename = new S3NFileName( SCHEME, BUCKET_NAME, "/" + BUCKET_NAME, FileType.FOLDER );
  S3NFileName rootFileName = new S3NFileName( SCHEME, BUCKET_NAME, "", FileType.FOLDER );
  S3NFileSystem fileSystem = new S3NFileSystem( rootFileName, new FileSystemOptions() );
  fileSystemSpy = spy( fileSystem );
  VfsComponentContext context = mock( VfsComponentContext.class );
  final DefaultFileSystemManager fsm = new DefaultFileSystemManager();
  FilesCache cache = mock( FilesCache.class );
  fsm.setFilesCache( cache );
  fsm.setCacheStrategy( CacheStrategy.ON_RESOLVE );
  when( context.getFileSystemManager() ).thenReturn( fsm );
  fileSystemSpy.setContext( context );

  S3NFileObject s3FileObject = new S3NFileObject( filename, fileSystemSpy );
  s3FileObjectBucketSpy = spy( s3FileObject );

  s3FileObjectFileSpy = spy( new S3NFileObject(
    new S3NFileName( SCHEME, BUCKET_NAME, "/" + BUCKET_NAME + "/" + origKey, FileType.IMAGINARY ), fileSystemSpy ) );

  S3NFileObject s3FileObjectRoot = new S3NFileObject( rootFileName, fileSystemSpy );
  s3FileObjectSpyRoot = spy( s3FileObjectRoot );

  // specify the behaviour of S3 Service
  //when( s3ServiceMock.getBucket( BUCKET_NAME ) ).thenReturn( testBucket );
  when( s3ServiceMock.getObject( BUCKET_NAME, OBJECT_NAME ) ).thenReturn( s3Object );
  when( s3ServiceMock.getObject( BUCKET_NAME, OBJECT_NAME ) ).thenReturn( s3Object );
  when( s3ServiceMock.listBuckets() ).thenReturn( createBuckets() );

  when( s3ServiceMock.doesBucketExistV2( BUCKET_NAME ) ).thenReturn( true );

  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( createObjectSummaries( 0 ) ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() ).thenReturn( createCommonPrefixes( 3 ) );
  when( childObjectListing.isTruncated() ).thenReturn( true ).thenReturn( false );

  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  when( s3ServiceMock.listObjects( anyString(), anyString() ) ).thenReturn( childObjectListing );
  when( s3ServiceMock.listNextBatchOfObjects( any( ObjectListing.class ) ) ).thenReturn( childObjectListing );

  s3ObjectMock = mock( S3Object.class );
  s3ObjectInputStream = mock( S3ObjectInputStream.class );
  s3ObjectMetadata = mock( ObjectMetadata.class );
  when( s3ObjectMock.getObjectContent() ).thenReturn( s3ObjectInputStream );
  when( s3ServiceMock.getObjectMetadata( anyString(), anyString() ) ).thenReturn( s3ObjectMetadata );
  when( s3ObjectMetadata.getContentLength() ).thenReturn( contentLength );
  when( s3ObjectMetadata.getLastModified() ).thenReturn( testDate );
  when( s3ServiceMock.getObject( anyString(), anyString() ) ).thenReturn( s3ObjectMock );

  when( fileSystemSpy.getS3Client() ).thenReturn( s3ServiceMock );
}

Example 8

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

4 votes

@Test
public void testGetObject() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertFalse(request.isRequesterPays());
    assertNull(request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}

Example 9

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

4 votes

@Test
public void testGetObjectWithRequesterPays() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.REQUESTER_PAYS, "true");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertTrue(request.isRequesterPays());
    assertNull(request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}