Java Code Examples for com.amazonaws.services.s3.model.S3Object#setObjectMetadata()

The following examples show how to use com.amazonaws.services.s3.model.S3Object#setObjectMetadata() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AwsS3EnvironmentRepositoryTests.java From spring-cloud-config with Apache License 2.0

6 votes

private void setupS3(String fileName, String version, String propertyContent)
		throws UnsupportedEncodingException {
	final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", fileName);
	final GetObjectRequest request = new GetObjectRequest(s3ObjectId);

	final S3Object s3Object = new S3Object();
	s3Object.setObjectContent(new StringInputStream(propertyContent));

	if (version != null) {
		final ObjectMetadata metadata = new ObjectMetadata();
		metadata.setHeader("x-amz-version-id", version);
		s3Object.setObjectMetadata(metadata);
	}

	when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
			.thenReturn(s3Object);
}

Example 2

Source File: SimpleStorageResourceTest.java From spring-cloud-aws with Apache License 2.0

6 votes

@Test
void getInputStream_existingObject_returnsInputStreamWithContent() throws Exception {
	// Arrange
	AmazonS3 amazonS3 = mock(AmazonS3.class);
	ObjectMetadata objectMetadata = mock(ObjectMetadata.class);
	when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
			.thenReturn(objectMetadata);

	S3Object s3Object = new S3Object();
	s3Object.setObjectMetadata(objectMetadata);
	s3Object.setObjectContent(new ByteArrayInputStream(new byte[] { 42 }));
	when(amazonS3.getObject(any(GetObjectRequest.class))).thenReturn(s3Object);

	// Act
	SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
			"bucket", "object", new SyncTaskExecutor());

	// Assert
	assertThat(simpleStorageResource.exists()).isTrue();
	assertThat(simpleStorageResource.getInputStream().read()).isEqualTo(42);
}

Example 3

Source File: MockAmazonS3.java From crate with Apache License 2.0

6 votes

@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final String blobName = request.getKey();
    final byte[] content = blobs.get(blobName);
    if (content == null) {
        AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
        exception.setStatusCode(404);
        throw exception;
    }

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(content.length);

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
    s3Object.setKey(blobName);
    s3Object.setObjectMetadata(metadata);

    return s3Object;
}

Example 4

Source File: MockS3OperationsImpl.java From herd with Apache License 2.0

5 votes

@Override
public S3Object getS3Object(GetObjectRequest getObjectRequest, AmazonS3 s3)
{
    MockS3Object mockS3Object = getMockS3Object(getObjectRequest.getBucketName(), getObjectRequest.getKey());

    S3Object s3Object = new S3Object();
    s3Object.setBucketName(getObjectRequest.getBucketName());
    s3Object.setKey(getObjectRequest.getKey());
    s3Object.setObjectContent(new ByteArrayInputStream(mockS3Object.getData()));
    s3Object.setObjectMetadata(mockS3Object.getObjectMetadata());

    return s3Object;
}

Example 5

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

5 votes

@Test
public void testGetObjectVersion() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.VERSION_ID, "${s3.version}");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    attrs.put("s3.version", "request-version");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    Mockito.when(metadata.getVersionId()).thenReturn("response-version");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertEquals("request-version", request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals("s3.version", "response-version");
    ff.assertContentEquals("Some Content");
}

Example 6

Source File: AmazonS3FileSystem.java From iaf with Apache License 2.0

5 votes

@Override
public Iterator<S3Object> listFiles(String folder) throws FileSystemException {
	List<S3ObjectSummary> summaries = null;
	String prefix = folder != null ? folder + "/" : "";
	try {
		ObjectListing listing = s3Client.listObjects(bucketName, prefix);
		summaries = listing.getObjectSummaries();
		while (listing.isTruncated()) {
			listing = s3Client.listNextBatchOfObjects(listing);
			summaries.addAll(listing.getObjectSummaries());
		}
	} catch (AmazonServiceException e) {
		throw new FileSystemException("Cannot process requested action", e);
	}

	List<S3Object> list = new ArrayList<S3Object>();
	for (S3ObjectSummary summary : summaries) {
		S3Object object = new S3Object();
		ObjectMetadata metadata = new ObjectMetadata();
		metadata.setContentLength(summary.getSize());

		object.setBucketName(summary.getBucketName());
		object.setKey(summary.getKey());
		object.setObjectMetadata(metadata);
		if(!object.getKey().endsWith("/") && !(prefix.isEmpty() && object.getKey().contains("/"))) {
			list.add(object);
		} 
	}

	return list.iterator();
}

Example 7

Source File: StashReaderTest.java From emodb with Apache License 2.0

4 votes

@Test
public void getSplit() throws Exception {
    // Intentionally create the gzip file as concatenated gzip files to verify the Java bug involving reading
    // concatenated gzip files is resolved.

    List<Map<String, Object>> expected = Lists.newArrayListWithCapacity(4);
    ByteArrayOutputStream splitOut = new ByteArrayOutputStream();

    for (int p=0; p < 2; p++) {
        for (int i=0; i < 2; i++) {
            ByteArrayOutputStream partialOut = new ByteArrayOutputStream();
            try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(partialOut)))) {
                Map<String, Object> value = ImmutableMap.<String, Object>builder()
                        .put("~id", "row" + (p * 2 + i))
                        .put("~table", "test:table")
                        .put("~version", 1)
                        .put("~signature", "3a0da59fabf298d389b7b0b59728e887")
                        .put("~lastUpdateAt", "2014-08-28T21:24:36.440Z")
                        .put("~firstUpdateAt", "2014-06-07T09:51:40.077Z")
                        .build();

                String json = JsonHelper.asJson(value);
                out.write(json);
                out.write("\n");

                expected.add(value);
            }
            partialOut.writeTo(splitOut);
        }
    }

    splitOut.close();

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(splitOut.toByteArray(), 0, splitOut.size()));
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(splitOut.size());
    s3Object.setObjectMetadata(objectMetadata);

    AmazonS3 s3 = mock(AmazonS3.class);
    when(s3.getObject(argThat(getsObject("stash-bucket", "stash/test/2015-01-01-00-00-00/test-table/split0.gz"))))
            .thenReturn(s3Object);

    StashSplit stashSplit = new StashSplit("test:table", "2015-01-01-00-00-00/test-table/split0.gz", splitOut.size());

    StandardStashReader reader = new StandardStashReader(URI.create("s3://stash-bucket/stash/test"), s3, 0);
    StashRowIterator contentIter = reader.getSplit(stashSplit);

    List<Map<String, Object>> content = ImmutableList.copyOf(contentIter);
    assertEquals(content, expected);
}

Example 8

Source File: StashReaderTest.java From emodb with Apache License 2.0

4 votes

/**
 * Note:  This test depends on the default character set to be US-ASCII
 */
@Test
public void testUTF8ClientDecoding() throws Exception {
    if (!Charset.defaultCharset().equals(Charsets.US_ASCII)) {
        throw new SkipException("testUTF8ClientDecoding() requires default charset to be US-ASCII");
    }

    String utf8Text = "ชิงรางวัลกันจ้า";
    ByteArrayOutputStream splitOut = new ByteArrayOutputStream();

    try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(splitOut), Charsets.UTF_8))) {
        Map<String, Object> value = ImmutableMap.<String, Object>builder()
                .put("~id", "1")
                .put("~table", "test:table")
                .put("~version", 1)
                .put("~signature", "3a0da59fabf298d389b7b0b59728e887")
                .put("~lastUpdateAt", "2014-08-28T21:24:36.440Z")
                .put("~firstUpdateAt", "2014-06-07T09:51:40.077Z")
                .put("text", utf8Text)
                .build();

        String json = JsonHelper.asJson(value);
        out.write(json);
        out.write("\n");
    }

    splitOut.close();

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(splitOut.toByteArray(), 0, splitOut.size()));
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(splitOut.size());
    s3Object.setObjectMetadata(objectMetadata);

    AmazonS3 s3 = mock(AmazonS3.class);
    when(s3.getObject(argThat(getsObject("stash-bucket", "stash/test/2015-01-01-00-00-00/test-table/split0.gz"))))
            .thenReturn(s3Object);

    StashSplit stashSplit = new StashSplit("test:table", "2015-01-01-00-00-00/test-table/split0.gz", splitOut.size());

    StandardStashReader reader = new StandardStashReader(URI.create("s3://stash-bucket/stash/test"), s3, 0);
    try (StashRowIterator rowIter = reader.getSplit(stashSplit)) {
        assertTrue(rowIter.hasNext());
        Map<String, Object> row = rowIter.next();
        assertEquals(row.get("text"), utf8Text);
    }
}

Example 9

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

4 votes

@Test
public void testGetObject() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertFalse(request.isRequesterPays());
    assertNull(request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}

Example 10

Source File: TestFetchS3Object.java From nifi with Apache License 2.0

4 votes

@Test
public void testGetObjectWithRequesterPays() throws IOException {
    runner.setProperty(FetchS3Object.REGION, "us-east-1");
    runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
    runner.setProperty(FetchS3Object.REQUESTER_PAYS, "true");
    final Map<String, String> attrs = new HashMap<>();
    attrs.put("filename", "request-key");
    runner.enqueue(new byte[0], attrs);

    S3Object s3ObjectResponse = new S3Object();
    s3ObjectResponse.setBucketName("response-bucket-name");
    s3ObjectResponse.setKey("response-key");
    s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
    ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
    metadata.setContentDisposition("key/path/to/file.txt");
    metadata.setContentType("text/plain");
    metadata.setContentMD5("testMD5hash");
    Date expiration = new Date();
    metadata.setExpirationTime(expiration);
    metadata.setExpirationTimeRuleId("testExpirationRuleId");
    Map<String, String> userMetadata = new HashMap<>();
    userMetadata.put("userKey1", "userValue1");
    userMetadata.put("userKey2", "userValue2");
    metadata.setUserMetadata(userMetadata);
    metadata.setSSEAlgorithm("testAlgorithm");
    Mockito.when(metadata.getETag()).thenReturn("test-etag");
    s3ObjectResponse.setObjectMetadata(metadata);
    Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);

    runner.run(1);

    ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
    GetObjectRequest request = captureRequest.getValue();
    assertEquals("request-bucket", request.getBucketName());
    assertEquals("request-key", request.getKey());
    assertTrue(request.isRequesterPays());
    assertNull(request.getVersionId());

    runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
    final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
    MockFlowFile ff = ffs.get(0);
    ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
    ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
    ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
    ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
    ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
    ff.assertAttributeEquals("hash.value", "testMD5hash");
    ff.assertAttributeEquals("hash.algorithm", "MD5");
    ff.assertAttributeEquals("s3.etag", "test-etag");
    ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
    ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
    ff.assertAttributeEquals("userKey1", "userValue1");
    ff.assertAttributeEquals("userKey2", "userValue2");
    ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
    ff.assertContentEquals("Some Content");
}