Java Code Examples for com.amazonaws.services.s3.model.ObjectMetadata#setContentMD5()
The following examples show how to use
com.amazonaws.services.s3.model.ObjectMetadata#setContentMD5() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: S3Utils.java From exhibitor with Apache License 2.0 | 6 votes |
|
public static ObjectMetadata simpleUploadFile(S3Client client, byte[] bytes, String bucket, String key) throws Exception
{
byte[] md5 = md5(bytes, bytes.length);
ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(bytes.length);
metadata.setLastModified(new Date());
metadata.setContentMD5(S3Utils.toBase64(md5));
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, new ByteArrayInputStream(bytes), metadata);
PutObjectResult putObjectResult = client.putObject(putObjectRequest);
if ( !putObjectResult.getETag().equals(S3Utils.toHex(md5)) )
{
throw new Exception("Unable to match MD5 for config");
}
return metadata;
}
Example 2
| Source File: S3DataGenerator.java From aws-big-data-blog with Apache License 2.0 | 6 votes |
|
private void putObject(final String key, final byte[] content, Map<String, String> userMetadata) {
final ObjectMetadata metadata = new ObjectMetadata();
metadata.setContentLength(content.length);
metadata.setContentMD5(md5b64(content));
metadata.setUserMetadata(userMetadata);
final PutObjectRequest request = new PutObjectRequest(bucket, key, new
ByteArrayInputStream(content), metadata);
S3.putObject(request);
}
Example 3
| Source File: SimpleStorageResource.java From spring-cloud-aws with Apache License 2.0 | 6 votes |
|
private void finishSimpleUpload() {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(this.currentOutputStream.size());
byte[] content = this.currentOutputStream.toByteArray();
try {
MessageDigest messageDigest = MessageDigest.getInstance("MD5");
String md5Digest = BinaryUtils.toBase64(messageDigest.digest(content));
objectMetadata.setContentMD5(md5Digest);
}
catch (NoSuchAlgorithmException e) {
throw new IllegalStateException(
"MessageDigest could not be initialized because it uses an unknown algorithm",
e);
}
SimpleStorageResource.this.amazonS3.putObject(
SimpleStorageResource.this.bucketName,
SimpleStorageResource.this.objectName,
new ByteArrayInputStream(content), objectMetadata);
// Release the memory early
this.currentOutputStream = null;
}
Example 4
| Source File: RepositoryS3.java From github-bucket with ISC License | 5 votes |
|
private boolean walk(Iterator<S3ObjectSummary> iter, ObjectId file, String path) throws IOException {
byte[] content;
byte[] newHash;
LOG.debug("Start processing file: {}", path);
try (DigestInputStream is = new DigestInputStream(repository.open(file).openStream(), DigestUtils.getMd5Digest())) {
// Get content
content = IOUtils.toByteArray(is);
// Get hash
newHash = is.getMessageDigest().digest();
}
if (isUploadFile(iter, path, Hex.encodeHexString(newHash))) {
LOG.info("Uploading file: {}", path);
ObjectMetadata bucketMetadata = new ObjectMetadata();
bucketMetadata.setContentMD5(Base64.encodeAsString(newHash));
bucketMetadata.setContentLength(content.length);
// Give Tika a few hints for the content detection
Metadata tikaMetadata = new Metadata();
tikaMetadata.set(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getName(FilenameUtils.normalize(path)));
// Fire!
try (InputStream bis = TikaInputStream.get(content, tikaMetadata)) {
bucketMetadata.setContentType(TIKA_DETECTOR.detect(bis, tikaMetadata).toString());
s3.putObject(bucket.getName(), path, bis, bucketMetadata);
return true;
}
}
LOG.info("Skipping file (same checksum): {}", path);
return false;
}
Example 5
| Source File: S3ScanWriter.java From emodb with Apache License 2.0 | 5 votes |
|
private void uploadContents(String bucket, String key, byte[] contents)
throws IOException {
int failures = 0;
boolean uploaded = false;
while (!uploaded) {
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentType(MediaType.TEXT_PLAIN);
objectMetadata.setContentLength(contents.length);
objectMetadata.setContentMD5(BinaryUtils.toBase64(Hashing.md5().hashBytes(contents).asBytes()));
try {
_amazonS3.putObject(
new PutObjectRequest(bucket, key, new ByteArrayInputStream(contents), objectMetadata));
uploaded = true;
} catch (AmazonClientException e) {
if (++failures == MAX_RETRIES) {
throw new IOException(e);
}
try {
Thread.sleep(_retryDelay.toMillis());
} catch (InterruptedException e2) {
// Stop retrying and propagate the original exception
throw new IOException(e);
}
}
}
}
Example 6
| Source File: S3Repository.java From hawkbit-extensions with Eclipse Public License 1.0 | 5 votes |
|
private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType, final File file) {
final ObjectMetadata objectMetadata = new ObjectMetadata();
final String mdMD5Hash64 = BaseEncoding.base64().encode(BaseEncoding.base16().lowerCase().decode(mdMD5Hash16));
objectMetadata.setContentMD5(mdMD5Hash64);
objectMetadata.setContentType(contentType);
objectMetadata.setContentLength(file.length());
if (s3Properties.isServerSideEncryption()) {
objectMetadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION, s3Properties.getServerSideEncryptionAlgorithm());
}
return objectMetadata;
}
Example 7
| Source File: RepositoryS3.java From github-bucket with ISC License | 5 votes |
|
private boolean walk(Iterator<S3ObjectSummary> iter, ObjectId file, String path) throws IOException {
byte[] content;
byte[] newHash;
LOG.debug("Start processing file: {}", path);
try (DigestInputStream is = new DigestInputStream(repository.open(file).openStream(), DigestUtils.getMd5Digest())) {
// Get content
content = IOUtils.toByteArray(is);
// Get hash
newHash = is.getMessageDigest().digest();
}
if (isUploadFile(iter, path, Hex.encodeHexString(newHash))) {
LOG.info("Uploading file: {}", path);
ObjectMetadata bucketMetadata = new ObjectMetadata();
bucketMetadata.setContentMD5(Base64.encodeAsString(newHash));
bucketMetadata.setContentLength(content.length);
// Give Tika a few hints for the content detection
Metadata tikaMetadata = new Metadata();
tikaMetadata.set(Metadata.RESOURCE_NAME_KEY, FilenameUtils.getName(FilenameUtils.normalize(path)));
// Fire!
try (InputStream bis = TikaInputStream.get(content, tikaMetadata)) {
bucketMetadata.setContentType(TIKA_DETECTOR.detect(bis, tikaMetadata).toString());
s3.putObject(bucket.getName(), path, bis, bucketMetadata);
return true;
}
}
LOG.info("Skipping file (same checksum): {}", path);
return false;
}
Example 8
| Source File: TestFetchS3Object.java From nifi with Apache License 2.0 | 4 votes |
|
@Test
public void testGetObject() throws IOException {
runner.setProperty(FetchS3Object.REGION, "us-east-1");
runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "request-key");
runner.enqueue(new byte[0], attrs);
S3Object s3ObjectResponse = new S3Object();
s3ObjectResponse.setBucketName("response-bucket-name");
s3ObjectResponse.setKey("response-key");
s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
metadata.setContentDisposition("key/path/to/file.txt");
metadata.setContentType("text/plain");
metadata.setContentMD5("testMD5hash");
Date expiration = new Date();
metadata.setExpirationTime(expiration);
metadata.setExpirationTimeRuleId("testExpirationRuleId");
Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("userKey1", "userValue1");
userMetadata.put("userKey2", "userValue2");
metadata.setUserMetadata(userMetadata);
metadata.setSSEAlgorithm("testAlgorithm");
Mockito.when(metadata.getETag()).thenReturn("test-etag");
s3ObjectResponse.setObjectMetadata(metadata);
Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
runner.run(1);
ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
GetObjectRequest request = captureRequest.getValue();
assertEquals("request-bucket", request.getBucketName());
assertEquals("request-key", request.getKey());
assertFalse(request.isRequesterPays());
assertNull(request.getVersionId());
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
MockFlowFile ff = ffs.get(0);
ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
ff.assertAttributeEquals("hash.value", "testMD5hash");
ff.assertAttributeEquals("hash.algorithm", "MD5");
ff.assertAttributeEquals("s3.etag", "test-etag");
ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
ff.assertAttributeEquals("userKey1", "userValue1");
ff.assertAttributeEquals("userKey2", "userValue2");
ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
ff.assertContentEquals("Some Content");
}
Example 9
| Source File: TestFetchS3Object.java From nifi with Apache License 2.0 | 4 votes |
|
@Test
public void testGetObjectWithRequesterPays() throws IOException {
runner.setProperty(FetchS3Object.REGION, "us-east-1");
runner.setProperty(FetchS3Object.BUCKET, "request-bucket");
runner.setProperty(FetchS3Object.REQUESTER_PAYS, "true");
final Map<String, String> attrs = new HashMap<>();
attrs.put("filename", "request-key");
runner.enqueue(new byte[0], attrs);
S3Object s3ObjectResponse = new S3Object();
s3ObjectResponse.setBucketName("response-bucket-name");
s3ObjectResponse.setKey("response-key");
s3ObjectResponse.setObjectContent(new StringInputStream("Some Content"));
ObjectMetadata metadata = Mockito.spy(ObjectMetadata.class);
metadata.setContentDisposition("key/path/to/file.txt");
metadata.setContentType("text/plain");
metadata.setContentMD5("testMD5hash");
Date expiration = new Date();
metadata.setExpirationTime(expiration);
metadata.setExpirationTimeRuleId("testExpirationRuleId");
Map<String, String> userMetadata = new HashMap<>();
userMetadata.put("userKey1", "userValue1");
userMetadata.put("userKey2", "userValue2");
metadata.setUserMetadata(userMetadata);
metadata.setSSEAlgorithm("testAlgorithm");
Mockito.when(metadata.getETag()).thenReturn("test-etag");
s3ObjectResponse.setObjectMetadata(metadata);
Mockito.when(mockS3Client.getObject(Mockito.any())).thenReturn(s3ObjectResponse);
runner.run(1);
ArgumentCaptor<GetObjectRequest> captureRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
Mockito.verify(mockS3Client, Mockito.times(1)).getObject(captureRequest.capture());
GetObjectRequest request = captureRequest.getValue();
assertEquals("request-bucket", request.getBucketName());
assertEquals("request-key", request.getKey());
assertTrue(request.isRequesterPays());
assertNull(request.getVersionId());
runner.assertAllFlowFilesTransferred(FetchS3Object.REL_SUCCESS, 1);
final List<MockFlowFile> ffs = runner.getFlowFilesForRelationship(FetchS3Object.REL_SUCCESS);
MockFlowFile ff = ffs.get(0);
ff.assertAttributeEquals("s3.bucket", "response-bucket-name");
ff.assertAttributeEquals(CoreAttributes.FILENAME.key(), "file.txt");
ff.assertAttributeEquals(CoreAttributes.PATH.key(), "key/path/to");
ff.assertAttributeEquals(CoreAttributes.ABSOLUTE_PATH.key(), "key/path/to/file.txt");
ff.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), "text/plain");
ff.assertAttributeEquals("hash.value", "testMD5hash");
ff.assertAttributeEquals("hash.algorithm", "MD5");
ff.assertAttributeEquals("s3.etag", "test-etag");
ff.assertAttributeEquals("s3.expirationTime", String.valueOf(expiration.getTime()));
ff.assertAttributeEquals("s3.expirationTimeRuleId", "testExpirationRuleId");
ff.assertAttributeEquals("userKey1", "userValue1");
ff.assertAttributeEquals("userKey2", "userValue2");
ff.assertAttributeEquals("s3.sseAlgorithm", "testAlgorithm");
ff.assertContentEquals("Some Content");
}
