/** * Copyright (c) 2015 Bosch Software Innovations GmbH and others. * * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html */ package org.eclipse.hawkbit.artifact.repository; import java.io.BufferedInputStream; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import org.eclipse.hawkbit.artifact.repository.model.AbstractDbArtifact; import org.eclipse.hawkbit.artifact.repository.model.DbArtifactHash; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.validation.annotation.Validated; import com.amazonaws.AmazonClientException; import com.amazonaws.RequestClientOptions; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.DeleteObjectRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectResult; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.io.BaseEncoding; /** * An {@link ArtifactRepository} implementation for the AWS S3 service. All * binaries are stored in single bucket using the configured name * {@link S3RepositoryProperties#getBucketName()}. * * From the AWS S3 documentation: * <p> * There is no limit to the number of objects that can be stored in a bucket and * no difference in performance whether you use many buckets or just a few. You * can store all of your objects in a single bucket, or you can organize them * across several buckets. * </p> */ @Validated public class S3Repository extends AbstractArtifactRepository { private static final Logger LOG = LoggerFactory.getLogger(S3Repository.class); private final AmazonS3 amazonS3; private final S3RepositoryProperties s3Properties; /** * Constructor. * * @param amazonS3 * the amazonS3 client to use * @param s3Properties * the properties which e.g. holds the name of the bucket to * store in */ public S3Repository(final AmazonS3 amazonS3, final S3RepositoryProperties s3Properties) { this.amazonS3 = amazonS3; this.s3Properties = s3Properties; } @Override protected AbstractDbArtifact store(final String tenant, final DbArtifactHash base16Hashes, final String contentType, final String tempFile) throws IOException { final File file = new File(tempFile); final S3Artifact s3Artifact = createS3Artifact(tenant, base16Hashes, contentType, file); final String key = objectKey(tenant, base16Hashes.getSha1()); LOG.info("Storing file {} with length {} to AWS S3 bucket {} with key {}", file.getName(), file.length(), s3Properties.getBucketName(), key); if (existsByTenantAndSha1(tenant, base16Hashes.getSha1())) { LOG.debug("Artifact {} already exists on S3 bucket {}, don't need to upload twice", key, s3Properties.getBucketName()); return s3Artifact; } try (final InputStream inputStream = new BufferedInputStream(new FileInputStream(file), RequestClientOptions.DEFAULT_STREAM_BUFFER_SIZE)) { final ObjectMetadata objectMetadata = createObjectMetadata(base16Hashes.getMd5(), contentType, file); final PutObjectResult result = amazonS3.putObject(s3Properties.getBucketName(), key, inputStream, objectMetadata); LOG.debug("Artifact {} stored on S3 bucket {} with server side Etag {} and MD5 hash {}", key, s3Properties.getBucketName(), result.getETag(), result.getContentMd5()); return s3Artifact; } catch (final AmazonClientException e) { throw new ArtifactStoreException("Failed to store artifact into S3 ", e); } } private S3Artifact createS3Artifact(final String tenant, final DbArtifactHash hashes, final String contentType, final File file) { return new S3Artifact(amazonS3, s3Properties, objectKey(tenant, hashes.getSha1()), hashes.getSha1(), hashes, file.length(), contentType); } private ObjectMetadata createObjectMetadata(final String mdMD5Hash16, final String contentType, final File file) { final ObjectMetadata objectMetadata = new ObjectMetadata(); final String mdMD5Hash64 = BaseEncoding.base64().encode(BaseEncoding.base16().lowerCase().decode(mdMD5Hash16)); objectMetadata.setContentMD5(mdMD5Hash64); objectMetadata.setContentType(contentType); objectMetadata.setContentLength(file.length()); if (s3Properties.isServerSideEncryption()) { objectMetadata.setHeader(Headers.SERVER_SIDE_ENCRYPTION, s3Properties.getServerSideEncryptionAlgorithm()); } return objectMetadata; } @Override public void deleteBySha1(final String tenant, final String sha1Hash) { final String key = objectKey(tenant, sha1Hash); LOG.info("Deleting S3 object from bucket {} and key {}", s3Properties.getBucketName(), key); amazonS3.deleteObject(new DeleteObjectRequest(s3Properties.getBucketName(), key)); } private static String objectKey(final String tenant, final String sha1Hash) { return sanitizeTenant(tenant) + "/" + sha1Hash; } @Override public AbstractDbArtifact getArtifactBySha1(final String tenant, final String sha1Hash) { final String key = objectKey(tenant, sha1Hash); LOG.info("Retrieving S3 object from bucket {} and key {}", s3Properties.getBucketName(), key); try (final S3Object s3Object = amazonS3.getObject(s3Properties.getBucketName(), key)) { if (s3Object == null) { return null; } final ObjectMetadata s3ObjectMetadata = s3Object.getObjectMetadata(); // the MD5Content is stored in the ETag return new S3Artifact(amazonS3, s3Properties, key, sha1Hash, new DbArtifactHash(sha1Hash, BaseEncoding.base16().lowerCase() .encode(BaseEncoding.base64().decode(s3ObjectMetadata.getETag())), null), s3ObjectMetadata.getContentLength(), s3ObjectMetadata.getContentType()); } catch (final IOException e) { LOG.error("Could not verify S3Object", e); return null; } } @Override public boolean existsByTenantAndSha1(final String tenant, final String sha1Hash) { return amazonS3.doesObjectExist(s3Properties.getBucketName(), objectKey(tenant, sha1Hash)); } @Override public void deleteByTenant(final String tenant) { final String folder = sanitizeTenant(tenant); LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder); // Delete artifacts ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/"); do { for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) { amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey()); } objects = amazonS3.listNextBatchOfObjects(objects); } while (objects.isTruncated()); } }