Java Code Examples for com.amazonaws.services.s3.model.S3Object

public ExternalResourceMetaData getMetaData(URI location, boolean revalidate) {
    LOGGER.debug("Attempting to get resource metadata: {}", location);
    S3Object s3Object = s3Client.getMetaData(location);
    if (s3Object == null) {
        return null;
    }
    try {
        ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
        return new DefaultExternalResourceMetaData(location,
            objectMetadata.getLastModified().getTime(),
            objectMetadata.getContentLength(),
            objectMetadata.getContentType(),
            objectMetadata.getETag(),
            null); // Passing null for sha1 - TODO - consider using the etag which is an MD5 hash of the file (when less than 5Gb)
    } finally {
        IoActions.closeQuietly(s3Object);
    }
}
 

@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
        throws AmazonClientException, AmazonServiceException {
    // in ESBlobStoreContainerTestCase.java, the prefix is empty,
    // so the key and blobName are equivalent to each other
    String blobName = getObjectRequest.getKey();

    if (!blobs.containsKey(blobName)) {
        throw new AmazonS3Exception("[" + blobName + "] does not exist.");
    }

    // the HTTP request attribute is irrelevant for reading
    S3ObjectInputStream stream = new S3ObjectInputStream(
            blobs.get(blobName), null, false);
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(stream);
    return s3Object;
}
 

private S3Object doGetS3Object(URI uri, boolean isLightWeight) {
    S3RegionalResource s3RegionalResource = new S3RegionalResource(uri);
    String bucketName = s3RegionalResource.getBucketName();
    String s3BucketKey = s3RegionalResource.getKey();
    configureClient(s3RegionalResource);

    GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, s3BucketKey);
    if (isLightWeight) {
        //Skip content download
        getObjectRequest.setRange(0, 0);
    }

    try {
        return amazonS3Client.getObject(getObjectRequest);
    } catch (AmazonServiceException e) {
        String errorCode = e.getErrorCode();
        if (null != errorCode && errorCode.equalsIgnoreCase("NoSuchKey")) {
            return null;
        }
        throw ResourceExceptions.getFailed(uri, e);
    }
}
 

/**
 * Performs a {@link GetObjectRequest} to the S3 bucket by file id for the file
 *
 * @param fileLocationId Id of the file to search for
 * @return file found from S3
 */
@Override
public InputStream getFileByLocationId(String fileLocationId) {
	final String bucketName = environment.getProperty(Constants.BUCKET_NAME_ENV_VARIABLE);
	if (Strings.isNullOrEmpty(bucketName)) {
		API_LOG.warn("No bucket name is specified.");
		return null;
	}

	GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, fileLocationId);

	S3Object s3Object = amazonS3.getObject(getObjectRequest);

	API_LOG.info("Successfully retrieved the file from S3 bucket {}", getObjectRequest.getBucketName());

	return s3Object.getObjectContent();
}
 

@Override
public InputStream readBlob(String blobName) throws IOException {
    int retry = 0;
    while (retry <= blobStore.numberOfRetries()) {
        try {
            S3Object s3Object = SocketAccess.doPrivileged(() -> blobStore.client().getObject(blobStore.bucket(), buildKey(blobName)));
            return s3Object.getObjectContent();
        } catch (AmazonClientException e) {
            if (blobStore.shouldRetry(e) && (retry < blobStore.numberOfRetries())) {
                retry++;
            } else {
                if (e instanceof AmazonS3Exception) {
                    if (404 == ((AmazonS3Exception) e).getStatusCode()) {
                        throw new NoSuchFileException("Blob object [" + blobName + "] not found: " + e.getMessage());
                    }
                }
                throw e;
            }
        }
    }
    throw new BlobStoreException("retries exhausted while attempting to access blob object [name:" + blobName + ", bucket:" + blobStore.bucket() + "]");
}
 

/**
 * Tests if an object can be uploaded asynchronously
 *
 * @throws Exception not expected
 */
@Test
public void shouldUploadInParallel() throws Exception {
  final File uploadFile = new File(UPLOAD_FILE_NAME);

  s3Client.createBucket(BUCKET_NAME);

  final TransferManager transferManager = createDefaultTransferManager();
  final Upload upload =
      transferManager.upload(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));
  final UploadResult uploadResult = upload.waitForUploadResult();

  assertThat(uploadResult.getKey(), equalTo(UPLOAD_FILE_NAME));

  final S3Object getResult = s3Client.getObject(BUCKET_NAME, UPLOAD_FILE_NAME);
  assertThat(getResult.getKey(), equalTo(UPLOAD_FILE_NAME));
}
 

private void writeLocalFile(
        final S3Object s3Object, 
        final File file) {

    try(FileOutputStream fileOutputStream 
            = new FileOutputStream(file)) {

        IOUtils.copy(
                s3Object.getObjectContent(), 
                fileOutputStream);
        
    } catch (IOException ioException) {
        throw new SecretsLockerException(
                ioException);
    }

    file.deleteOnExit();
}
 

/**
 * Get the original inventory report from S3, unzip it, and transfer it into a String format.
 * @return inventReport String
 * @throws IOException when getting object from S3 fails
 * or the checksum of the inventory report and the checksum specified in the manifest file not match
 */
public String getInventoryReportToString() throws IOException {
    String inventReportKey = locator.getKey();
    String bucketName = inventoryManifest.getSourceBucket();

    try (S3Object s3InventoryReport = s3Client.getObject(
            new GetObjectRequest(bucketName, inventReportKey))) {
        InputStream objectData = s3InventoryReport.getObjectContent();
        byte[] zippedData = IOUtils.toByteArray(objectData);
        String actualChecksum = DigestUtils.md5Hex(zippedData);
        String expectedChecksum = locator.getMD5checksum();
        if (!actualChecksum.equals(expectedChecksum)) {
            throw new ChecksumMismatchException (expectedChecksum, actualChecksum);
        }
        return IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(zippedData)));
    }
}
 

/**
 * Check if the MD5s of manifest.json and manifest.checksum equal
 * if so, pull out the manifest file and map it into a POJO
 * @return inventoryManifestStorage InventoryManifest, which stores all the elements of the manifest.json file
 */
public InventoryManifest getInventoryManifest() throws Exception {
    // Get manifest.json and transfer it to String
    GetObjectRequest requestJson = new GetObjectRequest(bucketName, bucketKeyJson);
    S3Object jsonObject = s3Client.getObject(requestJson);
    String jsonFile = inputStreamToString(jsonObject.getObjectContent());
    jsonObject.close();

    // Get manifest.checksum and transfer it to String with no whitespace
    GetObjectRequest requestChecksum = new GetObjectRequest(bucketName, bucketKeyChecksum);
    S3Object checksumObject = s3Client.getObject(requestChecksum);
    String expectedChecksum = inputStreamToString(checksumObject.getObjectContent())
            .replaceAll("\\s","");
    checksumObject.close();

    // Compare manifest.json and manifest.checksum's MD5 value
    String actualChecksum = DigestUtils.md5Hex(jsonFile);
    if (!actualChecksum.equals(expectedChecksum)) {
        throw new ChecksumMismatchException (expectedChecksum, actualChecksum);
    }

    return mapper.readValue(jsonFile, InventoryManifest.class);
}
 

@SuppressWarnings("resource")
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
    putObjectRequests.add(putObjectRequest);
    
    S3Object s3Object = new S3Object();
    s3Object.setBucketName(putObjectRequest.getBucketName());
    s3Object.setKey(putObjectRequest.getKey());
    if (putObjectRequest.getFile() != null) {
        try {
            s3Object.setObjectContent(new FileInputStream(putObjectRequest.getFile()));
        } catch (FileNotFoundException e) {
            throw new AmazonServiceException("Cannot store the file object.", e);
        }
    } else {
        s3Object.setObjectContent(putObjectRequest.getInputStream());
    }
    objects.add(s3Object);
    
    PutObjectResult putObjectResult = new PutObjectResult();
    putObjectResult.setETag("3a5c8b1ad448bca04584ecb55b836264");
    return putObjectResult;
}
 

/**
 * Puts an Object; Copies that object to a new bucket; Downloads the object from the new
 * bucket; compares checksums
 * of original and copied object
 *
 * @throws Exception if an Exception occurs
 */
@Test
public void shouldCopyObject() throws Exception {
  final File uploadFile = new File(UPLOAD_FILE_NAME);
  final String sourceKey = UPLOAD_FILE_NAME;
  final String destinationBucketName = "destinationBucket";
  final String destinationKey = "copyOf/" + sourceKey;

  final PutObjectResult putObjectResult =
      s3Client.putObject(new PutObjectRequest(BUCKET_NAME, sourceKey, uploadFile));

  final CopyObjectRequest copyObjectRequest =
      new CopyObjectRequest(BUCKET_NAME, sourceKey, destinationBucketName, destinationKey);
  s3Client.copyObject(copyObjectRequest);

  final com.amazonaws.services.s3.model.S3Object copiedObject =
      s3Client.getObject(destinationBucketName, destinationKey);

  final String copiedHash = HashUtil.getDigest(copiedObject.getObjectContent());
  copiedObject.close();

  assertThat("Sourcefile and copied File should have same Hashes",
      copiedHash,
      is(equalTo(putObjectResult.getETag())));
}
 

@SuppressWarnings("resource")
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
    putObjectRequests.add(putObjectRequest);
    
    S3Object s3Object = new S3Object();
    s3Object.setBucketName(putObjectRequest.getBucketName());
    s3Object.setKey(putObjectRequest.getKey());
    if (putObjectRequest.getFile() != null) {
        try {
            s3Object.setObjectContent(new FileInputStream(putObjectRequest.getFile()));
        } catch (FileNotFoundException e) {
            throw new AmazonServiceException("Cannot store the file object.", e);
        }
    } else {
        s3Object.setObjectContent(putObjectRequest.getInputStream());
    }
    objects.add(s3Object);
    
    PutObjectResult putObjectResult = new PutObjectResult();
    putObjectResult.setETag("3a5c8b1ad448bca04584ecb55b836264");
    return putObjectResult;
}
 

@SuppressWarnings("resource")
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
    putObjectRequests.add(putObjectRequest);
    
    S3Object s3Object = new S3Object();
    s3Object.setBucketName(putObjectRequest.getBucketName());
    s3Object.setKey(putObjectRequest.getKey());
    if (putObjectRequest.getFile() != null) {
        try {
            s3Object.setObjectContent(new FileInputStream(putObjectRequest.getFile()));
        } catch (FileNotFoundException e) {
            throw new AmazonServiceException("Cannot store the file object.", e);
        }
    } else {
        s3Object.setObjectContent(putObjectRequest.getInputStream());
    }
    objects.add(s3Object);
    
    PutObjectResult putObjectResult = new PutObjectResult();
    putObjectResult.setETag("3a5c8b1ad448bca04584ecb55b836264");
    return putObjectResult;
}
 

@Override
public boolean load(BuildCacheKey key, BuildCacheEntryReader reader) {
  if (s3.doesObjectExist(bucketName, key.getHashCode())) {
    logger.info("Found cache item '{}' in S3 bucket", key.getHashCode());
    S3Object object = s3.getObject(bucketName, key.getHashCode());
    try (InputStream is = object.getObjectContent()) {
      reader.readFrom(is);
      return true;
    } catch (IOException e) {
      throw new BuildCacheException("Error while reading cache object from S3 bucket", e);
    }
  } else {
    logger.info("Did not find cache item '{}' in S3 bucket", key.getHashCode());
    return false;
  }
}
 

@Test
public void copyOneObject() throws Exception {
  client.putObject("source", "data", inputData);

  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(7L));
  assertThat(metrics.getMetrics().get(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name()), is(7L));
  S3Object object = client.getObject("target", "data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
  assertThat(registry.getGauges().containsKey(RunningMetrics.S3S3_CP_BYTES_REPLICATED.name()), is(true));
}
 

@Test
public void copyMultipleObjects() throws Exception {
  // Making sure we only request 1 file at the time so we need to loop
  ListObjectsRequestFactory mockListObjectRequestFactory = Mockito.mock(ListObjectsRequestFactory.class);
  when(mockListObjectRequestFactory.newInstance()).thenReturn(new ListObjectsRequest().withMaxKeys(1));

  client.putObject("source", "bar/data1", inputData);
  client.putObject("source", "bar/data2", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      transferManagerFactory, mockListObjectRequestFactory, registry, s3S3CopierOptions);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(14L));

  S3Object object1 = client.getObject("target", "foo/data1");
  String data1 = IOUtils.toString(object1.getObjectContent());
  assertThat(data1, is("bar foo"));
  S3Object object2 = client.getObject("target", "foo/data2");
  String data2 = IOUtils.toString(object2.getObjectContent());
  assertThat(data2, is("bar foo"));
}
 

@SuppressWarnings("resource")
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
    putObjectRequests.add(putObjectRequest);
    
    S3Object s3Object = new S3Object();
    s3Object.setBucketName(putObjectRequest.getBucketName());
    s3Object.setKey(putObjectRequest.getKey());
    if (putObjectRequest.getFile() != null) {
        try {
            s3Object.setObjectContent(new FileInputStream(putObjectRequest.getFile()));
        } catch (FileNotFoundException e) {
            throw new AmazonServiceException("Cannot store the file object.", e);
        }
    } else {
        s3Object.setObjectContent(putObjectRequest.getInputStream());
    }
    objects.add(s3Object);
    
    PutObjectResult putObjectResult = new PutObjectResult();
    putObjectResult.setETag("3a5c8b1ad448bca04584ecb55b836264");
    return putObjectResult;
}
 

@SuppressWarnings("resource")
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
    putObjectRequests.add(putObjectRequest);
    
    S3Object s3Object = new S3Object();
    s3Object.setBucketName(putObjectRequest.getBucketName());
    s3Object.setKey(putObjectRequest.getKey());
    if (putObjectRequest.getFile() != null) {
        try {
            s3Object.setObjectContent(new FileInputStream(putObjectRequest.getFile()));
        } catch (FileNotFoundException e) {
            throw new AmazonServiceException("Cannot store the file object.", e);
        }
    } else {
        s3Object.setObjectContent(putObjectRequest.getInputStream());
    }
    objects.add(s3Object);
    
    PutObjectResult putObjectResult = new PutObjectResult();
    putObjectResult.setETag("3a5c8b1ad448bca04584ecb55b836264");
    return putObjectResult;
}
 

@SuppressWarnings("resource")
@Override
public PutObjectResult putObject(PutObjectRequest putObjectRequest) throws AmazonClientException, AmazonServiceException {
    putObjectRequests.add(putObjectRequest);
    
    S3Object s3Object = new S3Object();
    s3Object.setBucketName(putObjectRequest.getBucketName());
    s3Object.setKey(putObjectRequest.getKey());
    if (putObjectRequest.getFile() != null) {
        try {
            s3Object.setObjectContent(new FileInputStream(putObjectRequest.getFile()));
        } catch (FileNotFoundException e) {
            throw new AmazonServiceException("Cannot store the file object.", e);
        }
    } else {
        s3Object.setObjectContent(putObjectRequest.getInputStream());
    }
    objects.add(s3Object);
    
    PutObjectResult putObjectResult = new PutObjectResult();
    putObjectResult.setETag("3a5c8b1ad448bca04584ecb55b836264");
    return putObjectResult;
}
 

/**
 * Attempt to fetch a secret from S3.
 *
 * @param s3path where to fetch it from
 * @return the content of the file found on S3
 * @throws IOException on problems streaming the content of the file
 * @throws AmazonS3Exception on problems communicating with amazon
 */
private String getS3Value(final SecretPath s3path) throws IOException, AmazonS3Exception {
    LOG.info("Fetching secret from s3://" + s3path.bucket + "/" + s3path.key);
    if (s3Client == null) {
        if (awsCredentialsProvider != null) {
            s3Client = AmazonS3ClientBuilder.standard().withCredentials(awsCredentialsProvider)
                    .build();
        } else {
            s3Client = AmazonS3ClientBuilder.standard().build();
        }
    }
    final S3Object s3object
            = s3Client.getObject(new GetObjectRequest(s3path.bucket, s3path.key));
    final BufferedReader reader
            = new BufferedReader(new InputStreamReader(s3object.getObjectContent()));
    final StringBuilder b = new StringBuilder();
    String line;
    while((line = reader.readLine()) != null) {
        b.append(line);
    }
    LOG.info("Found secret");
    reader.close();
    return b.toString();
}
 

@Override
public ObjectMetadata getObject(final GetObjectRequest getObjectRequest, File destinationFile)
        throws SdkClientException, AmazonServiceException {
    rejectNull(destinationFile,
            "The destination file parameter must be specified when downloading an object directly to a file");

    S3Object s3Object = ServiceUtils.retryableDownloadS3ObjectToFile(destinationFile, new ServiceUtils.RetryableS3DownloadTask() {

        @Override
        public S3Object getS3ObjectStream() {
            return getObject(getObjectRequest);
        }

        @Override
        public boolean needIntegrityCheck() {
            return !skipMd5CheckStrategy.skipClientSideValidationPerRequest(getObjectRequest);
        }

    }, ServiceUtils.OVERWRITE_MODE);
    // getObject can return null if constraints were specified but not met
    if (s3Object == null) return null;

    return s3Object.getObjectMetadata();
}
 

/**
 * Retrieves an instruction file from S3; or null if no instruction file is
 * found.
 *
 * @param s3ObjectId
 *            the S3 object id (not the instruction file id)
 * @param instFileSuffix
 *            suffix of the instruction file to be retrieved; or null to use
 *            the default suffix.
 * @return an instruction file, or null if no instruction file is found.
 */
final S3ObjectWrapper fetchInstructionFile(S3ObjectId s3ObjectId,
        String instFileSuffix) {
    try {
        S3Object o = s3.getObject(
            createInstructionGetRequest(s3ObjectId, instFileSuffix));
        return o == null ? null : new S3ObjectWrapper(o, s3ObjectId);
    } catch (AmazonServiceException e) {
        // If no instruction file is found, log a debug message, and return
        // null.
        if (log.isDebugEnabled()) {
            log.debug("Unable to retrieve instruction file : "
                    + e.getMessage());
        }
        return null;
    }
}
 

/**
 * Same as {@link #decipher(GetObjectRequest, long[], long[], S3Object)}
 * but makes use of an instruction file with the specified suffix.
 * @param instFileSuffix never null or empty (which is assumed to have been
 * sanitized upstream.)
 */
private S3Object decipherWithInstFileSuffix(GetObjectRequest req,
        long[] desiredRange, long[] cryptoRange, S3Object retrieved,
        String instFileSuffix) {
    final S3ObjectId id = req.getS3ObjectId();
    // Check if encrypted info is in an instruction file
    final S3ObjectWrapper ifile = fetchInstructionFile(id, instFileSuffix);
    if (ifile == null) {
        throw new SdkClientException("Instruction file with suffix "
                + instFileSuffix + " is not found for " + retrieved);
    }
    try {
        return decipherWithInstructionFile(req, desiredRange,
                cryptoRange, new S3ObjectWrapper(retrieved, id), ifile);
    } finally {
        closeQuietly(ifile, log);
    }
}
 

@Override
public Stream<String> lines() {
	LOGGER.debug("starting download from {}", uri);
	AmazonS3URI s3URI = new AmazonS3URI(uri);
	S3Object s3Object = s3Client.getObject(s3URI.getBucket(), s3URI.getKey());

	InputStream stream = s3Object.getObjectContent();
	return new BufferedReader(new InputStreamReader(stream)).lines();
}
 

/**
 * Stores a file in a previously created bucket. Downloads the file again and compares checksums
 *
 * @throws Exception if FileStreams can not be read
 */
@Test
public void shouldUploadAndDownloadObject() throws Exception {
  final File uploadFile = new File(UPLOAD_FILE_NAME);

  s3Client.createBucket(BUCKET_NAME);
  s3Client.putObject(new PutObjectRequest(BUCKET_NAME, uploadFile.getName(), uploadFile));

  final S3Object s3Object = s3Client.getObject(BUCKET_NAME, uploadFile.getName());

  final InputStream uploadFileIS = new FileInputStream(uploadFile);
  final String uploadHash = HashUtil.getDigest(uploadFileIS);
  final String downloadedHash = HashUtil.getDigest(s3Object.getObjectContent());
  uploadFileIS.close();
  s3Object.close();

  assertThat("Up- and downloaded Files should have equal Hashes", uploadHash,
      is(equalTo(downloadedHash)));
}
 

public ExternalResourceReadResponse openResource(URI location, boolean revalidate) {
    LOGGER.debug("Attempting to get resource: {}", location);
    S3Object s3Object = s3Client.getResource(location);
    if (s3Object == null) {
        return null;
    }
    return new S3Resource(s3Object, location);
}
 

private String getObjectContent(String key) {
	S3Object stored = amazonS3Client.getObject(bucketName, key);

	try {
		return IOUtils.toString(stored.getObjectContent(), "UTF-8");
	} catch (IOException ioe) {
		fail("should have content");
	}
	return "";
}
 

/**
 * Verifies multipart copy.
 *
 * @throws InterruptedException
 */
@Test
public void multipartCopy() throws InterruptedException, IOException, NoSuchAlgorithmException {
  final int contentLen = 3 * _1MB;

  final ObjectMetadata objectMetadata = new ObjectMetadata();
  objectMetadata.setContentLength(contentLen);

  final String assumedSourceKey = UUID.randomUUID().toString();

  final Bucket sourceBucket = s3Client.createBucket(UUID.randomUUID().toString());
  final Bucket targetBucket = s3Client.createBucket(UUID.randomUUID().toString());

  final TransferManager transferManager = createTransferManager(_2MB, _1MB, _2MB, _1MB);

  final InputStream sourceInputStream = randomInputStream(contentLen);
  final Upload upload = transferManager
      .upload(sourceBucket.getName(), assumedSourceKey,
          sourceInputStream, objectMetadata);

  final UploadResult uploadResult = upload.waitForUploadResult();

  assertThat(uploadResult.getKey(), is(assumedSourceKey));

  final String assumedDestinationKey = UUID.randomUUID().toString();
  final Copy copy =
      transferManager.copy(sourceBucket.getName(), assumedSourceKey, targetBucket.getName(),
          assumedDestinationKey);
  final CopyResult copyResult = copy.waitForCopyResult();
  assertThat(copyResult.getDestinationKey(), is(assumedDestinationKey));

  final S3Object copiedObject = s3Client.getObject(targetBucket.getName(), assumedDestinationKey);

  assertThat("Hashes for source and target S3Object do not match.",
      HashUtil.getDigest(copiedObject.getObjectContent()) + "-1",
      is(uploadResult.getETag()));
}
 

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    if (shouldFail(bucketName, key, readFailureRate)) {
        logger.info("--> random read failure on getObject method: throwing an exception for [bucket={}, key={}]", bucketName, key);
        AmazonS3Exception ex = new AmazonS3Exception("Random S3 read exception");
        ex.setStatusCode(404);
        throw ex;
    } else {
        return super.getObject(bucketName, key);
    }
}
 

private S3Object downloadS3Object(
        final String fileName) {
    
    return s3Client
            .getObject(
                    new GetObjectRequest(
                            bucketName,
                            buildS3ObjectName(
                                    fileName)));
}
 

/**
 * Gets specified text file content from specified S3 bucket.
 *
 * @param bucketName
 * @param fileName
 * @return
 */
public String readTextFileContentFromBucket(String bucketName, String fileName) {

	final S3Object s3object = s3client.getObject(bucketName, fileName);
	final S3ObjectInputStream inputStream = s3object.getObjectContent();

	final StringWriter writer = new StringWriter();
	try {
		IOUtils.copy(inputStream, writer, "UTF-8");
	} catch (IOException ex) {
		log.error("Error copying file from s3: " + ex);
	}
	return writer.toString();
}
 

/**
 * Reads a file from S3 into a String object
 * @param s3Uri (eg. s3://bucket/file.ext)
 * @return String containing the content of the file in S3
 * @throws IOException if error reading file
 */
public String readFileFromS3(String s3Uri) throws IOException {
    AmazonS3URI s3FileUri = new AmazonS3URI(s3Uri);
    S3Object s3object = amazonS3Client.getObject(new GetObjectRequest(s3FileUri.getBucket(), s3FileUri.getKey()));
    
    return IOUtils.toString(s3object.getObjectContent());
}
 

@Test
public void testReadFileFromS3() throws IOException {
    final String testInput = "Test Input";
    final S3ObjectInputStream s3ObjectInputStream = new S3ObjectInputStream(
            new ByteArrayInputStream(testInput.getBytes()),
            mock(HttpRequestBase.class),
            false);

    final S3Object s3Object = mock(S3Object.class);
    when(s3Object.getObjectContent()).thenReturn(s3ObjectInputStream);

    when(amazonS3Client.getObject(any(GetObjectRequest.class))).thenReturn(s3Object);

    assertThat(awsHelperService.readFileFromS3("http://bucket.s3.amazonaws.com"), equalTo(testInput));
}
 

/**
 * Download an object data as a file
 *
 * @param remoteObjectName the name of object/key which contents should be downloaded
 * @param localFileName the location and file name on the local machine, where the file will be downloaded
 * @throws S3OperationException if there is an error during data transfer
 */
@PublicAtsApi
public void download( String remoteObjectName, String localFileName ) throws S3OperationException,
                                                                      IllegalArgumentException {

    AmazonS3 s3Client = getClient();

    localFileName = IoUtils.normalizeFilePath(localFileName);
    String localDirName = IoUtils.getFilePath(localFileName);
    String localFileOnlyName = IoUtils.getFileName(localFileName);
    File localDir = new File(localDirName);
    if (localDir.exists()) {
        if (localDir.isFile()) {
            throw new IllegalArgumentException("Could not create file " + localFileOnlyName + " into existing file "
                                               + localDirName);
        }
        // else dir exists
    } else {
        LOG.debug("Creating target directory path " + localDirName);
        if (!localDir.mkdirs()) {
            throw new S3OperationException("Could not create local directory path '" + localDirName
                                           + "' for local file specified '" + localFileName + "'");
        }
    }

    S3Object obj = s3Client.getObject(bucketName, remoteObjectName);
    try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(localFileName)));
            S3ObjectInputStream s3is = obj.getObjectContent();) {
        byte[] readBuffArr = new byte[4096];
        int readBytes = 0;
        while ( (readBytes = s3is.read(readBuffArr)) >= 0) {
            bos.write(readBuffArr, 0, readBytes);
        }
    } catch (Exception e) {
        handleExeption(e, "Error while downloading object " + remoteObjectName + " to local file " + localFileName
                          + ". If error persists check your endpoint, credentials and permissions.");
    }
    LOG.info("S3 object '" + remoteObjectName + "; is downloaded successfully from bucket '" + bucketName
             + "' to file " + localFileName);
}
 

private Position getPositionFromKey(String key) {
    try {
        S3Object object = s3.getObject(new GetObjectRequest(BUCKET_NAME, key));
        BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent()));
        StringBuilder sb = new StringBuilder();
        while (true) {
            String line = reader.readLine();
            if (line == null) break;
            sb.append(line);
        }
        return new Gson().fromJson(sb.toString(), Position.class);
    } catch (Exception e) {
        return null;
    }
}
 

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    for (S3Object s3Object : objects) {
        if (bucketName.equals(s3Object.getBucketName()) && key.equals(s3Object.getKey())) {
            return s3Object;
        }
    }
    
    return null;
}
 

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    for (S3Object s3Object : objects) {
        if (bucketName.equals(s3Object.getBucketName()) && key.equals(s3Object.getKey())) {
            return s3Object;
        }
    }
    
    return null;
}
 

@Test
public void copyOneObjectUsingKeys() throws Exception {
  client.putObject("source", "bar/data", inputData);

  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  s3s3Copier.copy();
  S3Object object = client.getObject("target", "foo/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Test
public void copyOneObjectPartitioned() throws Exception {
  client.putObject("source", "year=2016/data", inputData);

  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = Lists.newArrayList(new Path(sourceBaseLocation, "year=2016"));
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(7L));
  S3Object object = client.getObject("target", "foo/year=2016/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Test
public void copyOneObjectPartitionedHandlingS3ASchemes() throws Exception {
  client.putObject("source", "year=2016/data", inputData);

  Path sourceBaseLocation = new Path("s3a://source/");
  Path replicaLocation = new Path("s3a://target/foo/");
  List<Path> sourceSubLocations = Lists.newArrayList(new Path(sourceBaseLocation, "year=2016"));
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  s3s3Copier.copy();
  S3Object object = client.getObject("target", "foo/year=2016/data");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
}
 

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    for (S3Object s3Object : objects) {
        if (bucketName.equals(s3Object.getBucketName()) && key.equals(s3Object.getKey())) {
            return s3Object;
        }
    }
    
    return null;
}
 

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    for (S3Object s3Object : objects) {
        if (bucketName.equals(s3Object.getBucketName()) && key.equals(s3Object.getKey())) {
            return s3Object;
        }
    }
    
    return null;
}
 

@Override
public S3Object getObject(String bucketName, String key) throws AmazonClientException, AmazonServiceException {
    for (S3Object s3Object : objects) {
        if (bucketName.equals(s3Object.getBucketName()) && key.equals(s3Object.getKey())) {
            return s3Object;
        }
    }
    
    return null;
}
 

@Test
public void closesIfIOExceptionWhileReading() throws Exception {
    AmazonS3Client mockClient = mock(AmazonS3Client.class);
    when(mockClientBuilder.build()).thenReturn(mockClient);

    AWSKMSClient mockKmsClient = mock(AWSKMSClient.class);
    when(mockKmsClientBuilder.build()).thenReturn(mockKmsClient);

    S3Object mockS3Object = mock(S3Object.class);
    when(mockClient.getObject(any(GetObjectRequest.class))).thenReturn(mockS3Object);
    S3ObjectInputStream mockS3ObjectInputStream = mock(S3ObjectInputStream.class);
    when(mockS3Object.getObjectContent()).thenReturn(mockS3ObjectInputStream);
    when(mockS3ObjectInputStream.read(new byte[anyInt()], anyInt(), anyByte()))
            .thenAnswer(new WriteBufferAnswer("encryptedPassword".getBytes()))
            .thenThrow(new IOException("something went wrong"))
            .thenReturn(-1);

    DecryptResult result = new DecryptResult();
    CharsetEncoder charsetEncoder = Charset.forName("UTF-8").newEncoder();
    result.setPlaintext(charsetEncoder.encode(CharBuffer.wrap("password")));
    when(mockKmsClient.decrypt(any(DecryptRequest.class))).thenReturn(result);

    Secret secret = null;
    try {
        secret = test.getPassword();
        TestCase.fail("should have thrown exception");
    } catch (AwsBucketReadingException e) {
        assertThat(e.getCause()).isInstanceOf(IOException.class);
    }

    // have we used the bucket
    ArgumentCaptor<GetObjectRequest> capturedObjectRequest = ArgumentCaptor.forClass(GetObjectRequest.class);
    verify(mockClient).getObject(capturedObjectRequest.capture());
    assertThat(capturedObjectRequest.getValue().getBucketName()).isEqualTo("bucketUri");
    assertThat(capturedObjectRequest.getValue().getS3ObjectId().getKey()).isEqualTo("/bucketPath");

    // and we have closed it even if there was an exception
    verify(mockS3Object).close();
}
 

private S3Object getS3Object(String bucket, String key) {
    S3Object o = s3Client.getObject(bucket, key);

    if (o == null) {
        throw new RuntimeException("Could not get S3 object from bucket [" + bucket + "].");
    }
    return o;
}
 

@Override
public void run() {
    super.run();

    String awsCredentialsProfile = this.readStringArgument("awsProfile", "default");
    String bucket = this.readStringArgument("bucket");
    String objectKey = this.readStringArgument("objectKey");
    String targetFilePath = this.readStringArgument("targetFile");
    Boolean overwrite = this.readBooleanArgument("overwrite", false);

    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider(awsCredentialsProfile));
    S3Object object = s3Client.getObject(
            new GetObjectRequest(bucket, objectKey));
    InputStream objectDataStream = object.getObjectContent();

    if (targetFilePath != null) {
        File targetFile = new File(targetFilePath);
        if (!targetFile.isAbsolute()) {
            targetFile = Paths.get(this.getActor().getTempDir().getAbsolutePath(), targetFilePath).toFile();
        }
        targetFile.getParentFile().mkdirs();

        try {
            if (overwrite) {
                Files.copy(objectDataStream, targetFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
            } else {
                Files.copy(objectDataStream, targetFile.toPath());
            }
        } catch (Exception ex) {
            throw new RuntimeException(String.format(
                    "Failed to transfer data from the input stream into file %s",
                    targetFilePath), ex);
        }
        
        this.writeArgument("targetFile", targetFile.getAbsolutePath());
    } else {
        // TODO: Make targetFile arg optional so this branch can execute.
        // Read data in memory and write it to an output value
    }
}
 

public void load() throws IOException {
  log.debug("Loading: {}/{}", bucket, key);

  try (S3Object object = s3.getObject(bucket, key)) {
    try (InputStream inputStream = object.getObjectContent()) {
      load(inputStream);
    }
  }
}
 

public String getString(String bucket, String key) {
    GetObjectRequest request = new GetObjectRequest(bucket, key);
    S3Object response = client.getObject(request);
    try (S3ObjectInputStream is = response.getObjectContent()) {
        return CharStreams.toString(new InputStreamReader(is, Charsets.UTF_8));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}
 

@Override
public InputStream getStream(URI uri) throws IOException {
    if (client == null) {
        client = clientBuilder.client(uri);
    }
    S3Object object = client.getObject(uri.getHost(), uri.getPath().substring(1));

    if (object != null) {
        return object.getObjectContent();
    }
    return null;
}
 

@Override
public int read(ByteBuffer dst, long position) throws IOException {
    if (!dst.hasRemaining()) {
        return 0;
    }
    S3Object object = s3.getObject(new GetObjectRequest(bucket, key)
            .withRange(position, position + dst.remaining()));
    return ByteBufferUtils.readFromInputStream(object.getObjectContent(), dst, true);
}
 

public SecureShellAuthentication(Bucket bucket, AmazonS3 client) {
    factory = new JschConfigSessionFactory() {

        @Override
        public synchronized RemoteSession getSession(URIish uri, CredentialsProvider credentialsProvider, FS fs, int tms) throws TransportException {
            // Do not check for default ssh user config
            fs.setUserHome(null);
            return super.getSession(uri, credentialsProvider, fs, tms);
        }

        @Override
        protected void configure(OpenSshConfig.Host host, Session session) {
            session.setConfig("HashKnownHosts", "no");
            if ("localhost".equalsIgnoreCase(host.getHostName())) {
                session.setConfig("StrictHostKeyChecking", "no");
            }
        }

        @Override
        protected void configureJSch(JSch jsch) {
            S3Object file;
            file = client.getObject(bucket.getName(), ".ssh/known_hosts");
            try (InputStream is = file.getObjectContent()) {
                jsch.setKnownHosts(is);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing known hosts file on s3: .ssh/known_hosts", e);
            }
            file = client.getObject(bucket.getName(), ".ssh/id_rsa");
            try (InputStream is = file.getObjectContent()) {
                jsch.addIdentity("git", IOUtils.toByteArray(is), null, new byte[0]);
            } catch (IOException | JSchException e) {
                throw new IllegalArgumentException("Missing key file on s3: .ssh/id_rsa", e);
            }
        }
    };
}
 

/**
 * Takes the result from serial download,
 * updates the transfer state and monitor in downloadImpl object
 * based on the result.
 */
private void updateDownloadStatus(S3Object result) {
    if (result == null) {
        download.setState(TransferState.Canceled);
        download.setMonitor(new DownloadMonitor(download, null));
    } else {
        download.setState(TransferState.Completed);
    }
}
 

@Deprecated
public DownloadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
        GetObjectRequest getObjectRequest, File file) {
    this(description, transferProgress, progressListenerChain, s3Object, listener,
            getObjectRequest, file, null, false);
}
 

public DownloadImpl(String description, TransferProgress transferProgress,
        ProgressListenerChain progressListenerChain, S3Object s3Object, TransferStateChangeListener listener,
        GetObjectRequest getObjectRequest, File file,
        ObjectMetadata objectMetadata, boolean isDownloadParallel) {
    super(description, transferProgress, progressListenerChain, listener);
    this.s3Object = s3Object;
    this.objectMetadata = objectMetadata;
    this.getObjectRequest = getObjectRequest;
    this.file = file;
    this.progressListenerChain = progressListenerChain;
    this.persistableDownload = captureDownloadState(getObjectRequest, file);
    S3ProgressPublisher.publishTransferPersistable(progressListenerChain, persistableDownload);
}
 

/**
 * Parses instruction data retrieved from S3 and returns a JSON string
 * representing the instruction. Made for testing purposes.
 */
static String parseInstructionFile(S3Object instructionFile) {
    try {
        return convertStreamToString(instructionFile.getObjectContent());
    } catch (Exception e) {
        throw failure(e, "Error parsing JSON instruction file");
    }
}
 

@Override
public S3Object getObjectSecurely(GetObjectRequest req) {
    appendUserAgent(req, USER_AGENT);
    // Adjust the crypto range to retrieve all of the cipher blocks needed to contain the user's desired
    // range of bytes.
    long[] desiredRange = req.getRange();
    if (isStrict() && (desiredRange  != null || req.getPartNumber() != null))
        throw new SecurityException("Range get and getting a part are not allowed in strict crypto mode");
    long[] adjustedCryptoRange = getAdjustedCryptoRange(desiredRange);
    if (adjustedCryptoRange != null)
        req.setRange(adjustedCryptoRange[0], adjustedCryptoRange[1]);
    // Get the object from S3
    S3Object retrieved = s3.getObject(req);
    // If the caller has specified constraints, it's possible that super.getObject(...)
    // would return null, so we simply return null as well.
    if (retrieved == null)
        return null;
    String suffix = null;
    if (req instanceof EncryptedGetObjectRequest) {
        EncryptedGetObjectRequest ereq = (EncryptedGetObjectRequest)req;
        suffix = ereq.getInstructionFileSuffix();
    }
    try {
        return suffix == null || suffix.trim().isEmpty()
         ? decipher(req, desiredRange, adjustedCryptoRange, retrieved)
         : decipherWithInstFileSuffix(req,
                 desiredRange, adjustedCryptoRange, retrieved,
                 suffix)
         ;
    } catch (RuntimeException ex) {
        // If we're unable to set up the decryption, make sure we close the
        // HTTP connection
        closeQuietly(retrieved, log);
        throw ex;
    } catch (Error error) {
        closeQuietly(retrieved, log);
        throw error;
    }
}
 

private S3Object decipher(GetObjectRequest req,
        long[] desiredRange, long[] cryptoRange,
        S3Object retrieved) {
    S3ObjectWrapper wrapped = new S3ObjectWrapper(retrieved, req.getS3ObjectId());
    // Check if encryption info is in object metadata
    if (wrapped.hasEncryptionInfo())
        return decipherWithMetadata(req, desiredRange, cryptoRange, wrapped);
    // Check if encrypted info is in an instruction file
    S3ObjectWrapper ifile = fetchInstructionFile(req.getS3ObjectId(), null);
    if (ifile != null) {
        try {
            return decipherWithInstructionFile(req, desiredRange,
                    cryptoRange, wrapped, ifile);
        } finally {
            closeQuietly(ifile, log);
        }
    }

    if (isStrict() || !cryptoConfig.isIgnoreMissingInstructionFile()) {
        closeQuietly(wrapped, log);
        throw new SecurityException("Instruction file not found for S3 object with bucket name: "
                + retrieved.getBucketName() + ", key: "
                + retrieved.getKey());
    }
    // To keep backward compatible:
    // ignore the missing instruction file and treat the object as un-encrypted.
    log.warn(String.format(
            "Unable to detect encryption information for object '%s' in bucket '%s'. "
                    + "Returning object without decryption.",
            retrieved.getKey(),
            retrieved.getBucketName()));
    // Adjust the output to the desired range of bytes.
    S3ObjectWrapper adjusted = adjustToDesiredRange(wrapped, desiredRange, null);
    return adjusted.getS3Object();
}
 

private S3Object decipherWithInstructionFile(GetObjectRequest req,
        long[] desiredRange, long[] cryptoRange, S3ObjectWrapper retrieved,
        S3ObjectWrapper instructionFile) {
    ExtraMaterialsDescription extraMatDesc = NONE;
    boolean keyWrapExpected = isStrict();
    if (req instanceof EncryptedGetObjectRequest) {
        EncryptedGetObjectRequest ereq = (EncryptedGetObjectRequest)req;
        extraMatDesc = ereq.getExtraMaterialDescription();
        if (!keyWrapExpected)
            keyWrapExpected = ereq.isKeyWrapExpected();
    }
    String json = instructionFile.toJsonString();
    @SuppressWarnings("unchecked")
    Map<String, String> matdesc =
        Collections.unmodifiableMap(Jackson.fromJsonString(json, Map.class));
    ContentCryptoMaterial cekMaterial =
            ContentCryptoMaterial.fromInstructionFile(
                matdesc,
                kekMaterialsProvider,
                cryptoConfig.getCryptoProvider(),
                cryptoRange,   // range is sometimes necessary to compute the adjusted IV
                extraMatDesc,
                keyWrapExpected,
                kms
        );
    securityCheck(cekMaterial, retrieved);
    S3ObjectWrapper decrypted = decrypt(retrieved, cekMaterial, cryptoRange);
    // Adjust the output to the desired range of bytes.
    S3ObjectWrapper adjusted = adjustToDesiredRange(
            decrypted, desiredRange, matdesc);
    return adjusted.getS3Object();
}
 

private S3Object decipherWithMetadata(GetObjectRequest req,
        long[] desiredRange,
        long[] cryptoRange, S3ObjectWrapper retrieved) {
    ExtraMaterialsDescription extraMatDesc = NONE;
    boolean keyWrapExpected = isStrict();
    if (req instanceof EncryptedGetObjectRequest) {
        EncryptedGetObjectRequest ereq = (EncryptedGetObjectRequest)req;
        extraMatDesc = ereq.getExtraMaterialDescription();
        if (!keyWrapExpected)
            keyWrapExpected = ereq.isKeyWrapExpected();
    }
    ContentCryptoMaterial cekMaterial = ContentCryptoMaterial
        .fromObjectMetadata(retrieved.getObjectMetadata(),
            kekMaterialsProvider,
            cryptoConfig.getCryptoProvider(),
            // range is sometimes necessary to compute the adjusted IV
            cryptoRange,
            extraMatDesc,
            keyWrapExpected,
            kms
        );
    securityCheck(cekMaterial, retrieved);
    S3ObjectWrapper decrypted = decrypt(retrieved, cekMaterial, cryptoRange);
    // Adjust the output to the desired range of bytes.
    S3ObjectWrapper adjusted = adjustToDesiredRange(
            decrypted, desiredRange, null);
    return adjusted.getS3Object();
}
 

/**
 * @see com.amazonaws.http.HttpResponseHandler#handle(com.amazonaws.http.HttpResponse)
 */
public AmazonWebServiceResponse<S3Object> handle(HttpResponse response) throws Exception {
    /*
     * TODO: It'd be nice to set the bucket name and key here, but the information isn't easy to
     * pull out of the response/request currently.
     */
    S3Object object = new S3Object();
    AmazonWebServiceResponse<S3Object> awsResponse = parseResponseMetadata(response);
    if (response.getHeaders().get(Headers.REDIRECT_LOCATION) != null) {
        object.setRedirectLocation(response.getHeaders().get(Headers.REDIRECT_LOCATION));
    }
    // If the requester is charged when downloading a object from an
    // Requester Pays bucket, then this header is set.
    if (response.getHeaders().get(Headers.REQUESTER_CHARGED_HEADER) != null) {
        object.setRequesterCharged(true);
    }

    if (response.getHeaders().get(Headers.S3_TAGGING_COUNT) != null) {
        object.setTaggingCount(Integer.parseInt(response.getHeaders().get(Headers.S3_TAGGING_COUNT)));
    }

    ObjectMetadata metadata = object.getObjectMetadata();
    populateObjectMetadata(response, metadata);

    object.setObjectContent(new S3ObjectInputStream(abortableIs(response), response.getHttpRequest()));

    awsResponse.setResult(object);
    return awsResponse;
}