Java Code Examples for com.amazonaws.services.s3.model.AmazonS3Exception

@SuppressWarnings({"ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions"})
@Test
public void testReadRetryCounters()
        throws Exception
{
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        int maxRetries = 2;
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(HTTP_INTERNAL_ERROR);
        Configuration configuration = new Configuration(false);
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        }
        catch (Throwable expected) {
            assertInstanceOf(expected, AmazonS3Exception.class);
            assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
        }
    }
}
 

private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
  throws FileSystemException {
  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
    .withBucketName( bucket )
    .withPrefix( keyWithDelimiter )
    .withDelimiter( DELIMITER );
  ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

  if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
    injectType( FileType.FOLDER );
  } else {
    //Folders don't really exist - they will generate a "NoSuchKey" exception
    // confirms key doesn't exist but connection okay
    String errorCode = exception.getErrorCode();
    if ( !errorCode.equals( "NoSuchKey" ) ) {
      // bubbling up other connection errors
      logger.error( "Could not get information on " + getQualifiedName(),
        exception ); // make sure this gets printed for the user
      throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
    }
  }
}
 

@Test
public void testAwsV4SignatureBadIdentity() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(
                    new BasicAWSCredentials(
                            "bad-access-key", awsCreds.getAWSSecretKey())))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());

    try {
        client.putObject(containerName, "foo",
                BYTE_SOURCE.openStream(), metadata);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("InvalidAccessKeyId");
    }
}
 

@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  String testKey = BUCKET_NAME + "/" + origKey;
  String testBucket = "badBucketName";
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.handleAttachException( testKey, testBucket );
  } catch ( FileSystemException e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 

@Override
public ObjectStorageMetadataResponse getObjectStorageMetadata(ObjectStorageMetadataRequest request) {
    AwsCredentialView awsCredentialView = new AwsCredentialView(request.getCredential());
    try {
        AmazonS3 s3Client = awsClient.createS3Client(awsCredentialView);
        String bucketLocation = fixBucketLocation(s3Client.getBucketLocation(request.getObjectStoragePath()));
        return ObjectStorageMetadataResponse.builder()
                .withRegion(bucketLocation)
                .withStatus(ResponseStatus.OK)
                .build();
    } catch (AmazonS3Exception e) {
        // HACK let's assume that if the user gets back 403 Access Denied it is because s/he does not have the s3:GetBucketLocation permission.
        // It is also true though that if the bucket indeed exists but it is in another account or otherwise denied from the requesting user,
        // the same error code will be returned. However, this hack is mainly for QAAS.
        if (e.getStatusCode() != ACCESS_DENIED_ERROR_CODE) {
            throw new CloudConnectorException(String.format("Cannot get object storage location for %s. "
                    + "Provider error message: %s", request.getObjectStoragePath(), e.getErrorMessage()), e);
        }
        return ObjectStorageMetadataResponse.builder()
                .withStatus(ResponseStatus.ACCESS_DENIED)
                .build();
    }
}
 

@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 

@Test
public void testBlobRemove() throws Exception {
    String blobName = "blob";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);
    assertThat(client.getObjectMetadata(containerName, blobName))
            .isNotNull();

    client.deleteObject(containerName, blobName);
    try {
        client.getObjectMetadata(containerName, blobName);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
    }

    client.deleteObject(containerName, blobName);
}
 

@Override
public void downloadFile(final Path localPath, final RemoteObjectReference objectReference) throws Exception {
    final GetObjectRequest getObjectRequest = new GetObjectRequest(request.storageLocation.bucket, objectReference.canonicalPath);

    Files.createDirectories(localPath.getParent());

    final Optional<AmazonClientException> exception = ofNullable(transferManager.download(getObjectRequest,
                                                                                          localPath.toFile(),
                                                                                          new DownloadProgressListener(objectReference)).waitForException());

    if (exception.isPresent()) {
        if (exception.get() instanceof AmazonS3Exception && ((AmazonS3Exception) exception.get()).getStatusCode() == 404) {
            logger.error("Remote object reference {} does not exist.", objectReference);
        }

        throw exception.get();
    }
}
 

@Test
public void testPartNumberMarker() throws Exception {
    String blobName = "foo";
    InitiateMultipartUploadResult result = client.initiateMultipartUpload(
            new InitiateMultipartUploadRequest(containerName, blobName));
    ListPartsRequest request = new ListPartsRequest(containerName,
            blobName, result.getUploadId());

    client.listParts(request.withPartNumberMarker(0));

    try {
        client.listParts(request.withPartNumberMarker(1));
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("NotImplemented");
    }
}
 

public static boolean checkACLAccess(AmazonS3Client awsS3Client, String s3BucketName, String accessType) {
    logger.info("inside the checkACLAccess method");
    Boolean openAcces = false;
    AccessControlList bucketAcl;
    List<Permission> permissionList = null;
    try {
        bucketAcl = awsS3Client.getBucketAcl(s3BucketName);

        List<Grant> grants = bucketAcl.getGrantsAsList();

        // Check grants has which permission
        if (!CollectionUtils.isNullOrEmpty(grants)) {

            permissionList = checkAnyGrantHasOpenToReadOrWriteAccess(grants, accessType);
            if (!CollectionUtils.isNullOrEmpty(permissionList)) {
                openAcces = true;
            }
        }

    } catch (AmazonS3Exception s3Exception) {
        logger.error("error : ", s3Exception);
        throw new RuleExecutionFailedExeption(s3Exception.getMessage());
    }
    return openAcces;
}
 

@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
	try {
		return invocation.proceed();
	}
	catch (AmazonS3Exception e) {
		if (301 == e.getStatusCode()) {
			AmazonS3 redirectClient = buildAmazonS3ForRedirectLocation(
					this.amazonS3, e);
			return ReflectionUtils.invokeMethod(invocation.getMethod(),
					redirectClient, invocation.getArguments());
		}
		else {
			throw e;
		}
	}
}
 

@Override
protected ContainerHolder getUnknownContainer(String containerName) throws IOException {
  // Per docs, if invalid security credentials are used to execute
  // AmazonS3#doesBucketExist method, the client is not able to distinguish
  // between bucket permission errors and invalid credential errors, and the
  // method could return an incorrect result.

  // Coordinator node gets the new bucket information by overall refresh in the containerMap
  // This method is implemented only for the cases when executor is falling behind.
  boolean containerFound = false;
  try {
    // getBucketLocation ensures that given user account has permissions for the bucket.
    containerFound = s3.doesBucketExistV2(containerName) &&
      s3.getBucketAcl(containerName).getGrantsAsList().stream()
        .anyMatch(g -> g.getGrantee().getIdentifier().equals(s3.getS3AccountOwner().getId()));
  } catch (AmazonS3Exception e) {
    if (e.getMessage().contains("Access Denied")) {
      // Ignorable because user doesn't have permissions. We'll omit this case.
      logger.info("Ignoring \"" + containerName + "\" because of logged in AWS account doesn't have access rights on this bucket." + e.getMessage());
    }
    logger.error("Error while looking up for the unknown container " + containerName, e);
  }
  return containerFound ? new BucketCreator(getConf(), containerName).toContainerHolder() : null;
}
 

public static <T> T withClientCorrectingRegion(@NotNull final AmazonS3 s3Client,
                                               @NotNull final Map<String, String> settings,
                                               @NotNull final WithS3<T, AmazonS3Exception> withCorrectedClient) {
  try {
    return withCorrectedClient.run(s3Client);
  } catch (AmazonS3Exception awsException) {
    final String correctRegion = extractCorrectedRegion(awsException);
    if (correctRegion != null) {
      LOGGER.debug("Running operation with corrected S3 region [" + correctRegion + "]", awsException);
      final HashMap<String, String> correctedSettings = new HashMap<>(settings);
      correctedSettings.put(REGION_NAME_PARAM, correctRegion);
      return withS3Client(correctedSettings, withCorrectedClient);
    } else {
      throw awsException;
    }
  }
}
 

@Test
public void testAwsV4SignatureBadCredential() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(
                    new BasicAWSCredentials(
                            awsCreds.getAWSAccessKeyId(),
                            "bad-secret-key")))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());

    try {
        client.putObject(containerName, "foo",
                BYTE_SOURCE.openStream(), metadata);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("SignatureDoesNotMatch");
    }
}
 

public String getRegionForBucket(String bucket) {
    // Just querying for the location for a bucket can be done with the local client
    AmazonS3 client = getLocalS3Client();
    try {
        String region = client.getBucketLocation(bucket);
        if ("US".equals(region)) {
            // GetBucketLocation requests return null for us-east-1 which the SDK then replaces with "US".
            // So change it to the actual region.
            region = "us-east-1";
        }
        return region;
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
            // If the bucket doesn't exist then return null
            return null;
        }
        throw e;
    }
}
 

@Override
protected boolean writeScanCompleteFile(URI fileUri, byte[] contents)
        throws IOException {
    String bucket = fileUri.getHost();
    String key = getKeyFromPath(fileUri);

    try {
        // The following will throw an exception unless the file already exists
        _amazonS3.getObjectMetadata(bucket, key);
        return false;
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() != Response.Status.NOT_FOUND.getStatusCode()) {
            // Expected case is not found, meaning the file does not exist
            // All other cases are some unexpected error
            throw new IOException(e);
        }
    }

    uploadContents(bucket, key, contents);
    return true;
}
 

@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final String blobName = request.getKey();
    final byte[] content = blobs.get(blobName);
    if (content == null) {
        AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
        exception.setStatusCode(404);
        throw exception;
    }

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(content.length);

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
    s3Object.setKey(blobName);
    s3Object.setObjectMetadata(metadata);

    return s3Object;
}
 

@VisibleForTesting
MatchResult matchNonGlobPath(S3ResourceId path) {
  ObjectMetadata s3Metadata;
  try {
    s3Metadata = getObjectMetadata(path);
  } catch (AmazonClientException e) {
    if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 404) {
      return MatchResult.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException());
    }
    return MatchResult.create(MatchResult.Status.ERROR, new IOException(e));
  }

  return MatchResult.create(
      MatchResult.Status.OK,
      ImmutableList.of(
          createBeamMetadata(
              path.withSize(s3Metadata.getContentLength())
                  .withLastModified(s3Metadata.getLastModified()),
              Strings.nullToEmpty(s3Metadata.getContentEncoding()))));
}
 

@Test
public void matchNonGlobNotFound() {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
  AmazonS3Exception exception = new AmazonS3Exception("mock exception");
  exception.setStatusCode(404);
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
      .thenThrow(exception);

  MatchResult result = s3FileSystem.matchNonGlobPath(path);
  assertThat(
      result,
      MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()));
}
 

@Test
public void matchNonGlobForbidden() {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  AmazonS3Exception exception = new AmazonS3Exception("mock exception");
  exception.setStatusCode(403);
  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/keyname");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
      .thenThrow(exception);

  assertThat(
      s3FileSystem.matchNonGlobPath(path),
      MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(exception)));
}
 

private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
  LOG.trace("internal method - get file status by key {}, path {}", key, path);
  FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
  if (cachedFS != null) {
    return cachedFS;
  }
  ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
  String sparkOrigin = meta.getUserMetaDataOf("data-origin");
  boolean stocatorCreated = false;
  if (sparkOrigin != null) {
    String tmp = (String) sparkOrigin;
    if (tmp.equals("stocator")) {
      stocatorCreated = true;
    }
  }
  mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
  FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
  LOG.trace("getFileStatusKeyBased: key {} fs.path {}", key, fs.getPath());
  memoryCache.putFileStatus(path.toString(), fs);
  return fs;
}
 

public Reader reader() throws IOException {
	// Will never return null, but an IOException if not found
	try {
		final InputStream inputStream = this.resource.getInputStream();
		if (!StringUtils.isEmptyOrWhitespace(this.characterEncoding)) {
			return new BufferedReader(new InputStreamReader(new BufferedInputStream(inputStream), this.characterEncoding));
		}

		return new BufferedReader(new InputStreamReader(new BufferedInputStream(inputStream)));
	} catch (AmazonS3Exception e) {
		if (e.getStatusCode() == 404) {
			throw new IOException(e);
		}
		throw e;
	}
}
 

private S3Object getConfigObject() throws Exception
{
    try
    {
        S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey());
        if ( object.getObjectMetadata().getContentLength() > 0 )
        {
            return object;
        }
    }
    catch ( AmazonS3Exception e )
    {
        if ( !isNotFoundError(e) )
        {
            throw e;
        }
    }
    return null;
}
 

@Override
public void prepareStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
  String bucket = getConfiguredBucket(blobStoreConfiguration);
  checkPermissions(getConfiguredBucket(blobStoreConfiguration));
  if (!s3.doesBucketExistV2(bucket)) {
    try {
      s3.createBucket(bucket);
    }
    catch (AmazonS3Exception e) {
      if (ACCESS_DENIED_CODE.equals(e.getErrorCode())) {
        log.debug("Error creating bucket {}", bucket, e);
        throw insufficientCreatePermissionsError();
      }
      log.info("Error creating bucket {}", bucket, e);
      throw unexpectedError("creating bucket");
    }
    setBucketLifecycleConfiguration(s3, blobStoreConfiguration, null);
  }
  else {
    // bucket exists, we should test that the correct lifecycle config is present
    BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
    if (!isExpirationLifecycleConfigurationPresent(lifecycleConfiguration, blobStoreConfiguration)) {
      setBucketLifecycleConfiguration(s3, blobStoreConfiguration, lifecycleConfiguration);
    }
  }
}
 

@Override
public ObjectMetadata getObjectMetadata(GetObjectMetadataRequest getObjectMetadataRequest)
{
    this.getObjectMetadataRequest = getObjectMetadataRequest;
    if (getObjectMetadataHttpCode != HTTP_OK) {
        AmazonS3Exception exception = new AmazonS3Exception("Failing getObjectMetadata call with " + getObjectMetadataHttpCode);
        exception.setStatusCode(getObjectMetadataHttpCode);
        throw exception;
    }
    return null;
}
 

@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
{
    if (getObjectHttpCode != HTTP_OK) {
        AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode);
        exception.setStatusCode(getObjectHttpCode);
        throw exception;
    }
    return null;
}
 

protected void putFileWithUserMetadata(String key, File file, Map<String, String> userMetadata) throws AmazonS3Exception, FileNotFoundException {
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setUserMetadata(userMetadata);
    PutObjectRequest putRequest = new PutObjectRequest(BUCKET_NAME, key, new FileInputStream(file), objectMetadata);

    client.putObject(putRequest);
}
 

/**
 * {@inheritDoc}
 */
@Override
public void putFile(
    @NotBlank(message = "Source local path cannot be empty.") final String srcLocalPath,
    @NotBlank(message = "Destination remote path cannot be empty") final String dstRemotePath
) throws GenieException {
    final long start = System.nanoTime();
    final Set<Tag> tags = Sets.newHashSet();
    try {
        log.debug("Called with src path {} and destination path {}", srcLocalPath, dstRemotePath);

        final AmazonS3URI s3Uri = getS3Uri(dstRemotePath);
        try {
            this.s3ClientFactory
                .getClient(s3Uri)
                .putObject(s3Uri.getBucket(), s3Uri.getKey(), new File(srcLocalPath));
        } catch (final AmazonS3Exception ase) {
            log.error("Error posting file {} to s3 due to exception {}", dstRemotePath, ase.toString());
            throw new GenieServerException("Error uploading file to s3. Filename: " + dstRemotePath, ase);
        }
        MetricsUtils.addSuccessTags(tags);
    } catch (Throwable t) {
        MetricsUtils.addFailureTagsWithException(tags, t);
        throw t;
    } finally {
        this.registry.timer(UPLOAD_TIMER_NAME, tags).record(System.nanoTime() - start, TimeUnit.NANOSECONDS);
    }
}
 

@Test
public void testContainerExists() throws Exception {
    client.headBucket(new HeadBucketRequest(containerName));
    try {
        client.headBucket(new HeadBucketRequest(
                createRandomContainerName()));
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
    }
}
 

@Test
public void testHandleAttachException() throws FileSystemException {
  String testKey = BUCKET_NAME + "/" + origKey;
  String testBucket = "badBucketName";
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  try {
    s3FileObjectFileSpy.handleAttachException( testKey, testBucket );
  } catch ( FileSystemException e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.FOLDER, s3FileObjectFileSpy.getType() );
}