com.amazonaws.services.s3.model.AmazonS3Exception Java Examples

The following examples show how to use com.amazonaws.services.s3.model.AmazonS3Exception. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: COSAPIClient.java    From stocator with Apache License 2.0 6 votes vote down vote up
private FileStatus getFileStatusKeyBased(String key, Path path) throws AmazonS3Exception {
  LOG.trace("internal method - get file status by key {}, path {}", key, path);
  FileStatus cachedFS = memoryCache.getFileStatus(path.toString());
  if (cachedFS != null) {
    return cachedFS;
  }
  ObjectMetadata meta = mClient.getObjectMetadata(mBucket, key);
  String sparkOrigin = meta.getUserMetaDataOf("data-origin");
  boolean stocatorCreated = false;
  if (sparkOrigin != null) {
    String tmp = (String) sparkOrigin;
    if (tmp.equals("stocator")) {
      stocatorCreated = true;
    }
  }
  mCachedSparkOriginated.put(key, Boolean.valueOf(stocatorCreated));
  FileStatus fs = createFileStatus(meta.getContentLength(), key, meta.getLastModified(), path);
  LOG.trace("getFileStatusKeyBased: key {} fs.path {}", key, fs.getPath());
  memoryCache.putFileStatus(path.toString(), fs);
  return fs;
}
 
Example #2
Source File: AwsSdkTest.java    From s3proxy with Apache License 2.0 6 votes vote down vote up
@Test
public void testBlobRemove() throws Exception {
    String blobName = "blob";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);
    assertThat(client.getObjectMetadata(containerName, blobName))
            .isNotNull();

    client.deleteObject(containerName, blobName);
    try {
        client.getObjectMetadata(containerName, blobName);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("404 Not Found");
    }

    client.deleteObject(containerName, blobName);
}
 
Example #3
Source File: BucketManager.java    From nexus-public with Eclipse Public License 1.0 6 votes vote down vote up
@Override
public void prepareStorageLocation(final BlobStoreConfiguration blobStoreConfiguration) {
  String bucket = getConfiguredBucket(blobStoreConfiguration);
  checkPermissions(getConfiguredBucket(blobStoreConfiguration));
  if (!s3.doesBucketExistV2(bucket)) {
    try {
      s3.createBucket(bucket);
    }
    catch (AmazonS3Exception e) {
      if (ACCESS_DENIED_CODE.equals(e.getErrorCode())) {
        log.debug("Error creating bucket {}", bucket, e);
        throw insufficientCreatePermissionsError();
      }
      log.info("Error creating bucket {}", bucket, e);
      throw unexpectedError("creating bucket");
    }
    setBucketLifecycleConfiguration(s3, blobStoreConfiguration, null);
  }
  else {
    // bucket exists, we should test that the correct lifecycle config is present
    BucketLifecycleConfiguration lifecycleConfiguration = s3.getBucketLifecycleConfiguration(bucket);
    if (!isExpirationLifecycleConfigurationPresent(lifecycleConfiguration, blobStoreConfiguration)) {
      setBucketLifecycleConfiguration(s3, blobStoreConfiguration, lifecycleConfiguration);
    }
  }
}
 
Example #4
Source File: S3FileObjectTest.java    From hop with Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  String testKey = BUCKET_NAME + "/" + origKey;
  String testBucket = "badBucketName";
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.handleAttachException( testKey, testBucket );
  } catch ( FileSystemException e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 
Example #5
Source File: AwsSdkTest.java    From s3proxy with Apache License 2.0 6 votes vote down vote up
@Test
public void testAwsV4SignatureBadIdentity() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(
                    new BasicAWSCredentials(
                            "bad-access-key", awsCreds.getAWSSecretKey())))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());

    try {
        client.putObject(containerName, "foo",
                BYTE_SOURCE.openStream(), metadata);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("InvalidAccessKeyId");
    }
}
 
Example #6
Source File: AwsObjectStorageConnector.java    From cloudbreak with Apache License 2.0 6 votes vote down vote up
@Override
public ObjectStorageMetadataResponse getObjectStorageMetadata(ObjectStorageMetadataRequest request) {
    AwsCredentialView awsCredentialView = new AwsCredentialView(request.getCredential());
    try {
        AmazonS3 s3Client = awsClient.createS3Client(awsCredentialView);
        String bucketLocation = fixBucketLocation(s3Client.getBucketLocation(request.getObjectStoragePath()));
        return ObjectStorageMetadataResponse.builder()
                .withRegion(bucketLocation)
                .withStatus(ResponseStatus.OK)
                .build();
    } catch (AmazonS3Exception e) {
        // HACK let's assume that if the user gets back 403 Access Denied it is because s/he does not have the s3:GetBucketLocation permission.
        // It is also true though that if the bucket indeed exists but it is in another account or otherwise denied from the requesting user,
        // the same error code will be returned. However, this hack is mainly for QAAS.
        if (e.getStatusCode() != ACCESS_DENIED_ERROR_CODE) {
            throw new CloudConnectorException(String.format("Cannot get object storage location for %s. "
                    + "Provider error message: %s", request.getObjectStoragePath(), e.getErrorMessage()), e);
        }
        return ObjectStorageMetadataResponse.builder()
                .withStatus(ResponseStatus.ACCESS_DENIED)
                .build();
    }
}
 
Example #7
Source File: S3NFileObjectTest.java    From hop with Apache License 2.0 6 votes vote down vote up
@Test
public void testHandleAttachExceptionEmptyFolder() throws FileSystemException {
  AmazonS3Exception exception = new AmazonS3Exception( "NoSuchKey" );
  exception.setErrorCode( "NoSuchKey" );

  //test the case where the folder exists and contains things; no exception should be thrown
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey ) ).thenThrow( exception );
  when( s3ServiceMock.getObject( BUCKET_NAME, origKey + "/" ) ).thenThrow( exception );
  childObjectListing = mock( ObjectListing.class );
  when( childObjectListing.getObjectSummaries() ).thenReturn( new ArrayList<>() );
  when( childObjectListing.getCommonPrefixes() ).thenReturn( new ArrayList<>() );
  when( s3ServiceMock.listObjects( any( ListObjectsRequest.class ) ) ).thenReturn( childObjectListing );
  try {
    s3FileObjectFileSpy.doAttach();
  } catch ( Exception e ) {
    fail( "Caught exception " + e.getMessage() );
  }
  assertEquals( FileType.IMAGINARY, s3FileObjectFileSpy.getType() );
}
 
Example #8
Source File: S3CommonFileObject.java    From hop with Apache License 2.0 6 votes vote down vote up
private void handleAttachExceptionFallback( String bucket, String keyWithDelimiter, AmazonS3Exception exception )
  throws FileSystemException {
  ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
    .withBucketName( bucket )
    .withPrefix( keyWithDelimiter )
    .withDelimiter( DELIMITER );
  ObjectListing ol = fileSystem.getS3Client().listObjects( listObjectsRequest );

  if ( !( ol.getCommonPrefixes().isEmpty() && ol.getObjectSummaries().isEmpty() ) ) {
    injectType( FileType.FOLDER );
  } else {
    //Folders don't really exist - they will generate a "NoSuchKey" exception
    // confirms key doesn't exist but connection okay
    String errorCode = exception.getErrorCode();
    if ( !errorCode.equals( "NoSuchKey" ) ) {
      // bubbling up other connection errors
      logger.error( "Could not get information on " + getQualifiedName(),
        exception ); // make sure this gets printed for the user
      throw new FileSystemException( "vfs.provider/get-type.error", getQualifiedName(), exception );
    }
  }
}
 
Example #9
Source File: TestPrestoS3FileSystem.java    From presto with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions"})
@Test
public void testReadRetryCounters()
        throws Exception
{
    try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) {
        int maxRetries = 2;
        MockAmazonS3 s3 = new MockAmazonS3();
        s3.setGetObjectHttpErrorCode(HTTP_INTERNAL_ERROR);
        Configuration configuration = new Configuration(false);
        configuration.set(S3_MAX_BACKOFF_TIME, "1ms");
        configuration.set(S3_MAX_RETRY_TIME, "5s");
        configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries);
        fs.initialize(new URI("s3n://test-bucket/"), configuration);
        fs.setS3Client(s3);
        try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) {
            inputStream.read();
        }
        catch (Throwable expected) {
            assertInstanceOf(expected, AmazonS3Exception.class);
            assertEquals(((AmazonS3Exception) expected).getStatusCode(), HTTP_INTERNAL_ERROR);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries);
            assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries);
        }
    }
}
 
Example #10
Source File: S3ConfigProvider.java    From exhibitor with Apache License 2.0 6 votes vote down vote up
private S3Object getConfigObject() throws Exception
{
    try
    {
        S3Object object = s3Client.getObject(arguments.getBucket(), arguments.getKey());
        if ( object.getObjectMetadata().getContentLength() > 0 )
        {
            return object;
        }
    }
    catch ( AmazonS3Exception e )
    {
        if ( !isNotFoundError(e) )
        {
            throw e;
        }
    }
    return null;
}
 
Example #11
Source File: S3FileSystemTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void matchNonGlobForbidden() {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  AmazonS3Exception exception = new AmazonS3Exception("mock exception");
  exception.setStatusCode(403);
  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/keyname");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
      .thenThrow(exception);

  assertThat(
      s3FileSystem.matchNonGlobPath(path),
      MatchResultMatcher.create(MatchResult.Status.ERROR, new IOException(exception)));
}
 
Example #12
Source File: WallRideResourceTemplateResource.java    From wallride with Apache License 2.0 6 votes vote down vote up
public Reader reader() throws IOException {
	// Will never return null, but an IOException if not found
	try {
		final InputStream inputStream = this.resource.getInputStream();
		if (!StringUtils.isEmptyOrWhitespace(this.characterEncoding)) {
			return new BufferedReader(new InputStreamReader(new BufferedInputStream(inputStream), this.characterEncoding));
		}

		return new BufferedReader(new InputStreamReader(new BufferedInputStream(inputStream)));
	} catch (AmazonS3Exception e) {
		if (e.getStatusCode() == 404) {
			throw new IOException(e);
		}
		throw e;
	}
}
 
Example #13
Source File: S3FileSystemTest.java    From beam with Apache License 2.0 6 votes vote down vote up
@Test
public void matchNonGlobNotFound() {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
  AmazonS3Exception exception = new AmazonS3Exception("mock exception");
  exception.setStatusCode(404);
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(path.getBucket(), path.getKey())))))
      .thenThrow(exception);

  MatchResult result = s3FileSystem.matchNonGlobPath(path);
  assertThat(
      result,
      MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()));
}
 
Example #14
Source File: S3FileSystem.java    From beam with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
MatchResult matchNonGlobPath(S3ResourceId path) {
  ObjectMetadata s3Metadata;
  try {
    s3Metadata = getObjectMetadata(path);
  } catch (AmazonClientException e) {
    if (e instanceof AmazonS3Exception && ((AmazonS3Exception) e).getStatusCode() == 404) {
      return MatchResult.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException());
    }
    return MatchResult.create(MatchResult.Status.ERROR, new IOException(e));
  }

  return MatchResult.create(
      MatchResult.Status.OK,
      ImmutableList.of(
          createBeamMetadata(
              path.withSize(s3Metadata.getContentLength())
                  .withLastModified(s3Metadata.getLastModified()),
              Strings.nullToEmpty(s3Metadata.getContentEncoding()))));
}
 
Example #15
Source File: MockAmazonS3.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final String blobName = request.getKey();
    final byte[] content = blobs.get(blobName);
    if (content == null) {
        AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
        exception.setStatusCode(404);
        throw exception;
    }

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(content.length);

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
    s3Object.setKey(blobName);
    s3Object.setObjectMetadata(metadata);

    return s3Object;
}
 
Example #16
Source File: S3Restorer.java    From cassandra-backup with Apache License 2.0 6 votes vote down vote up
@Override
public void downloadFile(final Path localPath, final RemoteObjectReference objectReference) throws Exception {
    final GetObjectRequest getObjectRequest = new GetObjectRequest(request.storageLocation.bucket, objectReference.canonicalPath);

    Files.createDirectories(localPath.getParent());

    final Optional<AmazonClientException> exception = ofNullable(transferManager.download(getObjectRequest,
                                                                                          localPath.toFile(),
                                                                                          new DownloadProgressListener(objectReference)).waitForException());

    if (exception.isPresent()) {
        if (exception.get() instanceof AmazonS3Exception && ((AmazonS3Exception) exception.get()).getStatusCode() == 404) {
            logger.error("Remote object reference {} does not exist.", objectReference);
        }

        throw exception.get();
    }
}
 
Example #17
Source File: S3ScanWriter.java    From emodb with Apache License 2.0 6 votes vote down vote up
@Override
protected boolean writeScanCompleteFile(URI fileUri, byte[] contents)
        throws IOException {
    String bucket = fileUri.getHost();
    String key = getKeyFromPath(fileUri);

    try {
        // The following will throw an exception unless the file already exists
        _amazonS3.getObjectMetadata(bucket, key);
        return false;
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() != Response.Status.NOT_FOUND.getStatusCode()) {
            // Expected case is not found, meaning the file does not exist
            // All other cases are some unexpected error
            throw new IOException(e);
        }
    }

    uploadContents(bucket, key, contents);
    return true;
}
 
Example #18
Source File: AwsSdkTest.java    From s3proxy with Apache License 2.0 6 votes vote down vote up
@Test
public void testPartNumberMarker() throws Exception {
    String blobName = "foo";
    InitiateMultipartUploadResult result = client.initiateMultipartUpload(
            new InitiateMultipartUploadRequest(containerName, blobName));
    ListPartsRequest request = new ListPartsRequest(containerName,
            blobName, result.getUploadId());

    client.listParts(request.withPartNumberMarker(0));

    try {
        client.listParts(request.withPartNumberMarker(1));
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("NotImplemented");
    }
}
 
Example #19
Source File: PacmanUtils.java    From pacbot with Apache License 2.0 6 votes vote down vote up
public static boolean checkACLAccess(AmazonS3Client awsS3Client, String s3BucketName, String accessType) {
    logger.info("inside the checkACLAccess method");
    Boolean openAcces = false;
    AccessControlList bucketAcl;
    List<Permission> permissionList = null;
    try {
        bucketAcl = awsS3Client.getBucketAcl(s3BucketName);

        List<Grant> grants = bucketAcl.getGrantsAsList();

        // Check grants has which permission
        if (!CollectionUtils.isNullOrEmpty(grants)) {

            permissionList = checkAnyGrantHasOpenToReadOrWriteAccess(grants, accessType);
            if (!CollectionUtils.isNullOrEmpty(permissionList)) {
                openAcces = true;
            }
        }

    } catch (AmazonS3Exception s3Exception) {
        logger.error("error : ", s3Exception);
        throw new RuleExecutionFailedExeption(s3Exception.getMessage());
    }
    return openAcces;
}
 
Example #20
Source File: AmazonS3Provider.java    From emodb with Apache License 2.0 6 votes vote down vote up
public String getRegionForBucket(String bucket) {
    // Just querying for the location for a bucket can be done with the local client
    AmazonS3 client = getLocalS3Client();
    try {
        String region = client.getBucketLocation(bucket);
        if ("US".equals(region)) {
            // GetBucketLocation requests return null for us-east-1 which the SDK then replaces with "US".
            // So change it to the actual region.
            region = "us-east-1";
        }
        return region;
    } catch (AmazonS3Exception e) {
        if (e.getStatusCode() == Response.Status.NOT_FOUND.getStatusCode()) {
            // If the bucket doesn't exist then return null
            return null;
        }
        throw e;
    }
}
 
Example #21
Source File: AwsSdkTest.java    From s3proxy with Apache License 2.0 6 votes vote down vote up
@Test
public void testAwsV4SignatureBadCredential() throws Exception {
    client = AmazonS3ClientBuilder.standard()
            .withCredentials(new AWSStaticCredentialsProvider(
                    new BasicAWSCredentials(
                            awsCreds.getAWSAccessKeyId(),
                            "bad-secret-key")))
            .withEndpointConfiguration(s3EndpointConfig)
            .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());

    try {
        client.putObject(containerName, "foo",
                BYTE_SOURCE.openStream(), metadata);
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("SignatureDoesNotMatch");
    }
}
 
Example #22
Source File: S3Util.java    From teamcity-s3-artifact-storage-plugin with Apache License 2.0 6 votes vote down vote up
public static <T> T withClientCorrectingRegion(@NotNull final AmazonS3 s3Client,
                                               @NotNull final Map<String, String> settings,
                                               @NotNull final WithS3<T, AmazonS3Exception> withCorrectedClient) {
  try {
    return withCorrectedClient.run(s3Client);
  } catch (AmazonS3Exception awsException) {
    final String correctRegion = extractCorrectedRegion(awsException);
    if (correctRegion != null) {
      LOGGER.debug("Running operation with corrected S3 region [" + correctRegion + "]", awsException);
      final HashMap<String, String> correctedSettings = new HashMap<>(settings);
      correctedSettings.put(REGION_NAME_PARAM, correctRegion);
      return withS3Client(correctedSettings, withCorrectedClient);
    } else {
      throw awsException;
    }
  }
}
 
Example #23
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Override
protected ContainerHolder getUnknownContainer(String containerName) throws IOException {
  // Per docs, if invalid security credentials are used to execute
  // AmazonS3#doesBucketExist method, the client is not able to distinguish
  // between bucket permission errors and invalid credential errors, and the
  // method could return an incorrect result.

  // Coordinator node gets the new bucket information by overall refresh in the containerMap
  // This method is implemented only for the cases when executor is falling behind.
  boolean containerFound = false;
  try {
    // getBucketLocation ensures that given user account has permissions for the bucket.
    containerFound = s3.doesBucketExistV2(containerName) &&
      s3.getBucketAcl(containerName).getGrantsAsList().stream()
        .anyMatch(g -> g.getGrantee().getIdentifier().equals(s3.getS3AccountOwner().getId()));
  } catch (AmazonS3Exception e) {
    if (e.getMessage().contains("Access Denied")) {
      // Ignorable because user doesn't have permissions. We'll omit this case.
      logger.info("Ignoring \"" + containerName + "\" because of logged in AWS account doesn't have access rights on this bucket." + e.getMessage());
    }
    logger.error("Error while looking up for the unknown container " + containerName, e);
  }
  return containerFound ? new BucketCreator(getConf(), containerName).toContainerHolder() : null;
}
 
Example #24
Source File: AmazonS3ProxyFactory.java    From spring-cloud-aws with Apache License 2.0 6 votes vote down vote up
@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
	try {
		return invocation.proceed();
	}
	catch (AmazonS3Exception e) {
		if (301 == e.getStatusCode()) {
			AmazonS3 redirectClient = buildAmazonS3ForRedirectLocation(
					this.amazonS3, e);
			return ReflectionUtils.invokeMethod(invocation.getMethod(),
					redirectClient, invocation.getArguments());
		}
		else {
			throw e;
		}
	}
}
 
Example #25
Source File: S3RateLimiter.java    From emodb with Apache License 2.0 5 votes vote down vote up
private boolean isRequestRateExceededException(Throwable t) {
    if (t instanceof AmazonS3Exception) {
        AmazonS3Exception e = (AmazonS3Exception) t;
        // Several ways AWS communicates rate limit exceeded: 503 status codes and "SlowDown" error codes.
        // Check for either.
        return e.getStatusCode() == HttpStatus.SC_SERVICE_UNAVAILABLE ||
                (e.getErrorCode() != null && e.getErrorCode().toLowerCase().contains("slowdown"));

    }
    return false;
}
 
Example #26
Source File: ScholarBucketPaperSource.java    From science-parse with Apache License 2.0 5 votes vote down vote up
private S3Object getS3Object(final String paperId) {
    final String key = paperId.substring(0, 4) + "/" + paperId.substring(4) + ".pdf";

    for(int bucketIndex = 0; bucketIndex < buckets.length; ++bucketIndex) {
        try {
            return s3.getObject(buckets[bucketIndex], key);
        } catch (final AmazonS3Exception e) {
            if(bucketIndex < buckets.length - 1 && e.getStatusCode() == 404)
                continue;   // Try again with the next bucket.

            final AmazonS3Exception rethrown =
                new AmazonS3Exception(
                    String.format(
                        "Error for key s3://%s/%s",
                        bucket,
                        key),
                    e);
            rethrown.setExtendedRequestId(e.getExtendedRequestId());
            rethrown.setErrorCode(e.getErrorCode());
            rethrown.setErrorType(e.getErrorType());
            rethrown.setRequestId(e.getRequestId());
            rethrown.setServiceName(e.getServiceName());
            rethrown.setStatusCode(e.getStatusCode());
            throw rethrown;
        }
    }

    throw new IllegalStateException("We should never get here.");
}
 
Example #27
Source File: TestS3FileSystem.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Test
public void testUnknownContainerExistsButNoPermissions() {
  TestExtendedS3FileSystem fs = new TestExtendedS3FileSystem();
  AmazonS3 mockedS3Client = mock(AmazonS3.class);
  when(mockedS3Client.doesBucketExistV2(any(String.class))).thenReturn(true);
  when(mockedS3Client.getBucketAcl(any(String.class))).thenThrow(new AmazonS3Exception("Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied; Request ID: FF025EBC3B2BF017; S3 Extended Request ID: 9cbmmg2cbPG7+3mXBizXNJ1haZ/0FUhztplqsm/dJPJB32okQRAhRWVWyqakJrKjCNVqzT57IZU=), S3 Extended Request ID: 9cbmmg2cbPG7+3mXBizXNJ1haZ/0FUhztplqsm/dJPJB32okQRAhRWVWyqakJrKjCNVqzT57IZU="));
  fs.setCustomClient(mockedS3Client);
  try {
    assertNull(fs.getUnknownContainer("testunknown"));
  } catch (IOException e) {
    fail(e.getMessage());
  }
}
 
Example #28
Source File: S3Util.java    From teamcity-s3-artifact-storage-plugin with Apache License 2.0 5 votes vote down vote up
@Nullable
private static String extractCorrectedRegion(@NotNull final Throwable e) {
  @Nullable final AmazonS3Exception awsException = e instanceof AmazonS3Exception ? (AmazonS3Exception)e : ExceptionUtil.getCause(e, AmazonS3Exception.class);
  if (awsException != null && TeamCityProperties.getBooleanOrTrue("teamcity.internal.storage.s3.autoCorrectRegion") && awsException.getAdditionalDetails() != null) {
    final String correctRegion = awsException.getAdditionalDetails().get("Region");
    if (correctRegion != null) {
      return correctRegion;
    } else {
      return awsException.getAdditionalDetails().get("x-amz-bucket-region");
    }
  } else {
    return null;
  }
}
 
Example #29
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testListVersionsAssertHandleGenericAmazonS3Exception()
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    try
    {
        String s3BucketName = "s3BucketName";
        String s3KeyPrefix = "s3KeyPrefix";

        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
        s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName);
        s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefix);

        when(mockS3Operations.listVersions(any(), any())).thenThrow(new AmazonS3Exception("message"));

        try
        {
            s3Dao.listVersions(s3FileTransferRequestParamsDto);
            fail();
        }
        catch (Exception e)
        {
            assertEquals(IllegalStateException.class, e.getClass());
            assertEquals("Error accessing S3", e.getMessage());
        }
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
    }
}
 
Example #30
Source File: MockS3OperationsImpl.java    From herd with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
            {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}