Java Code Examples for com.amazonaws.services.s3.AmazonS3

The following are top voted examples for showing how to use com.amazonaws.services.s3.AmazonS3. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: s3-channels   File: S3AppendableObjectChannelTest.java   View source code 6 votes vote down vote up
@Test
void testExecutionExceptionOnClose() throws Exception {
    // test failed close
    final AmazonS3 mockedS3 = mock(AmazonS3.class);
    final S3WritableObjectChannelBuilder builder = defaultBuilder("id")
            .closeExecutorOnChannelClose(false)
            .amazonS3(mockedS3);
    s3channel = (S3AppendableObjectChannel) builder.build();
    when(mockedS3.completeMultipartUpload(any())).thenThrow(new TestException());

    assertThrows(TestException.class, () -> s3channel.close());

    s3channel.close(); // already closed, no effect, should we throw exception?

    while (s3channel.getCancellation() == null) {
        Thread.sleep(10);
    }
    s3channel.getCancellation().get();

    assertTrue(!s3channel.getCancellation().isCompletedExceptionally());
    verify(mockedS3, times(1)).abortMultipartUpload(any());
}
 
Example 2
Project: full-javaee-app   File: S3Object.java   View source code 6 votes vote down vote up
public static String webHookDump(InputStream stream, String school, String extension) {
    if (stream != null) {
        extension = extension == null || extension.isEmpty() ? ".xml" : extension.contains(".") ? extension : "." + extension;
        String fileName = "webhooks/" + school + "/" + school + "_" + Clock.getCurrentDateDashes() + "_" + Clock.getCurrentTime() + extension;
        AmazonS3 s3 = new AmazonS3Client();
        Region region = Region.getRegion(Regions.US_WEST_2);
        s3.setRegion(region);
        try {
            File file = CustomUtilities.inputStreamToFile(stream);
            s3.putObject(new PutObjectRequest(name, fileName, file));
            return CustomUtilities.fileToString(file);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return "";
}
 
Example 3
Project: radosgw-admin4j   File: RgwAdminImplTest.java   View source code 6 votes vote down vote up
@Test
public void checkBucketIndex() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();

        // not exist
        RGW_ADMIN.checkBucketIndex(bucketName, true, true);

        s3.createBucket(bucketName);
        // Do not know how to check the behavior...
        Optional result = RGW_ADMIN.checkBucketIndex(bucketName, true, true);
        assertTrue(result.isPresent());
      });
}
 
Example 4
Project: s3-channels   File: S3AppendableObjectChannelTest.java   View source code 6 votes vote down vote up
@Test
void testFailedUploadPart() throws Exception {
    final AmazonS3 mocked = mock(AmazonS3.class);
    s3channel = (S3AppendableObjectChannel) defaultBuilder("id")
            .failedPartUploadRetries(3)
            .amazonS3(mocked)
            .build();
    when(mocked.uploadPart(any())).thenThrow(new TestException());

    s3channel.skip(MIN_PART_SIZE).write(ByteBuffer.allocate(123));
    while (s3channel.getCancellation() == null) {
        Thread.sleep(25);
    }
    s3channel.getCancellation().get();
    assertTrue(!s3channel.getCancellation().isCompletedExceptionally());
    assertFalse(s3channel.isOpen());

    //coverage
    s3channel.startWorker(new UploadPartRequest().withPartNumber(1), 0);

    assertThrows(IllegalStateException.class, () -> s3channel.write(ByteBuffer.allocate(1)));

    verify(mocked, times(1)).abortMultipartUpload(any());
}
 
Example 5
Project: generator-jhipster-storage   File: _StorageService.java   View source code 6 votes vote down vote up
@Autowired
public StorageService(AmazonS3 s3Client) throws IOException {
    isLocalFileSystem = s3Client == null;

    if (isLocalFileSystem) {
        this.s3Client = null;
        this.buckets = null;
        localFileSystemDirectory =
                new File(String.format("%s/%s", new File("build").getAbsolutePath(), "storage"));
        FileUtils.forceMkdir(localFileSystemDirectory);
    } else {
        this.s3Client = s3Client;
        this.buckets = new ConcurrentHashMap<>();
        this.s3Client.listBuckets().forEach(bucket -> buckets.put(bucket.getName(), bucket));
    }
}
 
Example 6
Project: circus-train   File: S3S3CopierTest.java   View source code 6 votes vote down vote up
@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}
 
Example 7
Project: circus-train   File: S3S3CopierTest.java   View source code 6 votes vote down vote up
@Test
public void copySafelyShutDownTransferManagerWhenNotInitialised() throws Exception {
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenThrow(new RuntimeException("error in instance"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
  } catch (RuntimeException e) {
    assertThat(e.getMessage(), is("error in instance"));
  }
}
 
Example 8
Project: circus-train   File: S3S3CopierTest.java   View source code 6 votes vote down vote up
@Test
public void copyCheckTransferManagerIsShutdownWhenSubmittingJobExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenThrow(new AmazonServiceException("MyCause"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), startsWith("MyCause"));
  }
}
 
Example 9
Project: circus-train   File: S3S3CopierTest.java   View source code 6 votes vote down vote up
@Test
public void copyCheckTransferManagerIsShutdownWhenCopyExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), is("cause"));
  }
}
 
Example 10
Project: NGB-master   File: BamHelper.java   View source code 5 votes vote down vote up
@NotNull private SamInputResource getS3SamInputResource(BamFile bamFile) {
    final Bucket bucket = bucketManager.loadBucket(bamFile.getBucketId());
    Assert.notNull(bucket, getMessage(MessagesConstants.ERROR_S3_BUCKET));
    final AmazonS3 s3Client = new AmazonS3Client(new BasicAWSCredentials(bucket.getAccessKeyId(),
            bucket.getSecretAccessKey()));
    return SamInputResource.of(s3Client.generatePresignedUrl(bucket.getBucketName(), bamFile.getPath(),
            Utils.getTimeForS3URL()));
}
 
Example 11
Project: dataset-lib   File: S3Provider.java   View source code 5 votes vote down vote up
public Object get(Object[] params) {
    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider());
    try {
        S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, params[0].toString()));
        InputStream objectData = object.getObjectContent();
        byte[] bytes = IOUtils.toByteArray(is);
        ByteBuffer b = ByteBuffer.wrap(bytes);
        return b;
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which " +
            "means your request made it " +
            "to Amazon S3, but was rejected with an error response" +
            " for some reason.");
        System.out.println("Error Message:" + ase.getMessage());
        System.out.println("HTTP Status Code:" + ase.getStatusCode());
        System.out.println("AWS Error Code:" + ase.getErrorCode());
        System.out.println("Error Type:" + ase.getErrorType());
        System.out.println("Request ID:" + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which " +
            "means the client encountered " +
            "an internal error while trying to " +
            "communicate with S3, " +
            "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }

}
 
Example 12
Project: qpp-conversion-tool   File: S3Config.java   View source code 5 votes vote down vote up
/**
 * Creates the S3 client {@link Bean}.
 *
 * Uses the default client, but if a region is unspecified, uses {@code us-east-1}.
 *
 * @return The S3 client.
 */
@Bean
public AmazonS3 s3client() {
	try {
		return AmazonS3ClientBuilder.defaultClient();
	} catch (SdkClientException exception) {
		API_LOG.info("Default S3 client failed to build, trying again with region us-east-1", exception);
		return planB();
	}
}
 
Example 13
Project: elasticsearch_my   File: S3BlobContainer.java   View source code 5 votes vote down vote up
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}
 
Example 14
Project: ibm-cos-sdk-java   File: CompleteMultipartUpload.java   View source code 5 votes vote down vote up
public CompleteMultipartUpload(String uploadId, AmazonS3 s3,
        PutObjectRequest putObjectRequest, List<Future<PartETag>> futures,
        List<PartETag> eTagsBeforeResume, ProgressListenerChain progressListenerChain,
        UploadMonitor monitor) {
    this.uploadId = uploadId;
    this.s3 = s3;
    this.origReq = putObjectRequest;
    this.futures = futures;
    this.eTagsBeforeResume = eTagsBeforeResume;
    this.listener = progressListenerChain;
    this.monitor = monitor;
}
 
Example 15
Project: elasticsearch_my   File: TestAwsS3Service.java   View source code 5 votes vote down vote up
private AmazonS3 cachedWrapper(AmazonS3 client) {
    TestAmazonS3 wrapper = clients.get(client);
    if (wrapper == null) {
        wrapper = new TestAmazonS3(client, settings);
        clients.put(client, wrapper);
    }
    return wrapper;
}
 
Example 16
Project: ibm-cos-sdk-java   File: CompleteMultipartCopy.java   View source code 5 votes vote down vote up
public CompleteMultipartCopy(String uploadId, AmazonS3 s3,
                             CopyObjectRequest copyObjectRequest, List<Future<PartETag>> futures,
                             ProgressListenerChain progressListenerChain, CopyMonitor monitor) {
    this.uploadId = uploadId;
    this.s3 = s3;
    this.origReq = copyObjectRequest;
    this.futures = futures;
    this.listener = progressListenerChain;
    this.monitor = monitor;
}
 
Example 17
Project: ooso   File: Commons.java   View source code 5 votes vote down vote up
public static void storeObject(String contentType,
                               String content,
                               String destBucket,
                               String destKey) throws UnsupportedEncodingException {
    AmazonS3 s3Client = AmazonS3Provider.getS3Client();

    ObjectMetadata metadata = prepareObjectMetadata(contentType, content);

    s3Client.putObject(
            destBucket,
            destKey,
            new StringInputStream(content),
            metadata);
}
 
Example 18
Project: ooso   File: Commons.java   View source code 5 votes vote down vote up
public static void storeObject(String contentType,
                               File content,
                               String destBucket,
                               String destKey) {
    AmazonS3 s3Client = AmazonS3Provider.getS3Client();

    ObjectMetadata metadata = prepareObjectMetadata(contentType);
    s3Client.putObject(new PutObjectRequest(destBucket, destKey, content).withMetadata(metadata));
}
 
Example 19
Project: opentest   File: GetS3Metadata.java   View source code 5 votes vote down vote up
@Override
public void run() {
    super.run();

    String awsCredentialsProfile = this.readStringArgument("awsProfile", "default");
    String bucket = this.readStringArgument("bucket");
    String objectKey = this.readStringArgument("objectKey");

    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider(awsCredentialsProfile));
    ObjectMetadata metadata = s3Client.getObjectMetadata(
            new GetObjectMetadataRequest(bucket, objectKey));

    try {
        Date expirationTime = metadata.getExpirationTime();
        if (expirationTime != null) {
            this.writeOutput("expirationTime", metadata.getExpirationTime().getTime());
        } else {
            this.writeOutput("expirationTime", null);
        }
        this.writeOutput("lastModified", metadata.getLastModified().getTime());
        this.writeOutput("userMetadata", metadata.getUserMetadata());
        this.writeOutput("size", metadata.getContentLength());
        this.writeOutput("storageClass", metadata.getStorageClass());
        this.writeOutput("versionId", metadata.getVersionId());
    } catch (Exception ex) {
        throw new RuntimeException(String.format(
                "Failed to get object metadata for object key %s in bucket %s",
                objectKey,
                bucket), ex);
    }
}
 
Example 20
Project: ooso   File: Commons.java   View source code 5 votes vote down vote up
public static BufferedReader getReaderFromObjectInfo(ObjectInfoSimple objectInfo) {
    AmazonS3 s3Client = AmazonS3Provider.getS3Client();

    S3Object object = s3Client.getObject(objectInfo.getBucket(), objectInfo.getKey());
    S3ObjectInputStream objectContentRawStream = object.getObjectContent();

    return new BufferedReader(new InputStreamReader(objectContentRawStream));
}
 
Example 21
Project: s3-inventory-usage-examples   File: ManifestWriter.java   View source code 5 votes vote down vote up
public ManifestWriter(AmazonS3 client, String destBucketName, String destPrefix, String srcBucket,
                      InventoryManifest originalManifest){
    this.s3Client = client;
    this.bucketName = destBucketName;
    String time = this.getTime();
    this.manifestKey = destPrefix + "/" + srcBucket + "/" + time + "/manifest.json";
    this.checksumKey = destPrefix + "/" + srcBucket + "/" + time + "/manifest.checksum";
    this.originalManifest = originalManifest;
}
 
Example 22
Project: s3-inventory-usage-examples   File: InventoryReportLineWriter.java   View source code 5 votes vote down vote up
public InventoryReportLineWriter(AmazonS3 client, String destBucketName, String destPrefix,
                                 String srcBucket, InventoryManifest inventoryManifest) throws IOException{
    this.s3Client = client;
    this.bucketName = destBucketName;
    String uuid = UUID.randomUUID().toString();
    this.outputInventoryReportKey = destPrefix + "/" + srcBucket + "/data/" + uuid + ".csv.gz";
    this.schema = CsvSchemaFactory.buildSchema(inventoryManifest);
}
 
Example 23
Project: aem-orchestrator   File: AwsConfig.java   View source code 5 votes vote down vote up
@Bean
public AmazonS3 amazonS3Client(final AWSCredentialsProvider awsCredentialsProvider, 
    final ClientConfiguration awsClientConfig, final Region awsRegion) {
    return AmazonS3ClientBuilder.standard()
        .withCredentials(awsCredentialsProvider)
        .withClientConfiguration(awsClientConfig)
        .withRegion(awsRegion.getName())
        .build();
}
 
Example 24
Project: ats-framework   File: S3Operations.java   View source code 5 votes vote down vote up
/**
 * Download an object data as a file
 *
 * @param remoteObjectName the name of object/key which contents should be downloaded
 * @param localFileName the location and file name on the local machine, where the file will be downloaded
 * @throws S3OperationException if there is an error during data transfer
 */
@PublicAtsApi
public void download( String remoteObjectName, String localFileName ) throws S3OperationException,
                                                                      IllegalArgumentException {

    AmazonS3 s3Client = getClient();

    localFileName = IoUtils.normalizeFilePath(localFileName);
    String localDirName = IoUtils.getFilePath(localFileName);
    String localFileOnlyName = IoUtils.getFileName(localFileName);
    File localDir = new File(localDirName);
    if (localDir.exists()) {
        if (localDir.isFile()) {
            throw new IllegalArgumentException("Could not create file " + localFileOnlyName + " into existing file "
                                               + localDirName);
        }
        // else dir exists
    } else {
        LOG.debug("Creating target directory path " + localDirName);
        if (!localDir.mkdirs()) {
            throw new S3OperationException("Could not create local directory path '" + localDirName
                                           + "' for local file specified '" + localFileName + "'");
        }
    }

    S3Object obj = s3Client.getObject(bucketName, remoteObjectName);
    try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(new File(localFileName)));
            S3ObjectInputStream s3is = obj.getObjectContent();) {
        byte[] readBuffArr = new byte[4096];
        int readBytes = 0;
        while ( (readBytes = s3is.read(readBuffArr)) >= 0) {
            bos.write(readBuffArr, 0, readBytes);
        }
    } catch (Exception e) {
        handleExeption(e, "Error while downloading object " + remoteObjectName + " to local file " + localFileName
                          + ". If error persists check your endpoint, credentials and permissions.");
    }
    LOG.info("S3 object '" + remoteObjectName + "; is downloaded successfully from bucket '" + bucketName
             + "' to file " + localFileName);
}
 
Example 25
Project: Elasticsearch   File: S3ClientHelper.java   View source code 5 votes vote down vote up
protected AmazonS3 initClient(@Nullable String accessKey, @Nullable String secretKey) throws IOException {
    if (accessKey == null || secretKey == null) {
        return new AmazonS3Client(DEFAULT_CREDENTIALS_PROVIDER_CHAIN, CLIENT_CONFIGURATION);
    }
    return new AmazonS3Client(
            new BasicAWSCredentials(accessKey, secretKey),
            CLIENT_CONFIGURATION
    );
}
 
Example 26
Project: radosgw-admin4j   File: SubUserExample.java   View source code 5 votes vote down vote up
@Test
@Ignore("Not a test")
public void subuserWithS3CredentialIncorporated() throws Exception {
  testWithUserAndS3(
      (user, s3) -> {
        createSomeObjects(s3);

        List<SubUser> subUser =
            RGW_ADMIN.createSubUser(
                user.getUserId(), "QQQ", ImmutableMap.of("key-type", "s3", "access", "full"));

        User userInfo = RGW_ADMIN.getUserInfo(user.getUserId()).get();

        S3Credential subUserKey =
            userInfo
                .getS3Credentials()
                .stream()
                .filter(v -> v.getUserId().equals(subUser.get(0).getId()))
                .findFirst()
                .get();

        AmazonS3 subUserS3 = createS3(subUserKey.getAccessKey(), subUserKey.getSecretKey());

        createSomeObjects(subUserS3);

        // The S3 bucket created by parent user and created by child subuser are incorporated.
        assertEquals(s3.listBuckets().size(), subUserS3.listBuckets().size());

        for (String bucketName :
            s3.listBuckets().stream().map(v -> v.getName()).collect(Collectors.toList())) {
          assertEquals(
              s3.listObjects(bucketName).getObjectSummaries().toString(),
              subUserS3.listObjects(bucketName).getObjectSummaries().toString());
          subUserS3.getBucketAcl(bucketName);
          subUserS3.setBucketAcl(bucketName, CannedAccessControlList.AuthenticatedRead);
        }
      });
}
 
Example 27
Project: radosgw-admin4j   File: RgwAdminImplTest.java   View source code 5 votes vote down vote up
@Test
public void removeBucket() throws Exception {
  String bucketName = "testremovebkbk" + UUID.randomUUID().toString();

  // remove bucket not exist
  Thread.sleep(3000);
  RGW_ADMIN.removeBucket(bucketName);

  testWithAUser(
      v -> {
        String userId = "testremovebk" + UUID.randomUUID().toString();

        User response = RGW_ADMIN.createUser(userId);
        AmazonS3 s3 =
            createS3(
                response.getS3Credentials().get(0).getAccessKey(),
                response.getS3Credentials().get(0).getSecretKey());
        s3.createBucket(bucketName);

        ByteArrayInputStream input = new ByteArrayInputStream("Hello World!".getBytes());
        s3.putObject(bucketName, "hello.txt", input, new ObjectMetadata());

        RGW_ADMIN.removeBucket(bucketName);

        try {
          s3.headBucket(new HeadBucketRequest(bucketName));
          fail();
        } catch (Exception e) {
          assertTrue("Not Found".equals(((AmazonS3Exception) e).getErrorMessage()));
        }
      });
}
 
Example 28
Project: S3Decorators   File: FailsafeS3DecoratorTest.java   View source code 5 votes vote down vote up
@Test
public void itDoesntCount404AsFailure() throws InterruptedException {
  AmazonS3 s3 = FailsafeS3Decorator.decorate(new MissingS3Client());

  for (int i = 0; i < 100; i++ ) {
    try {
      s3.getObjectMetadata("test-bucket", "test-key");
    } catch (AmazonServiceException e) {
      assertThat(e.getStatusCode()).isEqualTo(404);
    }

    Thread.sleep(50);
  }
}
 
Example 29
Project: radosgw-admin4j   File: RgwAdminImplTest.java   View source code 5 votes vote down vote up
@Test
public void listBucket() throws Exception {
  testWithASubUser(
      v -> {
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        for (int i = 0; i < 3; i++) {
          s3.createBucket(UUID.randomUUID().toString().toLowerCase());
        }
        List<String> response = RGW_ADMIN.listBucket(v.getUserId());
        assertEquals(3, response.size());
      });
}
 
Example 30
Project: radosgw-admin4j   File: RgwAdminImplTest.java   View source code 5 votes vote down vote up
@Test
public void userQuotaMaxObjects() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        Quota quota;

        // max object = 2
        RGW_ADMIN.setUserQuota(userId, 2, -1);
        quota = RGW_ADMIN.getUserQuota(userId).get();
        assertEquals(true, quota.getEnabled());

        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();
        s3.createBucket(bucketName);

        // allow 1st,2ed obj
        s3.putObject(bucketName, userId + "1", "qqqq");
        s3.putObject(bucketName, userId + "2", "qqqq");

        // deny 3rd obj
        try {
          s3.putObject(bucketName, userId + "3", "qqqq");
          fail();
        } catch (AmazonS3Exception e) {
          assertEquals("QuotaExceeded", e.getErrorCode());
        }
      });
}
 
Example 31
Project: radosgw-admin4j   File: RgwAdminImplTest.java   View source code 5 votes vote down vote up
@Test
public void userQuotaMaxSize() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        Quota quota;

        // max size = 6 bytes
        RGW_ADMIN.setUserQuota(userId, -1, 12);
        quota = RGW_ADMIN.getUserQuota(userId).get();
        assertEquals(true, quota.getEnabled());

        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();
        s3.createBucket(bucketName);

        // ok, ok, ok, since the total to used size exceed 12KiB
        s3.putObject(bucketName, userId + "1", createString(4096));
        s3.putObject(bucketName, userId + "2", createString(4096));
        s3.putObject(bucketName, userId + "3", createString(4096));

        // not ok, since the total to used size exceed 12KiB +1
        try {
          s3.putObject(bucketName, userId + "4", createString(1));
          fail();
        } catch (AmazonS3Exception e) {
          assertEquals("QuotaExceeded", e.getErrorCode());
        }
      });
}
 
Example 32
Project: radosgw-admin4j   File: RgwAdminImplTest.java   View source code 5 votes vote down vote up
@Test
public void removeObject() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();
        s3.createBucket(bucketName);
        String objectKey = userId.toLowerCase();
        s3.putObject(bucketName, objectKey, "qqq");

        // basic
        RGW_ADMIN.removeObject(bucketName, objectKey);
        try {
          s3.getObjectMetadata(bucketName, objectKey);
          fail();
        } catch (AmazonS3Exception e) {
          assertEquals(404, e.getStatusCode());
        }

        // not exist
        RGW_ADMIN.removeObject(bucketName, objectKey);
      });
}
 
Example 33
Project: radosgw-admin4j   File: BaseTest.java   View source code 5 votes vote down vote up
protected static void createSomeObjects(AmazonS3 s3) {
  String bucketName = "bucket-" + UUID.randomUUID().toString().toLowerCase();
  s3.createBucket(bucketName);
  for (int i = 0; i < 3; i++) {

    s3.putObject(bucketName, "OBJECT-" + UUID.randomUUID(), createString(4096));
  }
  // Usage data are generated in the async way, hope it will be available after wait.
  try {
    Thread.sleep(5000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}
 
Example 34
Project: S3Decorators   File: HystrixS3DecoratorTest.java   View source code 5 votes vote down vote up
@Test
public void itDoesntCount404AsFailure() throws InterruptedException {
  AmazonS3 s3 = HystrixS3Decorator.decorate(new MissingS3Client());

  for (int i = 0; i < 100; i++ ) {
    try {
      s3.getObjectMetadata("test-bucket", "test-key");
    } catch (AmazonServiceException e) {
      assertThat(e.getStatusCode()).isEqualTo(404);
    }

    Thread.sleep(50);
  }
}
 
Example 35
Project: radosgw-admin4j   File: BaseTest.java   View source code 5 votes vote down vote up
protected static AmazonS3 createS3(String accessKey, String secretKey) {
  AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
  ClientConfiguration clientConfig = new ClientConfiguration();
  clientConfig.setProtocol(Protocol.HTTP);
  clientConfig.withSignerOverride("S3SignerType");
  //noinspection deprecation
  AmazonS3 s3 = new AmazonS3Client(credentials, clientConfig);
  s3.setEndpoint(s3Endpoint);
  return s3;
}
 
Example 36
Project: s3-channels   File: S3WritableObjectChannel.java   View source code 5 votes vote down vote up
protected S3WritableObjectChannel(String key, String bucket, String uploadId, int partSize, AmazonS3 s3,
                                  ExecutorService executor, boolean closeExecutorOnClose, int failedPartUploadRetries) {
    this.key = key;
    this.bucket = bucket;
    this.uploadId = uploadId;
    this.partSize = partSize;
    this.s3 = s3;
    this.executor = executor;
    this.closeExecutorOnClose = closeExecutorOnClose;
    this.failedPartUploadRetries = failedPartUploadRetries;
}
 
Example 37
Project: xm-ms-entity   File: AmazonS3Template.java   View source code 5 votes vote down vote up
/**
 * Gets an Amazon S3 client from basic session credentials.
 *
 * @return an authenticated Amazon S3 amazonS3
 */
public AmazonS3 getAmazonS3Client() {
    if (amazonS3 == null) {
        amazonS3 = AmazonS3ClientBuilder.standard()
            .withEndpointConfiguration(new EndpointConfiguration(endpoint, region))
            .withClientConfiguration(new ClientConfiguration().withProtocol(Protocol.HTTP))
            .withCredentials(
                new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKeyId, accessKeySecret)))
            .build();
    }
    return amazonS3;
}
 
Example 38
Project: circus-train   File: JceksAmazonS3ClientFactory.java   View source code 5 votes vote down vote up
@Override
public AmazonS3 newInstance(AmazonS3URI uri, S3S3CopierOptions s3s3CopierOptions) {
  LOG.debug("trying to get a client for uri '{}'", uri);
  AmazonS3 globalClient = newGlobalInstance(s3s3CopierOptions);
  try {
    String bucketRegion = regionForUri(globalClient, uri);
    LOG.debug("Bucket region: {}", bucketRegion);
    return newInstance(bucketRegion, s3s3CopierOptions);
  } catch (IllegalArgumentException e) {
    LOG.warn("Using global (non region specific) client", e);
    return globalClient;
  }
}
 
Example 39
Project: github-bucket   File: RepositoryS3.java   View source code 5 votes vote down vote up
public RepositoryS3(Bucket bucket, Repository repository, AmazonS3 s3, Branch branch) {
    this.s3 = s3;
    this.bucket = bucket;
    this.repository = repository;
    this.branch = branch;
    this.uri = new URIish().setScheme("amazon-s3").setHost(bucket.getName()).setPath(Constants.DOT_GIT);
}
 
Example 40
Project: circus-train   File: JceksAmazonS3ClientFactory.java   View source code 5 votes vote down vote up
private AmazonS3 newInstance(String region, S3S3CopierOptions s3s3CopierOptions) {
  HadoopAWSCredentialProviderChain credentialsChain = getCredentialsProviderChain();
  AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard().withCredentials(credentialsChain);
  URI s3Endpoint = s3s3CopierOptions.getS3Endpoint(region);
  if (s3Endpoint != null) {
    EndpointConfiguration endpointConfiguration = new EndpointConfiguration(s3Endpoint.toString(), region);
    builder.withEndpointConfiguration(endpointConfiguration);
  } else {
    builder.withRegion(region);
  }
  return builder.build();
}