Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary

The following examples show how to use com.amazonaws.services.s3.model.S3ObjectSummary. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: herd   Source File: S3DaoTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTagObjectsAmazonServiceException()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects with a mocked S3 bucket name that would trigger an AWS exception.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    try
    {
        s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Failed to tag S3 object with \"%s\" key and \"null\" version id in \"%s\" bucket. " +
                "Reason: InternalError (Service: null; Status Code: 0; Error Code: InternalError; Request ID: null)", TARGET_S3_KEY,
            MockS3OperationsImpl.MOCK_S3_BUCKET_NAME_INTERNAL_ERROR), e.getMessage());
    }
}
 
Example 2
Source Project: s3proxy   Source File: AwsSdkTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnicodeObject() throws Exception {
    String blobName = "ŪņЇЌœđЗ/☺ unicode € rocks ™";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, blobName, BYTE_SOURCE.openStream(),
            metadata);

    metadata = client.getObjectMetadata(containerName, blobName);
    assertThat(metadata).isNotNull();

    ObjectListing listing = client.listObjects(containerName);
    List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    assertThat(summaries).hasSize(1);
    S3ObjectSummary summary = summaries.iterator().next();
    assertThat(summary.getKey()).isEqualTo(blobName);
}
 
Example 3
Source Project: herd   Source File: StorageFileHelperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testValidateRegisteredS3FilesUnexpectedNonEmptyS3FileFound() throws IOException
{
    // Create two lists of expected and actual storage files, with an actual file not being added to the list of expected files.
    List<StorageFile> testExpectedFiles = new ArrayList<>();
    List<S3ObjectSummary> testActualFiles = Collections.singletonList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB));

    // Create a business object data key.
    BusinessObjectDataKey businessObjectDataKey =
        new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES,
            DATA_VERSION);

    // Try to validate S3 files when unexpected non-empty S3 file exists.
    // The validation is expected to fail when detecting an unregistered non-empty S3 file.
    try
    {
        storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey);
        fail();
    }
    catch (IllegalStateException e)
    {
        assertEquals(String.format("Found unregistered non-empty S3 file \"%s\" in \"%s\" storage. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME,
            businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage());
    }
}
 
Example 4
Source Project: nifi   Source File: ListS3.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
Example 5
Source Project: crate   Source File: MockAmazonS3.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ObjectListing listObjects(final ListObjectsRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final ObjectListing listing = new ObjectListing();
    listing.setBucketName(request.getBucketName());
    listing.setPrefix(request.getPrefix());

    for (Map.Entry<String, byte[]> blob : blobs.entrySet()) {
        if (Strings.isEmpty(request.getPrefix()) || blob.getKey().startsWith(request.getPrefix())) {
            S3ObjectSummary summary = new S3ObjectSummary();
            summary.setBucketName(request.getBucketName());
            summary.setKey(blob.getKey());
            summary.setSize(blob.getValue().length);
            listing.getObjectSummaries().add(summary);
        }
    }
    return listing;
}
 
Example 6
Source Project: aws-ant-tasks   Source File: AWSTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void emptyAndDeleteBucket(AmazonS3Client client,
        String bucketName) {
    ObjectListing objectListing = client.listObjects(bucketName);

    while (true) {
        for (Iterator<?> iterator = objectListing.getObjectSummaries()
                .iterator(); iterator.hasNext();) {
            S3ObjectSummary objectSummary = (S3ObjectSummary) iterator
                    .next();
            client.deleteObject(bucketName, objectSummary.getKey());
        }

        if (objectListing.isTruncated()) {
            objectListing = client.listNextBatchOfObjects(objectListing);
        } else {
            break;
        }
    }
    client.deleteBucket(bucketName);
}
 
Example 7
Source Project: cantor   Source File: ObjectsOnS3.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private Collection<String> getKeys(final String bucket, final int start, final int count) {
    final Set<String> keys = new HashSet<>();
    int index = 0;
    ObjectListing listing = this.s3Client.listObjects(bucket);
    do {
        for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
            if (index < start) {
                logger.debug("skipping {} at index={} start={}", summary.getKey(), index++, start);
                continue;
            }
            keys.add(summary.getKey());

            if (keys.size() == count) {
                logger.debug("retrieved {}/{} keys, returning early", keys.size(), count);
                return keys;
            }
        }

        logger.debug("got {} keys from {}", listing.getObjectSummaries().size(), listing);
        listing = this.s3Client.listNextBatchOfObjects(listing);
    } while (listing.isTruncated());

    return keys;
}
 
Example 8
Source Project: localization_nifi   Source File: ListS3.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public VersionListing listVersions() {
    VersionListing versionListing = new VersionListing();
    this.objectListing = client.listObjects(listObjectsRequest);
    for(S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        S3VersionSummary versionSummary = new S3VersionSummary();
        versionSummary.setBucketName(objectSummary.getBucketName());
        versionSummary.setETag(objectSummary.getETag());
        versionSummary.setKey(objectSummary.getKey());
        versionSummary.setLastModified(objectSummary.getLastModified());
        versionSummary.setOwner(objectSummary.getOwner());
        versionSummary.setSize(objectSummary.getSize());
        versionSummary.setStorageClass(objectSummary.getStorageClass());
        versionSummary.setIsLatest(true);

        versionListing.getVersionSummaries().add(versionSummary);
    }

    return versionListing;
}
 
Example 9
Source Project: herd   Source File: S3DaoTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Put a file in S3.
    s3Operations.putObject(new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata()), null);

    // Tag the file with an S3 object tag.
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tag);

    // Validate that the object got tagged.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Collections.singletonList(tag), getObjectTaggingResult.getTagSet());
}
 
Example 10
Source Project: jobcacher-plugin   Source File: S3Profile.java    License: MIT License 6 votes vote down vote up
public void delete(String bucketName, String pathPrefix) {
    ObjectListing listing = null;
    do {
        listing = listing == null ? helper.client().listObjects(bucketName, pathPrefix) : helper.client().listNextBatchOfObjects(listing);

        DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName);

        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        }
        req.withKeys(keys);

        helper.client().deleteObjects(req);
    } while (listing.isTruncated());
}
 
Example 11
Source Project: herd   Source File: S3ServiceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Call the method under test.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(s3Dao).tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);
    verifyNoMoreInteractions(s3Dao);
}
 
Example 12
Source Project: pipeline-aws-plugin   Source File: S3FindFilesStep.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This creates a new FileWrapper instance based on the S3ObjectSummary information.
 *
 * @param pathComponentCount The root path component count.
 * @param javaPath           The Path instance for the file.
 * @param entry              The S3ObjectSummary for the file.
 * @return A new FileWrapper instance.
 */
public static FileWrapper createFileWrapperFromFile(int pathComponentCount, Path javaPath, S3ObjectSummary entry) {
	Path pathFileName = javaPath.getFileName();
	if (pathFileName == null) {
		return null;
	}
	return new FileWrapper(
			// Name:
			convertPathToAwsFormat(pathFileName),
			// Path (relative to the `path` parameter):
			convertPathToAwsFormat(javaPath.subpath(pathComponentCount, javaPath.getNameCount())),
			// Directory?
			false,
			// Size:
			entry.getSize(),
			// Last modified (milliseconds):
			entry.getLastModified().getTime()
	);
}
 
Example 13
Source Project: ignite   Source File: S3CheckpointSpiSelfTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @throws Exception If error.
 */
@Override protected void afterSpiStopped() throws Exception {
    AWSCredentials cred = new BasicAWSCredentials(IgniteS3TestSuite.getAccessKey(),
        IgniteS3TestSuite.getSecretKey());

    AmazonS3 s3 = new AmazonS3Client(cred);

    String bucketName = S3CheckpointSpi.BUCKET_NAME_PREFIX + "unit-test-bucket";

    try {
        ObjectListing list = s3.listObjects(bucketName);

        while (true) {
            for (S3ObjectSummary sum : list.getObjectSummaries())
                s3.deleteObject(bucketName, sum.getKey());

            if (list.isTruncated())
                list = s3.listNextBatchOfObjects(list);
            else
                break;
        }
    }
    catch (AmazonClientException e) {
        throw new IgniteSpiException("Failed to read checkpoint bucket: " + bucketName, e);
    }
}
 
Example 14
Source Project: herd   Source File: S3ServiceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testListDirectoryIgnoreZeroByteDirectoryMarkers()
{
    // Create an S3 file transfer request parameters DTO.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create a list of S3 object summaries.
    List<S3ObjectSummary> s3ObjectSummaries = Collections.singletonList(new S3ObjectSummary());

    // Mock the external calls.
    when(s3Dao.listDirectory(s3FileTransferRequestParamsDto, true)).thenReturn(s3ObjectSummaries);

    // Call the method under test.
    List<S3ObjectSummary> result = s3Service.listDirectory(s3FileTransferRequestParamsDto, true);

    // Verify the external calls.
    verify(s3Dao).listDirectory(s3FileTransferRequestParamsDto, true);
    verifyNoMoreInteractions(s3Dao);

    // Validate the returned object.
    assertEquals(s3ObjectSummaries, result);
}
 
Example 15
Source Project: incubator-gobblin   Source File: AWSSdkClient.java    License: Apache License 2.0 6 votes vote down vote up
/***
 * Download a S3 object to local directory
 *
 * @param s3ObjectSummary S3 object summary for the object to download
 * @param targetDirectory Local target directory to download the object to
 * @throws IOException If any errors were encountered in downloading the object
 */
public void downloadS3Object(S3ObjectSummary s3ObjectSummary,
    String targetDirectory)
    throws IOException {

  final AmazonS3 amazonS3 = getS3Client();

  final GetObjectRequest getObjectRequest = new GetObjectRequest(
      s3ObjectSummary.getBucketName(),
      s3ObjectSummary.getKey());

  final S3Object s3Object = amazonS3.getObject(getObjectRequest);

  final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey();
  FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile));

  LOGGER.info("S3 object downloaded to file: " + targetFile);
}
 
Example 16
Source Project: emodb   Source File: StashReaderTest.java    License: Apache License 2.0 6 votes vote down vote up
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 
Example 17
Source Project: hawkbit-extensions   Source File: S3Repository.java    License: Eclipse Public License 1.0 6 votes vote down vote up
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting S3 object folder (tenant) from bucket {} and key {}", s3Properties.getBucketName(), folder);

    // Delete artifacts
    ObjectListing objects = amazonS3.listObjects(s3Properties.getBucketName(), folder + "/");
    do {
        for (final S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
            amazonS3.deleteObject(s3Properties.getBucketName(), objectSummary.getKey());
        }
        objects = amazonS3.listNextBatchOfObjects(objects);
    } while (objects.isTruncated());

}
 
Example 18
/**
 * Returns the size of the dataset, in bytes. Will be zero if this dataset is a collection or non-existent.
 *
 * @return the size of the dataset
 */
@Override
public long length() {
  // If the summary is already in the cache, return it.
  // It'll have been added by a listDatasets() call on the parent directory.
  S3ObjectSummary objectSummary = objectSummaryCache.getIfPresent(s3uri);
  if (objectSummary != null) {
    return objectSummary.getSize();
  }

  /*
   * Get the metadata directly from S3. This will be expensive.
   * We get punished hard if length() and/or lastModified() is called on a bunch of datasets without
   * listDatasets() first being called on their parent directory.
   *
   * So, is the right thing to do here "getParentDataset().listDatasets()" and then query the cache again?
   * Perhaps, but listDatasets() throws an IOException, and length() and lastModified() do not.
   * We would have to change their signatures and the upstream client code to make it work.
   */
  ObjectMetadata metadata = threddsS3Client.getObjectMetadata(s3uri);
  if (metadata != null) {
    return metadata.getContentLength();
  } else {
    // "this" may be a collection or non-existent. In both cases, we return 0.
    return 0;
  }
}
 
Example 19
Source Project: datacollector   Source File: S3FileRef.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public S3FileRef(
    AmazonS3 s3Client,
    S3ObjectSummary s3ObjectSummary,
    boolean useSSE,
    CredentialValue customerKey,
    CredentialValue customerKeyMd5,
    int bufferSize,
    boolean createMetrics,
    long totalSizeInBytes,
    double rateLimit,
    boolean verifyChecksum,
    String checksum,
    HashingUtil.HashType checksumAlgorithm) {
  super(
      ImmutableSet.<Class<? extends AutoCloseable>>of(InputStream.class),
      bufferSize,
      createMetrics,
      totalSizeInBytes,
      rateLimit,
      verifyChecksum,
      checksum,
      checksumAlgorithm
  );
  this.s3Client = s3Client;
  this.s3ObjectSummary = s3ObjectSummary;
  this.useSSE = useSSE;
  this.customerKey = customerKey;
  this.customerKeyMd5 = customerKeyMd5;
}
 
Example 20
Source Project: s3proxy   Source File: AwsSdkTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBlobListRecursive() throws Exception {
    ObjectListing listing = client.listObjects(containerName);
    assertThat(listing.getObjectSummaries()).isEmpty();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "prefix/blob1",
            BYTE_SOURCE.openStream(), metadata);
    client.putObject(containerName, "prefix/blob2",
            BYTE_SOURCE.openStream(), metadata);

    ImmutableList.Builder<String> builder = ImmutableList.builder();
    listing = client.listObjects(new ListObjectsRequest()
            .withBucketName(containerName)
            .withDelimiter("/"));
    assertThat(listing.getObjectSummaries()).isEmpty();
    for (String prefix : listing.getCommonPrefixes()) {
        builder.add(prefix);
    }
    assertThat(builder.build()).containsOnly("prefix/");

    builder = ImmutableList.builder();
    listing = client.listObjects(containerName);
    for (S3ObjectSummary summary : listing.getObjectSummaries()) {
        builder.add(summary.getKey());
    }
    assertThat(builder.build()).containsOnly("prefix/blob1",
            "prefix/blob2");
    assertThat(listing.getCommonPrefixes()).isEmpty();
}
 
Example 21
Source Project: presto   Source File: S3TableConfigClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map.
 */
private void updateTablesFromS3()
{
    long now = System.currentTimeMillis();

    AmazonS3Client s3client = clientManager.getS3Client();

    for (S3ObjectSummary summary : getObjectSummaries()) {
        if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) {
            // New or updated file, so we must read from AWS
            if (summary.getKey().endsWith("/")) {
                continue;
            }

            log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey());
            S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey()));

            try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) {
                KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader));
                descriptors.put(summary.getKey(), table);
                log.info("Put table description into the map from %s", summary.getKey());
            }
            catch (IOException iox) {
                log.error("Problem reading input stream from object.", iox);
                throwIfUnchecked(iox);
                throw new RuntimeException(iox);
            }
        }
    }

    log.info("Completed updating table definitions from S3.");
    lastCheck = now;
}
 
Example 22
Source Project: athenz   Source File: S3ChangeLogStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAllSignedDomainsException() throws FileNotFoundException {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null, 1);

    InputStream is1 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is1 = new MockS3ObjectInputStream(is1, null);

    InputStream is2 = new FileInputStream("src/test/resources/iaas.json");
    MockS3ObjectInputStream s3Is2 = new MockS3ObjectInputStream(is2, null);

    S3Object object = mock(S3Object.class);
    when(object.getObjectContent()).thenReturn(s3Is1).thenReturn(s3Is2);

    when(store.awsS3Client.getObject("s3-unit-test-bucket-name", "iaas")).thenReturn(object);
    ObjectListing mockObjectListing = mock(ObjectListing.class);
    when(store.awsS3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(mockObjectListing);
    List<S3ObjectSummary> tempList = new ArrayList<>();
    S3ObjectSummary s3ObjectSummary = mock(S3ObjectSummary.class);
    when(s3ObjectSummary.getKey()).thenReturn("iaas");
    tempList.add(s3ObjectSummary);
    when(mockObjectListing.getObjectSummaries()).thenReturn(tempList);


    List<String> temp = new LinkedList<>();
    temp.add("iaas");

    try {
        when(store.executorService.awaitTermination(defaultTimeoutSeconds, TimeUnit.SECONDS)).thenThrow(new InterruptedException());
        assertFalse(store.getAllSignedDomains(temp));
        assertTrue(store.getLocalDomainList().size() > 0);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}
 
Example 23
Source Project: tutorials   Source File: SpringCloudS3LiveTest.java    License: MIT License 5 votes vote down vote up
@AfterClass
public static void cleanUpResources() {
    AmazonS3 amazonS3 = SpringCloudAwsTestUtil.amazonS3();
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(bucketName);
    for (S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) {
        amazonS3.deleteObject(bucketName, objectSummary.getKey());
    }
    amazonS3.deleteBucket(bucketName);

    new File(testFileToDownload).delete();
    new File(testFileToUpload).delete();
    similarNameFiles.forEach(File::delete);
}
 
Example 24
Source Project: terrapin   Source File: TerrapinUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
  AmazonS3Client s3Client = mock(AmazonS3Client.class);
  ObjectListing objectListing = mock(ObjectListing.class);
  S3ObjectSummary summary1 = new S3ObjectSummary();
  S3ObjectSummary summary2 = new S3ObjectSummary();
  S3ObjectSummary summary3 = new S3ObjectSummary();
  summary1.setKey("/abc/123");
  summary2.setKey("/abc/456");
  summary3.setKey("/def/123");
  summary1.setSize(32432);
  summary2.setSize(213423);
  summary3.setSize(2334);
  List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
  whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
  when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
  when(objectListing.getObjectSummaries()).thenReturn(summaries);

  List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
      "bucket", "/abc");

  assertEquals(2, results.size());
  assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
  assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
  assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
  assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
 
Example 25
Source Project: crate   Source File: S3FileInput.java    License: Apache License 2.0 5 votes vote down vote up
private void addKeyUris(List<URI> uris, ObjectListing list, URI uri, Predicate<URI> uriPredicate) {
    List<S3ObjectSummary> summaries = list.getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        String key = summary.getKey();
        if (!key.endsWith("/")) {
            URI keyUri = uri.resolve("/" + key);
            if (uriPredicate.test(keyUri)) {
                uris.add(keyUri);
                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug("{}", keyUri);
                }
            }
        }
    }
}
 
Example 26
Source Project: singleton   Source File: S3ProductDaoImpl.java    License: Eclipse Public License 2.0 5 votes vote down vote up
/**
 * get locale list from S3 server
 */
@Override
public List<String> getLocaleList(String productName, String version) throws DataException {
   List<String> localeList = new ArrayList<String>();
   String filePathPrefix = S3Utils.genProductVersionS3Path(productName, version);
   ListObjectsV2Result result =
         s3Client.getS3Client().listObjectsV2(config.getBucketName(), S3Utils.S3_L10N_BUNDLES_PATH);
   if (result == null) {
      throw new DataException("Can't find S3 resource from " + productName + "\\" + version);
   }
   List<S3ObjectSummary> objects = result.getObjectSummaries();
   if (objects == null || objects.size() < 1) {
      throw new DataException("S3 Component list is empty.");
   }
   for (S3ObjectSummary s3os : objects) {
      String s3obKey = s3os.getKey().replace(filePathPrefix, "");
      if((!s3obKey.startsWith(ConstantsFile.CREATION_INFO)) && (!s3obKey.startsWith(ConstantsFile.VERSION_FILE))) {
         String resultKey =s3obKey.split(ConstantsChar.BACKSLASH)[1];
         String localeKey = S3Utils.getLocaleByFileName(resultKey);
         if (localeKey != null && !localeList.contains(localeKey)) {
            localeList.add(localeKey);
         }
         
      }
      
      
   }
   return localeList;
}
 
Example 27
Source Project: herd   Source File: UploaderController.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Logs all files found in the specified S3 location.
 *
 * @param params the S3 file transfer request parameters
 */
private void logS3KeyPrefixContents(S3FileTransferRequestParamsDto params)
{
    List<S3ObjectSummary> s3ObjectSummaries = s3Service.listDirectory(params);
    LOGGER.info(
        String.format("Found %d keys with prefix \"%s\" in bucket \"%s\":", s3ObjectSummaries.size(), params.getS3KeyPrefix(), params.getS3BucketName()));

    for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries)
    {
        LOGGER.info(String.format("    s3://%s/%s", params.getS3BucketName(), s3ObjectSummary.getKey()));
    }
}
 
Example 28
Source Project: iaf   Source File: AmazonS3FileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean folderExists(String folder) throws FileSystemException {
	ObjectListing objectListing = s3Client.listObjects(bucketName);
	Iterator<S3ObjectSummary> objIter = objectListing.getObjectSummaries().iterator();
	while (objIter.hasNext()) {
		S3ObjectSummary s3ObjectSummary = objIter.next();
		String key = s3ObjectSummary.getKey();
		if(key.endsWith("/") && key.equals(folder+"/")){
			return true;
		}
	}
	return false;
}
 
Example 29
Source Project: datacollector   Source File: TestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static void createBucket(AmazonS3 s3client, String bucketName) {
  if(s3client.doesBucketExist(bucketName)) {
    for(S3ObjectSummary s : S3Objects.inBucket(s3client, bucketName)) {
      s3client.deleteObject(bucketName, s.getKey());
    }
    s3client.deleteBucket(bucketName);
  }
  Assert.assertFalse(s3client.doesBucketExist(bucketName));
  // Note that CreateBucketRequest does not specify region. So bucket is
  // bucketName
  s3client.createBucket(new CreateBucketRequest(bucketName));
}
 
Example 30
Source Project: Scribengin   Source File: S3Folder.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
public List<String> getChildrenNames() {
  List<String> holder = new ArrayList<String>() ;
  for(S3ObjectSummary sel : getChildren()) {
    String name = sel.getKey().substring(folderPath.length() + 1);
    holder.add(name);
  }
  return holder;
}