com.amazonaws.services.s3.model.ListObjectsV2Result Java Examples

private void clearDest(AWSCredentials credentials, RedshiftConnection.UnloadConfig unloadConfig)
{
    try {
        RetryExecutor.retryExecutor() .run(() -> {
            AmazonS3Client s3Client = new AmazonS3Client(credentials);
            ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(unloadConfig.s3Bucket).withPrefix(unloadConfig.s3Prefix);
            ListObjectsV2Result result;
            // This operation shouldn't be skipped since remaining files created by other operation can cause duplicated data
            do {
                result = s3Client.listObjectsV2(req);

                for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
                    s3Client.deleteObject(unloadConfig.s3Bucket, objectSummary.getKey());
                }
                req.setContinuationToken(result.getNextContinuationToken());
            } while (result.isTruncated());
        });
    }
    catch (RetryExecutor.RetryGiveupException e) {
        throw Throwables.propagate(e);
    }
}
 

/**
 * Calls DescribeDBInstances on the AWS RDS Client returning all DB Instances that match the supplied predicate and attempting
 * to push down certain predicates (namely queries for specific DB Instance) to EC2.
 *
 * @See TableProvider
 */
@Override
public void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker)
{
    ValueSet bucketConstraint = recordsRequest.getConstraints().getSummary().get("bucket_name");
    String bucket;
    if (bucketConstraint != null && bucketConstraint.isSingleValue()) {
        bucket = bucketConstraint.getSingleValue().toString();
    }
    else {
        throw new IllegalArgumentException("Queries against the objects table must filter on a single bucket " +
                "(e.g. where bucket_name='my_bucket'.");
    }

    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(bucket).withMaxKeys(MAX_KEYS);
    ListObjectsV2Result result;
    do {
        result = amazonS3.listObjectsV2(req);
        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            toRow(objectSummary, spiller);
        }
        req.setContinuationToken(result.getNextContinuationToken());
    }
    while (result.isTruncated() && queryStatusChecker.isQueryRunning());
}
 

protected Collection<String> getSiteListFromBucketKeys(String bucketName, String rootPrefix) {
    List<String> siteNames = new ArrayList<>();
    AmazonS3 client = clientBuilder.getClient();

    ListObjectsV2Request request = new ListObjectsV2Request()
            .withBucketName(bucketName)
            .withPrefix(rootPrefix)
            .withDelimiter(DELIMITER);

    ListObjectsV2Result result = client.listObjectsV2(request);
    if(CollectionUtils.isNotEmpty(result.getCommonPrefixes())) {
        result.getCommonPrefixes()
              .stream()
              .map(prefix -> StringUtils.stripEnd(StringUtils.removeStart(prefix, rootPrefix), DELIMITER))
              .forEach(siteNames::add);
    }

    return siteNames;
}
 

/**
 * get the compose list from S3 server
 */
@Override
public List<String> getComponentList(String productName, String version) throws DataException {
   List<String> componentList = new ArrayList<String>();
   String filePathPrefix = S3Utils.genProductVersionS3Path(productName, version);
   ListObjectsV2Result result =
         s3Client.getS3Client().listObjectsV2(config.getBucketName(), filePathPrefix);
   if (result == null) {
      throw new DataException("Can't find S3 resource from " + productName + "\\" + version);
   }
   List<S3ObjectSummary> objects = result.getObjectSummaries();
   if (objects == null || objects.size() < 1) {
      throw new DataException("S3 Component list is empty.");
   }
   for (S3ObjectSummary s3os : objects) {
      String resultKey =
            (s3os.getKey().replace(filePathPrefix, "")).split(ConstantsChar.BACKSLASH)[0];
      if (!componentList.contains(resultKey) && (!resultKey.endsWith(ConstantsFile.FILE_TPYE_JSON))) {
         componentList.add(resultKey);
      }
   }
   return componentList;
}
 

/**
  * get one product's all available versions
  */
@Override
public List<String> getVersionList(String productName) throws DataException {
    String basePath = S3Utils.S3_L10N_BUNDLES_PATH+productName +  ConstantsChar.BACKSLASH;
    ListObjectsV2Result versionListResult = s3Client.getS3Client().listObjectsV2(config.getBucketName(),basePath);
    
    if(versionListResult != null) {
        List<S3ObjectSummary> versionListSummary = versionListResult.getObjectSummaries();
        Set<String> versionset = new HashSet<>();
        for (S3ObjectSummary s3productName : versionListSummary) {
            versionset.add(s3productName.getKey().replace(basePath, "").split(ConstantsChar.BACKSLASH)[0]);
          }
        List<String> result = new ArrayList<>();
        for(String version: versionset) {
            result.add(version);
        }
      return result;
      
    }else {
     throw new DataException(productName + " no available version in s3");
    }
}
 

public static void main(String[] args) {
    final String USAGE = "\n" +
            "To run this example, supply the name of a bucket to list!\n" +
            "\n" +
            "Ex: ListObjects <bucket-name>\n";

    if (args.length < 1) {
        System.out.println(USAGE);
        System.exit(1);
    }

    String bucket_name = args[0];

    System.out.format("Objects in S3 bucket %s:\n", bucket_name);
    final AmazonS3 s3 = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    ListObjectsV2Result result = s3.listObjectsV2(bucket_name);
    List<S3ObjectSummary> objects = result.getObjectSummaries();
    for (S3ObjectSummary os : objects) {
        System.out.println("* " + os.getKey());
    }
}
 

@Override
protected void setUpRead()
{
    AtomicLong count = new AtomicLong(0);
    when(mockS3.listObjectsV2(any(ListObjectsV2Request.class))).thenAnswer((InvocationOnMock invocation) -> {
        ListObjectsV2Request request = (ListObjectsV2Request) invocation.getArguments()[0];
        assertEquals(getIdValue(), request.getBucketName());

        ListObjectsV2Result mockResult = mock(ListObjectsV2Result.class);
        List<S3ObjectSummary> values = new ArrayList<>();
        values.add(makeObjectSummary(getIdValue()));
        values.add(makeObjectSummary(getIdValue()));
        values.add(makeObjectSummary("fake-id"));
        when(mockResult.getObjectSummaries()).thenReturn(values);

        if (count.get() > 0) {
            assertNotNull(request.getContinuationToken());
        }

        if (count.incrementAndGet() < 2) {
            when(mockResult.isTruncated()).thenReturn(true);
            when(mockResult.getNextContinuationToken()).thenReturn("token");
        }

        return mockResult;
    });
}
 

@AfterClass
public static void cleanUpResources() {
    AmazonS3 amazonS3 = SpringCloudAwsTestUtil.amazonS3();
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(bucketName);
    for (S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) {
        amazonS3.deleteObject(bucketName, objectSummary.getKey());
    }
    amazonS3.deleteBucket(bucketName);

    new File(testFileToDownload).delete();
    new File(testFileToUpload).delete();
    similarNameFiles.forEach(File::delete);
}
 

public void deleteBucket(String bucketName) {
    logger.trace("Deleting S3 objects under {} bucket...", bucketName);
    ListObjectsV2Result listObjectsV2Result = amazonS3.listObjectsV2(bucketName);
    for (S3ObjectSummary objectSummary : listObjectsV2Result.getObjectSummaries()) {
        logger.info("Deleting S3 object: {}", objectSummary.getKey());
        amazonS3.deleteObject(bucketName, objectSummary.getKey());
    }
    logger.info("Deleting S3 bucket: {}", bucketName);
    amazonS3.deleteBucket(bucketName);
}
 

@Override
public List<FileInfo> listDirectoriesInDirectory(Path dirPath) {
    String prefix = dirPath.toString();
    if (!prefix.isEmpty()) {
        prefix += File.separator;
    }

    ListObjectsV2Result result = s3.listObjectsV2(bucketName, prefix);

    Set<String> seenDirectoryNames = Sets.newHashSet();
    ImmutableList.Builder<FileInfo> fileInfos = ImmutableList.builder();
    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
        String key = objectSummary.getKey().substring(prefix.length());
        if (key.endsWith(File.separator) || !key.contains(File.separator)) {
            continue;
        }
        key = key.substring(0, key.lastIndexOf(File.separator));
        if (key.contains(File.separator) || seenDirectoryNames.contains(key)) {
            continue;
        }

        seenDirectoryNames.add(key);
        fileInfos.add(new FileInfo.Builder()
                .name(key)
                .size(objectSummary.getSize())
                .lastModifiedTime(objectSummary.getLastModified().toInstant())
                .build());
    }
    return fileInfos.build();
}
 

private Iterator<LocatedFileStatus> listPrefix(Path path)
{
    String key = keyFromPath(path);
    if (!key.isEmpty()) {
        key += PATH_SEPARATOR;
    }

    ListObjectsV2Request request = new ListObjectsV2Request()
            .withBucketName(getBucketName(uri))
            .withPrefix(key)
            .withDelimiter(PATH_SEPARATOR)
            .withRequesterPays(requesterPaysEnabled);

    STATS.newListObjectsCall();
    Iterator<ListObjectsV2Result> listings = new AbstractSequentialIterator<ListObjectsV2Result>(s3.listObjectsV2(request))
    {
        @Override
        protected ListObjectsV2Result computeNext(ListObjectsV2Result previous)
        {
            if (!previous.isTruncated()) {
                return null;
            }

            request.setContinuationToken(previous.getNextContinuationToken());

            return s3.listObjectsV2(request);
        }
    };

    return Iterators.concat(Iterators.transform(listings, this::statusFromListing));
}
 

@Override
public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request)
{
    final String continuationToken = "continue";

    ListObjectsV2Result listingV2 = new ListObjectsV2Result();

    if (continuationToken.equals(listObjectsV2Request.getContinuationToken())) {
        S3ObjectSummary standardTwo = new S3ObjectSummary();
        standardTwo.setStorageClass(StorageClass.Standard.toString());
        standardTwo.setKey("test/standardTwo");
        standardTwo.setLastModified(new Date());
        listingV2.getObjectSummaries().add(standardTwo);

        if (hasGlacierObjects) {
            S3ObjectSummary glacier = new S3ObjectSummary();
            glacier.setStorageClass(StorageClass.Glacier.toString());
            glacier.setKey("test/glacier");
            glacier.setLastModified(new Date());
            listingV2.getObjectSummaries().add(glacier);
        }
    }
    else {
        S3ObjectSummary standardOne = new S3ObjectSummary();
        standardOne.setStorageClass(StorageClass.Standard.toString());
        standardOne.setKey("test/standardOne");
        standardOne.setLastModified(new Date());
        listingV2.getObjectSummaries().add(standardOne);
        listingV2.setTruncated(true);
        listingV2.setNextContinuationToken(continuationToken);
    }

    return listingV2;
}
 

/**
 * get locale list from S3 server
 */
@Override
public List<String> getLocaleList(String productName, String version) throws DataException {
   List<String> localeList = new ArrayList<String>();
   String filePathPrefix = S3Utils.genProductVersionS3Path(productName, version);
   ListObjectsV2Result result =
         s3Client.getS3Client().listObjectsV2(config.getBucketName(), S3Utils.S3_L10N_BUNDLES_PATH);
   if (result == null) {
      throw new DataException("Can't find S3 resource from " + productName + "\\" + version);
   }
   List<S3ObjectSummary> objects = result.getObjectSummaries();
   if (objects == null || objects.size() < 1) {
      throw new DataException("S3 Component list is empty.");
   }
   for (S3ObjectSummary s3os : objects) {
      String s3obKey = s3os.getKey().replace(filePathPrefix, "");
      if((!s3obKey.startsWith(ConstantsFile.CREATION_INFO)) && (!s3obKey.startsWith(ConstantsFile.VERSION_FILE))) {
         String resultKey =s3obKey.split(ConstantsChar.BACKSLASH)[1];
         String localeKey = S3Utils.getLocaleByFileName(resultKey);
         if (localeKey != null && !localeList.contains(localeKey)) {
            localeList.add(localeKey);
         }
         
      }
      
      
   }
   return localeList;
}
 

@Override
public List<FileInfo> listFilesInDirectory(Path dirPath) {
    String prefix = dirPath.toString();
    if (!prefix.isEmpty()) {
        prefix += File.separator;
    }

    ListObjectsV2Result result = s3.listObjectsV2(bucketName, prefix);

    List<FileInfo> fileInfos = Lists.newArrayList();
    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
        String key = objectSummary.getKey().substring(prefix.length());
        if (key.contains(File.separator)) {
            continue;
        }

        fileInfos.add(new FileInfo.Builder()
                .name(key)
                .size(objectSummary.getSize())
                .lastModifiedTime(objectSummary.getLastModified().toInstant())
                .build());
    }

    Comparator<String> comparator = new NaturalFilenameComparator();
    fileInfos.sort((FileInfo f1, FileInfo f2) -> comparator.compare(f1.getName(), f2.getName()));
    return ImmutableList.copyOf(fileInfos);
}
 

@Test
public void matchGlobWithSlashes() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/foo/bar\\baz*");

  ListObjectsV2Request request =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken(null);

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary firstMatch = new S3ObjectSummary();
  firstMatch.setBucketName(path.getBucket());
  firstMatch.setKey("foo/bar\\baz0");
  firstMatch.setSize(100);
  firstMatch.setLastModified(new Date(1540000000001L));

  // Expected to not be returned; prefix matches, but substring after wildcard does not
  S3ObjectSummary secondMatch = new S3ObjectSummary();
  secondMatch.setBucketName(path.getBucket());
  secondMatch.setKey("foo/bar/baz1");
  secondMatch.setSize(200);
  secondMatch.setLastModified(new Date(1540000000002L));

  // Expected first request returns continuation token
  ListObjectsV2Result result = new ListObjectsV2Result();
  result.getObjectSummaries().add(firstMatch);
  result.getObjectSummaries().add(secondMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(request))))
      .thenReturn(result);

  // Expect object metadata queries for content encoding
  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);

  assertThat(
      s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0),
      MatchResultMatcher.create(
          ImmutableList.of(
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          firstMatch.getBucketName(), firstMatch.getKey()))
                  .setSizeBytes(firstMatch.getSize())
                  .setLastModifiedMillis(firstMatch.getLastModified().getTime())
                  .build())));
}
 

@Test
public void matchVariousInvokeThreadPool() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
  notFoundException.setStatusCode(404);
  S3ResourceId pathNotExist =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathNotExist.getBucket(), pathNotExist.getKey())))))
      .thenThrow(notFoundException);

  AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
  forbiddenException.setStatusCode(403);
  S3ResourceId pathForbidden =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathForbidden.getBucket(), pathForbidden.getKey())))))
      .thenThrow(forbiddenException);

  S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
  ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
  s3ObjectMetadata.setContentLength(100);
  s3ObjectMetadata.setLastModified(new Date(1540000000000L));
  s3ObjectMetadata.setContentEncoding("not-gzip");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey())))))
      .thenReturn(s3ObjectMetadata);

  S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");

  S3ObjectSummary foundListObject = new S3ObjectSummary();
  foundListObject.setBucketName(pathGlob.getBucket());
  foundListObject.setKey("path/part-0");
  foundListObject.setSize(200);
  foundListObject.setLastModified(new Date(1541000000000L));

  ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
  listObjectsResult.setNextContinuationToken(null);
  listObjectsResult.getObjectSummaries().add(foundListObject);
  when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class)))
      .thenReturn(listObjectsResult);

  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0")))))
      .thenReturn(metadata);

  assertThat(
      s3FileSystem.match(
          ImmutableList.of(
              pathNotExist.toString(),
              pathForbidden.toString(),
              pathExist.toString(),
              pathGlob.toString())),
      contains(
          MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()),
          MatchResultMatcher.create(
              MatchResult.Status.ERROR, new IOException(forbiddenException)),
          MatchResultMatcher.create(100, 1540000000000L, pathExist, true),
          MatchResultMatcher.create(
              200,
              1541000000000L,
              S3ResourceId.fromComponents(pathGlob.getBucket(), foundListObject.getKey()),
              true)));
}
 

@Test
public void matchGlob() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/foo/bar*baz");

  ListObjectsV2Request firstRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken(null);

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary firstMatch = new S3ObjectSummary();
  firstMatch.setBucketName(path.getBucket());
  firstMatch.setKey("foo/bar0baz");
  firstMatch.setSize(100);
  firstMatch.setLastModified(new Date(1540000000001L));

  // Expected to not be returned; prefix matches, but substring after wildcard does not
  S3ObjectSummary secondMatch = new S3ObjectSummary();
  secondMatch.setBucketName(path.getBucket());
  secondMatch.setKey("foo/bar1qux");
  secondMatch.setSize(200);
  secondMatch.setLastModified(new Date(1540000000002L));

  // Expected first request returns continuation token
  ListObjectsV2Result firstResult = new ListObjectsV2Result();
  firstResult.setNextContinuationToken("token");
  firstResult.getObjectSummaries().add(firstMatch);
  firstResult.getObjectSummaries().add(secondMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest))))
      .thenReturn(firstResult);

  // Expect second request with continuation token
  ListObjectsV2Request secondRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken("token");

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary thirdMatch = new S3ObjectSummary();
  thirdMatch.setBucketName(path.getBucket());
  thirdMatch.setKey("foo/bar2baz");
  thirdMatch.setSize(300);
  thirdMatch.setLastModified(new Date(1540000000003L));

  // Expected second request returns third prefix match and no continuation token
  ListObjectsV2Result secondResult = new ListObjectsV2Result();
  secondResult.setNextContinuationToken(null);
  secondResult.getObjectSummaries().add(thirdMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest))))
      .thenReturn(secondResult);

  // Expect object metadata queries for content encoding
  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);

  assertThat(
      s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0),
      MatchResultMatcher.create(
          ImmutableList.of(
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          firstMatch.getBucketName(), firstMatch.getKey()))
                  .setSizeBytes(firstMatch.getSize())
                  .setLastModifiedMillis(firstMatch.getLastModified().getTime())
                  .build(),
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          thirdMatch.getBucketName(), thirdMatch.getKey()))
                  .setSizeBytes(thirdMatch.getSize())
                  .setLastModifiedMillis(thirdMatch.getLastModified().getTime())
                  .build())));
}
 

private void assertS3Contents(List<Map<String, Object>> expected)
        throws IOException
{
    ListObjectsV2Request req = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3ParentKey);
    ListObjectsV2Result result;
    List<String> lines = new ArrayList<>();
    do {
        result = s3Client.listObjectsV2(req);

        for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
            if (objectSummary.getKey().endsWith("_part_00")) {
                try (BufferedReader reader =
                        new BufferedReader(
                                new InputStreamReader(
                                        s3Client.getObject(
                                                objectSummary.getBucketName(),
                                                objectSummary.getKey()).getObjectContent()))) {
                    lines.addAll(reader.lines().collect(Collectors.toList()));
                }
            }
            else {
                assertThat(objectSummary.getKey(), endsWith("_manifest"));
            }

            try {
                s3Client.deleteObject(objectSummary.getBucketName(), objectSummary.getKey());
            }
            catch (Exception e) {
                logger.warn("Failed to delete S3 object: bucket={}, key={}", s3Bucket, objectSummary.getKey(), e);
            }
        }
        req.setContinuationToken(result.getNextContinuationToken());
    } while (result.isTruncated());

    List<ImmutableMap<String, ? extends Serializable>> actual =
            lines.stream()
                    .map(
                            l -> {
                                String[] values = l.split("\\|");
                                assertThat(values.length, is(3));
                                return ImmutableMap.of(
                                        "id", Integer.valueOf(values[0]),
                                        "name", values[1],
                                        "score", Float.valueOf(values[2]));
                            }
                    )
                    .sorted((o1, o2) -> ((Integer)o1.get("id")) - ((Integer) o2.get("id")))
                    .collect(Collectors.toList());
    assertThat(actual, is(expected));
}
 

/** Unsupported Operation. */
@Override public ListObjectsV2Result listObjectsV2(String bucketName) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

/** Unsupported Operation. */
@Override public ListObjectsV2Result listObjectsV2(String bucketName,
    String prefix) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

/** Unsupported Operation. */
@Override public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Req) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 

@Test
public void testBlobListV2() throws Exception {
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    for (int i = 1; i < 5; ++i) {
        client.putObject(containerName, String.valueOf(i),
                BYTE_SOURCE.openStream(), metadata);
    }

    ListObjectsV2Result result = client.listObjectsV2(
            new ListObjectsV2Request()
            .withBucketName(containerName)
            .withMaxKeys(1)
            .withStartAfter("1"));
    assertThat(result.getContinuationToken()).isEmpty();
    assertThat(result.getStartAfter()).isEqualTo("1");
    assertThat(result.getNextContinuationToken()).isEqualTo("2");
    assertThat(result.isTruncated()).isTrue();
    assertThat(result.getObjectSummaries()).hasSize(1);
    assertThat(result.getObjectSummaries().get(0).getKey()).isEqualTo("2");

    result = client.listObjectsV2(
            new ListObjectsV2Request()
            .withBucketName(containerName)
            .withMaxKeys(1)
            .withContinuationToken(result.getNextContinuationToken()));
    assertThat(result.getContinuationToken()).isEqualTo("2");
    assertThat(result.getStartAfter()).isEmpty();
    assertThat(result.getNextContinuationToken()).isEqualTo("3");
    assertThat(result.isTruncated()).isTrue();
    assertThat(result.getObjectSummaries()).hasSize(1);
    assertThat(result.getObjectSummaries().get(0).getKey()).isEqualTo("3");

    result = client.listObjectsV2(
            new ListObjectsV2Request()
            .withBucketName(containerName)
            .withMaxKeys(1)
            .withContinuationToken(result.getNextContinuationToken()));
    assertThat(result.getContinuationToken()).isEqualTo("3");
    assertThat(result.getStartAfter()).isEmpty();
    assertThat(result.getNextContinuationToken()).isNull();
    assertThat(result.isTruncated()).isFalse();
    assertThat(result.getObjectSummaries()).hasSize(1);
    assertThat(result.getObjectSummaries().get(0).getKey()).isEqualTo("4");
}
 

@Test
public void testListVersion2() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 

@Test
public void testListVersion2WithRequesterPays() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.REQUESTER_PAYS, "true");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertTrue(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 

private ExpandedGlob expandGlob(S3ResourceId glob) {
  // The S3 API can list objects, filtered by prefix, but not by wildcard.
  // Here, we find the longest prefix without wildcard "*",
  // then filter the results with a regex.
  checkArgument(glob.isWildcard(), "isWildcard");
  String keyPrefix = glob.getKeyNonWildcardPrefix();
  Pattern wildcardRegexp = Pattern.compile(wildcardToRegexp(glob.getKey()));

  LOG.debug(
      "expanding bucket {}, prefix {}, against pattern {}",
      glob.getBucket(),
      keyPrefix,
      wildcardRegexp.toString());

  ImmutableList.Builder<S3ResourceId> expandedPaths = ImmutableList.builder();
  String continuationToken = null;

  do {
    ListObjectsV2Request request =
        new ListObjectsV2Request()
            .withBucketName(glob.getBucket())
            .withPrefix(keyPrefix)
            .withContinuationToken(continuationToken);
    ListObjectsV2Result result;
    try {
      result = amazonS3.get().listObjectsV2(request);
    } catch (AmazonClientException e) {
      return ExpandedGlob.create(glob, new IOException(e));
    }
    continuationToken = result.getNextContinuationToken();

    for (S3ObjectSummary objectSummary : result.getObjectSummaries()) {
      // Filter against regex.
      if (wildcardRegexp.matcher(objectSummary.getKey()).matches()) {
        S3ResourceId expandedPath =
            S3ResourceId.fromComponents(objectSummary.getBucketName(), objectSummary.getKey())
                .withSize(objectSummary.getSize())
                .withLastModified(objectSummary.getLastModified());
        LOG.debug("Expanded S3 object path {}", expandedPath);
        expandedPaths.add(expandedPath);
      }
    }
  } while (continuationToken != null);

  return ExpandedGlob.create(glob, expandedPaths.build());
}
 

private Iterator<LocatedFileStatus> statusFromListing(ListObjectsV2Result listing)
{
    return Iterators.concat(
            statusFromPrefixes(listing.getCommonPrefixes()),
            statusFromObjects(listing.getObjectSummaries()));
}