Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary#setLastModified()

The following examples show how to use com.amazonaws.services.s3.model.S3ObjectSummary#setLastModified() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestListS3.java From nifi with Apache License 2.0

6 votes

@Test
public void testWriteObjectTags() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_OBJECT_TAGS, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectTaggingRequest> captureRequest = ArgumentCaptor.forClass(GetObjectTaggingRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectTagging(captureRequest.capture());
    GetObjectTaggingRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}

Example 2

Source File: MockAmazonS3.java From presto with Apache License 2.0

5 votes

@Override
public ListObjectsV2Result listObjectsV2(ListObjectsV2Request listObjectsV2Request)
{
    final String continuationToken = "continue";

    ListObjectsV2Result listingV2 = new ListObjectsV2Result();

    if (continuationToken.equals(listObjectsV2Request.getContinuationToken())) {
        S3ObjectSummary standardTwo = new S3ObjectSummary();
        standardTwo.setStorageClass(StorageClass.Standard.toString());
        standardTwo.setKey("test/standardTwo");
        standardTwo.setLastModified(new Date());
        listingV2.getObjectSummaries().add(standardTwo);

        if (hasGlacierObjects) {
            S3ObjectSummary glacier = new S3ObjectSummary();
            glacier.setStorageClass(StorageClass.Glacier.toString());
            glacier.setKey("test/glacier");
            glacier.setLastModified(new Date());
            listingV2.getObjectSummaries().add(glacier);
        }
    }
    else {
        S3ObjectSummary standardOne = new S3ObjectSummary();
        standardOne.setStorageClass(StorageClass.Standard.toString());
        standardOne.setKey("test/standardOne");
        standardOne.setLastModified(new Date());
        listingV2.getObjectSummaries().add(standardOne);
        listingV2.setTruncated(true);
        listingV2.setNextContinuationToken(continuationToken);
    }

    return listingV2;
}

Example 3

Source File: S3FindFilesStepTest.java From pipeline-aws-plugin with Apache License 2.0

5 votes

@Test
public void createFileWrapperFromFile() throws Exception {
	FileWrapper file;
	S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
	s3ObjectSummary.setBucketName("my-bucket");
	s3ObjectSummary.setKey("path/to/my/file.ext");
	s3ObjectSummary.setLastModified(new Date(9000));
	s3ObjectSummary.setSize(12);

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(0, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("path/to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(1, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(2, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());
}

Example 4

Source File: AmazonS3Util.java From datacollector with Apache License 2.0

5 votes

@NotNull
private static S3ObjectSummary getObjectSummary(AmazonS3 s3Client, String bucket, String key) {
  S3Object currentObject = s3Client.getObject(bucket, key);
  S3ObjectSummary currentObjectSummary = new S3ObjectSummary();
  currentObjectSummary.setBucketName(currentObject.getBucketName());
  currentObjectSummary.setKey(currentObject.getKey());
  currentObjectSummary.setETag(currentObject.getObjectMetadata().getETag());
  currentObjectSummary.setSize(currentObject.getObjectMetadata().getContentLength());
  currentObjectSummary.setLastModified(currentObject.getObjectMetadata().getLastModified());
  currentObjectSummary.setStorageClass(currentObject.getObjectMetadata().getStorageClass());
  return currentObjectSummary;
}

Example 5

Source File: TestListS3.java From nifi with Apache License 2.0

5 votes

@Test
public void testListObjectsNothingNew() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    Calendar calendar = Calendar.getInstance();
    calendar.set(2017, 5, 2);
    Date objectLastModified = calendar.getTime();
    long stateCurrentTimestamp = objectLastModified.getTime();

    Map<String, String> state = new HashMap<>();
    state.put(ListS3.CURRENT_TIMESTAMP, String.valueOf(stateCurrentTimestamp));
    state.put(ListS3.CURRENT_KEY_PREFIX+"0", "test-key");
    MockStateManager mockStateManager = runner.getStateManager();
    mockStateManager.setState(state, Scope.CLUSTER);

    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("test-key");
    objectSummary1.setLastModified(objectLastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 0);
}

Example 6

Source File: TestListS3.java From nifi with Apache License 2.0

5 votes

@Test
public void testWriteUserMetadata() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_USER_METADATA, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectMetadataRequest> captureRequest = ArgumentCaptor.forClass(GetObjectMetadataRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectMetadata(captureRequest.capture());
    GetObjectMetadataRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());

    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}

Example 7

Source File: S3FileSystemTest.java From beam with Apache License 2.0

4 votes

@Test
public void matchGlob() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/foo/bar*baz");

  ListObjectsV2Request firstRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken(null);

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary firstMatch = new S3ObjectSummary();
  firstMatch.setBucketName(path.getBucket());
  firstMatch.setKey("foo/bar0baz");
  firstMatch.setSize(100);
  firstMatch.setLastModified(new Date(1540000000001L));

  // Expected to not be returned; prefix matches, but substring after wildcard does not
  S3ObjectSummary secondMatch = new S3ObjectSummary();
  secondMatch.setBucketName(path.getBucket());
  secondMatch.setKey("foo/bar1qux");
  secondMatch.setSize(200);
  secondMatch.setLastModified(new Date(1540000000002L));

  // Expected first request returns continuation token
  ListObjectsV2Result firstResult = new ListObjectsV2Result();
  firstResult.setNextContinuationToken("token");
  firstResult.getObjectSummaries().add(firstMatch);
  firstResult.getObjectSummaries().add(secondMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest))))
      .thenReturn(firstResult);

  // Expect second request with continuation token
  ListObjectsV2Request secondRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken("token");

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary thirdMatch = new S3ObjectSummary();
  thirdMatch.setBucketName(path.getBucket());
  thirdMatch.setKey("foo/bar2baz");
  thirdMatch.setSize(300);
  thirdMatch.setLastModified(new Date(1540000000003L));

  // Expected second request returns third prefix match and no continuation token
  ListObjectsV2Result secondResult = new ListObjectsV2Result();
  secondResult.setNextContinuationToken(null);
  secondResult.getObjectSummaries().add(thirdMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest))))
      .thenReturn(secondResult);

  // Expect object metadata queries for content encoding
  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);

  assertThat(
      s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0),
      MatchResultMatcher.create(
          ImmutableList.of(
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          firstMatch.getBucketName(), firstMatch.getKey()))
                  .setSizeBytes(firstMatch.getSize())
                  .setLastModifiedMillis(firstMatch.getLastModified().getTime())
                  .build(),
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          thirdMatch.getBucketName(), thirdMatch.getKey()))
                  .setSizeBytes(thirdMatch.getSize())
                  .setLastModifiedMillis(thirdMatch.getLastModified().getTime())
                  .build())));
}

Example 8

Source File: S3FileSystemTest.java From beam with Apache License 2.0

4 votes

@Test
public void matchGlobWithSlashes() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/foo/bar\\baz*");

  ListObjectsV2Request request =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken(null);

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary firstMatch = new S3ObjectSummary();
  firstMatch.setBucketName(path.getBucket());
  firstMatch.setKey("foo/bar\\baz0");
  firstMatch.setSize(100);
  firstMatch.setLastModified(new Date(1540000000001L));

  // Expected to not be returned; prefix matches, but substring after wildcard does not
  S3ObjectSummary secondMatch = new S3ObjectSummary();
  secondMatch.setBucketName(path.getBucket());
  secondMatch.setKey("foo/bar/baz1");
  secondMatch.setSize(200);
  secondMatch.setLastModified(new Date(1540000000002L));

  // Expected first request returns continuation token
  ListObjectsV2Result result = new ListObjectsV2Result();
  result.getObjectSummaries().add(firstMatch);
  result.getObjectSummaries().add(secondMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(request))))
      .thenReturn(result);

  // Expect object metadata queries for content encoding
  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);

  assertThat(
      s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0),
      MatchResultMatcher.create(
          ImmutableList.of(
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          firstMatch.getBucketName(), firstMatch.getKey()))
                  .setSizeBytes(firstMatch.getSize())
                  .setLastModifiedMillis(firstMatch.getLastModified().getTime())
                  .build())));
}

Example 9

Source File: S3FileSystemTest.java From beam with Apache License 2.0

4 votes

@Test
public void matchVariousInvokeThreadPool() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
  notFoundException.setStatusCode(404);
  S3ResourceId pathNotExist =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathNotExist.getBucket(), pathNotExist.getKey())))))
      .thenThrow(notFoundException);

  AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
  forbiddenException.setStatusCode(403);
  S3ResourceId pathForbidden =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathForbidden.getBucket(), pathForbidden.getKey())))))
      .thenThrow(forbiddenException);

  S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
  ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
  s3ObjectMetadata.setContentLength(100);
  s3ObjectMetadata.setLastModified(new Date(1540000000000L));
  s3ObjectMetadata.setContentEncoding("not-gzip");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey())))))
      .thenReturn(s3ObjectMetadata);

  S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");

  S3ObjectSummary foundListObject = new S3ObjectSummary();
  foundListObject.setBucketName(pathGlob.getBucket());
  foundListObject.setKey("path/part-0");
  foundListObject.setSize(200);
  foundListObject.setLastModified(new Date(1541000000000L));

  ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
  listObjectsResult.setNextContinuationToken(null);
  listObjectsResult.getObjectSummaries().add(foundListObject);
  when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class)))
      .thenReturn(listObjectsResult);

  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0")))))
      .thenReturn(metadata);

  assertThat(
      s3FileSystem.match(
          ImmutableList.of(
              pathNotExist.toString(),
              pathForbidden.toString(),
              pathExist.toString(),
              pathGlob.toString())),
      contains(
          MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()),
          MatchResultMatcher.create(
              MatchResult.Status.ERROR, new IOException(forbiddenException)),
          MatchResultMatcher.create(100, 1540000000000L, pathExist, true),
          MatchResultMatcher.create(
              200,
              1541000000000L,
              S3ResourceId.fromComponents(pathGlob.getBucket(), foundListObject.getKey()),
              true)));
}

Example 10

Source File: TestListS3.java From nifi with Apache License 2.0

4 votes

@Test
public void testList() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}

Example 11

Source File: TestListS3.java From nifi with Apache License 2.0

4 votes

@Test
public void testListWithRecords() throws InitializationException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    final MockRecordWriter recordWriter = new MockRecordWriter(null, false);
    runner.addControllerService("record-writer", recordWriter);
    runner.enableControllerService(recordWriter);
    runner.setProperty(ListS3.RECORD_WRITER, "record-writer");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);

    final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
    final String lastModifiedString = dateFormat.format(lastModified);

    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS).get(0);
    flowFile.assertAttributeEquals("record.count", "3");
    flowFile.assertContentEquals("a,test-bucket,,," + lastModifiedString + ",0,,true,,,\n"
        + "b/c,test-bucket,,," + lastModifiedString + ",0,,true,,,\n"
        + "d/e,test-bucket,,," + lastModifiedString + ",0,,true,,,\n");
}

Example 12

Source File: TestListS3.java From nifi with Apache License 2.0

4 votes

@Test
public void testListWithRequesterPays() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.REQUESTER_PAYS, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertTrue(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}

Example 13

Source File: TestListS3.java From nifi with Apache License 2.0

4 votes

@Test
public void testListVersion2() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}

Example 14

Source File: TestListS3.java From nifi with Apache License 2.0

4 votes

@Test
public void testListVersion2WithRequesterPays() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.REQUESTER_PAYS, "true");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertTrue(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}

Example 15

Source File: TestListS3.java From nifi with Apache License 2.0

4 votes

@Test
public void testListIgnoreByMinAge() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.MIN_AGE, "30 sec");

    Date lastModifiedNow = new Date();
    Date lastModifiedMinus1Hour = DateUtils.addHours(lastModifiedNow, -1);
    Date lastModifiedMinus3Hour = DateUtils.addHours(lastModifiedNow, -3);
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("minus-3hour");
    objectSummary1.setLastModified(lastModifiedMinus3Hour);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("minus-1hour");
    objectSummary2.setLastModified(lastModifiedMinus1Hour);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("now");
    objectSummary3.setLastModified(lastModifiedNow);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    Map<String,String> stateMap = new HashMap<>();
    String previousTimestamp = String.valueOf(lastModifiedMinus3Hour.getTime());
    stateMap.put(ListS3.CURRENT_TIMESTAMP, previousTimestamp);
    stateMap.put(ListS3.CURRENT_KEY_PREFIX + "0", "minus-3hour");
    runner.getStateManager().setState(stateMap, Scope.CLUSTER);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "minus-1hour");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModifiedMinus1Hour.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}