Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary#setKey()

The following examples show how to use com.amazonaws.services.s3.model.S3ObjectSummary#setKey() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: S3ServiceTest.java    From herd with Apache License 2.0 6 votes vote down vote up
@Test
public void testTagObjects()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Call the method under test.
    s3Service.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(s3Dao).tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);
    verifyNoMoreInteractions(s3Dao);
}
 
Example 2
Source File: S3UtilsTest.java    From micro-server with Apache License 2.0 5 votes vote down vote up
private List<S3ObjectSummary> createSummaries() {
    List<S3ObjectSummary> summaries = new ArrayList<>();

    for (int i = 0; i < 1000; i++) {
        S3ObjectSummary summary = new S3ObjectSummary();
        summary.setKey("" + i);
        summaries.add(summary);
    }

    return summaries;
}
 
Example 3
Source File: TerrapinUtilTest.java    From terrapin with Apache License 2.0 5 votes vote down vote up
@Test
@PrepareForTest(TerrapinUtil.class)
public void testGetS3FileList() throws Exception {
  AmazonS3Client s3Client = mock(AmazonS3Client.class);
  ObjectListing objectListing = mock(ObjectListing.class);
  S3ObjectSummary summary1 = new S3ObjectSummary();
  S3ObjectSummary summary2 = new S3ObjectSummary();
  S3ObjectSummary summary3 = new S3ObjectSummary();
  summary1.setKey("/abc/123");
  summary2.setKey("/abc/456");
  summary3.setKey("/def/123");
  summary1.setSize(32432);
  summary2.setSize(213423);
  summary3.setSize(2334);
  List<S3ObjectSummary> summaries = ImmutableList.of(summary1, summary2, summary3);
  whenNew(AmazonS3Client.class).withAnyArguments().thenReturn(s3Client);
  when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
  when(objectListing.getObjectSummaries()).thenReturn(summaries);

  List<Pair<Path, Long>> results = TerrapinUtil.getS3FileList(new AnonymousAWSCredentials(),
      "bucket", "/abc");

  assertEquals(2, results.size());
  assertTrue(results.get(0).getLeft().toString().endsWith(summary1.getKey()));
  assertEquals(new Long(summary1.getSize()), results.get(0).getRight());
  assertTrue(results.get(1).getLeft().toString().endsWith(summary2.getKey()));
  assertEquals(new Long(summary2.getSize()), results.get(1).getRight());
}
 
Example 4
Source File: S3FileInputTest.java    From crate with Apache License 2.0 5 votes vote down vote up
private List<S3ObjectSummary> objectSummaries() {
    listObjectSummaries = new LinkedList<>();

    S3ObjectSummary firstObj = new S3ObjectSummary();
    S3ObjectSummary secondObj = new S3ObjectSummary();
    firstObj.setBucketName(BUCKET_NAME);
    secondObj.setBucketName(BUCKET_NAME);
    firstObj.setKey("prefix/test1.json.gz");
    secondObj.setKey("prefix/test2.json.gz");
    listObjectSummaries.add(firstObj);
    listObjectSummaries.add(secondObj);
    return listObjectSummaries;
}
 
Example 5
Source File: MockS3OperationsImpl.java    From herd with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 * <p/>
 * If the bucket does not exist, returns a listing with an empty list. If a prefix is specified in listObjectsRequest, only keys starting with the prefix
 * will be returned.
 */
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest, AmazonS3 s3Client)
{
    LOGGER.debug("listObjects(): listObjectsRequest.getBucketName() = " + listObjectsRequest.getBucketName());

    String bucketName = listObjectsRequest.getBucketName();

    if (MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION.equals(bucketName))
    {
        AmazonS3Exception amazonS3Exception = new AmazonS3Exception(MOCK_S3_BUCKET_NAME_NO_SUCH_BUCKET_EXCEPTION);
        amazonS3Exception.setErrorCode("NoSuchBucket");
        throw amazonS3Exception;
    }

    ObjectListing objectListing = new ObjectListing();
    objectListing.setBucketName(bucketName);

    MockS3Bucket mockS3Bucket = mockS3Buckets.get(bucketName);
    if (mockS3Bucket != null)
    {
        for (MockS3Object mockS3Object : mockS3Bucket.getObjects().values())
        {
            String s3ObjectKey = mockS3Object.getKey();
            if (listObjectsRequest.getPrefix() == null || s3ObjectKey.startsWith(listObjectsRequest.getPrefix()))
            {
                S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
                s3ObjectSummary.setBucketName(bucketName);
                s3ObjectSummary.setKey(s3ObjectKey);
                s3ObjectSummary.setSize(mockS3Object.getData().length);
                s3ObjectSummary.setStorageClass(mockS3Object.getObjectMetadata() != null ? mockS3Object.getObjectMetadata().getStorageClass() : null);

                objectListing.getObjectSummaries().add(s3ObjectSummary);
            }
        }
    }

    return objectListing;
}
 
Example 6
Source File: S3DaoTest.java    From herd with Apache License 2.0 5 votes vote down vote up
@Test
public void testTagObjectsOtherTagKeyAlreadyExists()
{
    // Create two S3 object tags having different tag keys.
    List<Tag> tags = Arrays.asList(new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE), new Tag(S3_OBJECT_TAG_KEY_2, S3_OBJECT_TAG_VALUE_2));

    // Put a file in S3 that is already tagged with the first S3 object tag.
    PutObjectRequest putObjectRequest = new PutObjectRequest(S3_BUCKET_NAME, TARGET_S3_KEY, new ByteArrayInputStream(new byte[1]), new ObjectMetadata());
    putObjectRequest.setTagging(new ObjectTagging(Collections.singletonList(tags.get(0))));
    s3Operations.putObject(putObjectRequest, null);

    // Validate that the S3 object is tagged with the first tag only.
    GetObjectTaggingResult getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(Collections.singletonList(tags.get(0)), getObjectTaggingResult.getTagSet());

    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto params = new S3FileTransferRequestParamsDto();
    params.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(TARGET_S3_KEY);

    // Tag the S3 file with the second S3 object tag.
    s3Dao.tagObjects(params, new S3FileTransferRequestParamsDto(), Collections.singletonList(s3ObjectSummary), tags.get(1));

    // Validate that the S3 object is now tagged with both tags.
    getObjectTaggingResult = s3Operations.getObjectTagging(new GetObjectTaggingRequest(S3_BUCKET_NAME, TARGET_S3_KEY), null);
    assertEquals(tags.size(), getObjectTaggingResult.getTagSet().size());
    assertTrue(getObjectTaggingResult.getTagSet().containsAll(tags));
}
 
Example 7
Source File: TestListS3.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testListObjectsNothingNew() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    Calendar calendar = Calendar.getInstance();
    calendar.set(2017, 5, 2);
    Date objectLastModified = calendar.getTime();
    long stateCurrentTimestamp = objectLastModified.getTime();

    Map<String, String> state = new HashMap<>();
    state.put(ListS3.CURRENT_TIMESTAMP, String.valueOf(stateCurrentTimestamp));
    state.put(ListS3.CURRENT_KEY_PREFIX+"0", "test-key");
    MockStateManager mockStateManager = runner.getStateManager();
    mockStateManager.setState(state, Scope.CLUSTER);

    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("test-key");
    objectSummary1.setLastModified(objectLastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 0);
}
 
Example 8
Source File: S3FindFilesStepTest.java    From pipeline-aws-plugin with Apache License 2.0 5 votes vote down vote up
@Test
public void createFileWrapperFromFile() throws Exception {
	FileWrapper file;
	S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
	s3ObjectSummary.setBucketName("my-bucket");
	s3ObjectSummary.setKey("path/to/my/file.ext");
	s3ObjectSummary.setLastModified(new Date(9000));
	s3ObjectSummary.setSize(12);

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(0, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("path/to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(1, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("to/my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());

	file = S3FindFilesStep.Execution.createFileWrapperFromFile(2, Paths.get(s3ObjectSummary.getKey()), s3ObjectSummary);
	Assert.assertEquals("file.ext", file.getName());
	Assert.assertEquals("my/file.ext", file.getPath());
	Assert.assertFalse(file.isDirectory());
	Assert.assertEquals(12, file.getLength());
	Assert.assertEquals(9000, file.getLastModified());
}
 
Example 9
Source File: TestListS3.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteUserMetadata() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.WRITE_USER_METADATA, "true");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);

    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<GetObjectMetadataRequest> captureRequest = ArgumentCaptor.forClass(GetObjectMetadataRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).getObjectMetadata(captureRequest.capture());
    GetObjectMetadataRequest request = captureRequest.getValue();

    assertEquals("test-bucket", request.getBucketName());
    assertEquals("a", request.getKey());

    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());
}
 
Example 10
Source File: TestListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testListVersion2WithRequesterPays() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.REQUESTER_PAYS, "true");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertTrue(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 
Example 11
Source File: S3FileSystemTest.java    From beam with Apache License 2.0 4 votes vote down vote up
@Test
public void matchVariousInvokeThreadPool() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  AmazonS3Exception notFoundException = new AmazonS3Exception("mock exception");
  notFoundException.setStatusCode(404);
  S3ResourceId pathNotExist =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/nonexistentfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathNotExist.getBucket(), pathNotExist.getKey())))))
      .thenThrow(notFoundException);

  AmazonS3Exception forbiddenException = new AmazonS3Exception("mock exception");
  forbiddenException.setStatusCode(403);
  S3ResourceId pathForbidden =
      S3ResourceId.fromUri("s3://testbucket/testdirectory/forbiddenfile");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(
                          pathForbidden.getBucket(), pathForbidden.getKey())))))
      .thenThrow(forbiddenException);

  S3ResourceId pathExist = S3ResourceId.fromUri("s3://testbucket/testdirectory/filethatexists");
  ObjectMetadata s3ObjectMetadata = new ObjectMetadata();
  s3ObjectMetadata.setContentLength(100);
  s3ObjectMetadata.setLastModified(new Date(1540000000000L));
  s3ObjectMetadata.setContentEncoding("not-gzip");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathExist.getBucket(), pathExist.getKey())))))
      .thenReturn(s3ObjectMetadata);

  S3ResourceId pathGlob = S3ResourceId.fromUri("s3://testbucket/path/part*");

  S3ObjectSummary foundListObject = new S3ObjectSummary();
  foundListObject.setBucketName(pathGlob.getBucket());
  foundListObject.setKey("path/part-0");
  foundListObject.setSize(200);
  foundListObject.setLastModified(new Date(1541000000000L));

  ListObjectsV2Result listObjectsResult = new ListObjectsV2Result();
  listObjectsResult.setNextContinuationToken(null);
  listObjectsResult.getObjectSummaries().add(foundListObject);
  when(s3FileSystem.getAmazonS3Client().listObjectsV2(notNull(ListObjectsV2Request.class)))
      .thenReturn(listObjectsResult);

  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem
          .getAmazonS3Client()
          .getObjectMetadata(
              argThat(
                  new GetObjectMetadataRequestMatcher(
                      new GetObjectMetadataRequest(pathGlob.getBucket(), "path/part-0")))))
      .thenReturn(metadata);

  assertThat(
      s3FileSystem.match(
          ImmutableList.of(
              pathNotExist.toString(),
              pathForbidden.toString(),
              pathExist.toString(),
              pathGlob.toString())),
      contains(
          MatchResultMatcher.create(MatchResult.Status.NOT_FOUND, new FileNotFoundException()),
          MatchResultMatcher.create(
              MatchResult.Status.ERROR, new IOException(forbiddenException)),
          MatchResultMatcher.create(100, 1540000000000L, pathExist, true),
          MatchResultMatcher.create(
              200,
              1541000000000L,
              S3ResourceId.fromComponents(pathGlob.getBucket(), foundListObject.getKey()),
              true)));
}
 
Example 12
Source File: AccessValidatorControllerTest.java    From herd with Apache License 2.0 4 votes vote down vote up
@Test
public void testValidateAccessNoStorageFiles() throws Exception
{
    // Create business object data without any registered storage files, but with storage unit directory path and an actual non zero-byte S3 file.
    BusinessObjectData businessObjectData = createBusinessObjectData();
    businessObjectData.getStorageUnits().get(0).setStorageFiles(null);
    StorageDirectory storageDirectory = new StorageDirectory();
    storageDirectory.setDirectoryPath(STORAGE_DIRECTORY_PATH);
    businessObjectData.getStorageUnits().get(0).setStorageDirectory(storageDirectory);

    // Create AWS list objects response with a non zero-byte S3 file.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);
    s3ObjectSummary.setSize(FILE_SIZE_1_KB);
    objectListing = Mockito.mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(Lists.newArrayList(s3ObjectSummary));

    // Create AWS get object request.
    GetObjectRequest getObjectRequest = new GetObjectRequest(S3_BUCKET_NAME, S3_KEY).withRange(0, MAX_BYTE_DOWNLOAD);

    // Create S3 object.
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(RandomStringUtils.randomAlphabetic(MAX_BYTE_DOWNLOAD).getBytes()));

    // Create S3 object metadata.
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(MAX_BYTE_DOWNLOAD);

    // Mock the external calls.
    when(propertiesHelper.getProperty(HERD_BASE_URL_PROPERTY)).thenReturn(HERD_BASE_URL);
    when(propertiesHelper.getProperty(HERD_USERNAME_PROPERTY)).thenReturn(HERD_USERNAME);
    when(propertiesHelper.getProperty(HERD_PASSWORD_PROPERTY)).thenReturn(HERD_PASSWORD);
    when(propertiesHelper.getProperty(AWS_REGION_PROPERTY)).thenReturn(AWS_REGION_NAME_US_EAST_1);
    when(propertiesHelper.getProperty(AWS_ROLE_ARN_PROPERTY)).thenReturn(AWS_ROLE_ARN);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_VERSION.toString());
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_DATA_VERSION.toString());
    when(propertiesHelper.getProperty(NAMESPACE_PROPERTY)).thenReturn(NAMESPACE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY)).thenReturn(BUSINESS_OBJECT_DEFINITION_NAME);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_USAGE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_FILE_TYPE);
    when(propertiesHelper.getProperty(PRIMARY_PARTITION_VALUE_PROPERTY)).thenReturn(PRIMARY_PARTITION_VALUE);
    when(propertiesHelper.getProperty(SUB_PARTITION_VALUES_PROPERTY)).thenReturn(SUB_PARTITION_VALUES);
    when(herdApiClientOperations
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false))).thenReturn(businessObjectData);
    when(s3Operations.listObjects(any(ListObjectsRequest.class), any(AmazonS3.class))).thenReturn(objectListing);
    when(s3Operations.getS3Object(eq(getObjectRequest), any(AmazonS3.class))).thenReturn(s3Object);

    // Call the method under test with message flag set to "false".
    accessValidatorController.validateAccess(new File(PROPERTIES_FILE_PATH), false);

    // Verify the external calls.
    verify(herdApiClientOperations).checkPropertiesFile(propertiesHelper, false);
    verify(propertiesHelper).loadProperties(new File(PROPERTIES_FILE_PATH));
    verify(propertiesHelper).getProperty(HERD_BASE_URL_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_USERNAME_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_PASSWORD_PROPERTY);
    verify(herdApiClientOperations).applicationGetBuildInfo(any(ApplicationApi.class));
    verify(herdApiClientOperations).currentUserGetCurrentUser(any(CurrentUserApi.class));
    verify(propertiesHelper).getProperty(AWS_REGION_PROPERTY);
    verify(propertiesHelper).getProperty(AWS_ROLE_ARN_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(NAMESPACE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY);
    verify(propertiesHelper).getProperty(PRIMARY_PARTITION_VALUE_PROPERTY);
    verify(propertiesHelper).getProperty(SUB_PARTITION_VALUES_PROPERTY);
    verify(herdApiClientOperations)
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false));
    verify(s3Operations).listObjects(any(ListObjectsRequest.class), any(AmazonS3.class));
    verify(s3Operations).getS3Object(eq(getObjectRequest), any(AmazonS3.class));
    verifyNoMoreInteractionsHelper();
}
 
Example 13
Source File: AccessValidatorControllerTest.java    From herd with Apache License 2.0 4 votes vote down vote up
@Test
public void testValidateAccessNoStorageFilesZeroByteS3File() throws Exception
{
    // Create business object data without any registered storage files, but with storage unit directory path and a zero-byte S3 file.
    BusinessObjectData businessObjectData = createBusinessObjectData();
    businessObjectData.getStorageUnits().get(0).setStorageFiles(null);
    StorageDirectory storageDirectory = new StorageDirectory();
    storageDirectory.setDirectoryPath(STORAGE_DIRECTORY_PATH);
    businessObjectData.getStorageUnits().get(0).setStorageDirectory(storageDirectory);

    // Create AWS list objects response with a zero-byte S3 file.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);
    s3ObjectSummary.setSize(FILE_SIZE_0_BYTE);
    objectListing = Mockito.mock(ObjectListing.class);
    when(objectListing.getObjectSummaries()).thenReturn(Lists.newArrayList(s3ObjectSummary));

    // Create AWS get object request.
    GetObjectRequest getObjectRequest = new GetObjectRequest(S3_BUCKET_NAME, S3_KEY).withRange(0, MAX_BYTE_DOWNLOAD);

    // Create S3 object.
    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new ByteArrayInputStream(RandomStringUtils.randomAlphabetic(MAX_BYTE_DOWNLOAD).getBytes()));

    // Create S3 object metadata.
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setContentLength(MAX_BYTE_DOWNLOAD);

    // Mock the external calls.
    when(propertiesHelper.getProperty(HERD_BASE_URL_PROPERTY)).thenReturn(HERD_BASE_URL);
    when(propertiesHelper.getProperty(HERD_USERNAME_PROPERTY)).thenReturn(HERD_USERNAME);
    when(propertiesHelper.getProperty(HERD_PASSWORD_PROPERTY)).thenReturn(HERD_PASSWORD);
    when(propertiesHelper.getProperty(AWS_REGION_PROPERTY)).thenReturn(AWS_REGION_NAME_US_EAST_1);
    when(propertiesHelper.getProperty(AWS_ROLE_ARN_PROPERTY)).thenReturn(AWS_ROLE_ARN);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_VERSION.toString());
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY)).thenReturn(BUSINESS_OBJECT_DATA_VERSION.toString());
    when(propertiesHelper.getProperty(NAMESPACE_PROPERTY)).thenReturn(NAMESPACE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY)).thenReturn(BUSINESS_OBJECT_DEFINITION_NAME);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_USAGE);
    when(propertiesHelper.getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY)).thenReturn(BUSINESS_OBJECT_FORMAT_FILE_TYPE);
    when(propertiesHelper.getProperty(PRIMARY_PARTITION_VALUE_PROPERTY)).thenReturn(PRIMARY_PARTITION_VALUE);
    when(propertiesHelper.getProperty(SUB_PARTITION_VALUES_PROPERTY)).thenReturn(SUB_PARTITION_VALUES);
    when(herdApiClientOperations
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false))).thenReturn(businessObjectData);
    when(s3Operations.listObjects(any(ListObjectsRequest.class), any(AmazonS3.class))).thenReturn(objectListing);

    // Call the method under test with message flag set to "false".
    accessValidatorController.validateAccess(new File(PROPERTIES_FILE_PATH), false);

    // Verify the external calls.
    verify(herdApiClientOperations).checkPropertiesFile(propertiesHelper, false);
    verify(propertiesHelper).loadProperties(new File(PROPERTIES_FILE_PATH));
    verify(propertiesHelper).getProperty(HERD_BASE_URL_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_USERNAME_PROPERTY);
    verify(propertiesHelper).getProperty(HERD_PASSWORD_PROPERTY);
    verify(herdApiClientOperations).applicationGetBuildInfo(any(ApplicationApi.class));
    verify(herdApiClientOperations).currentUserGetCurrentUser(any(CurrentUserApi.class));
    verify(propertiesHelper).getProperty(AWS_REGION_PROPERTY);
    verify(propertiesHelper).getProperty(AWS_ROLE_ARN_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DATA_VERSION_PROPERTY);
    verify(propertiesHelper).getProperty(NAMESPACE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_DEFINITION_NAME_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_USAGE_PROPERTY);
    verify(propertiesHelper).getProperty(BUSINESS_OBJECT_FORMAT_FILE_TYPE_PROPERTY);
    verify(propertiesHelper).getProperty(PRIMARY_PARTITION_VALUE_PROPERTY);
    verify(propertiesHelper).getProperty(SUB_PARTITION_VALUES_PROPERTY);
    verify(herdApiClientOperations)
        .businessObjectDataGetBusinessObjectData(any(BusinessObjectDataApi.class), eq(NAMESPACE), eq(BUSINESS_OBJECT_DEFINITION_NAME),
            eq(BUSINESS_OBJECT_FORMAT_USAGE), eq(BUSINESS_OBJECT_FORMAT_FILE_TYPE), eq(null), eq(PRIMARY_PARTITION_VALUE), eq(SUB_PARTITION_VALUES),
            eq(BUSINESS_OBJECT_FORMAT_VERSION), eq(BUSINESS_OBJECT_DATA_VERSION), eq(null), eq(false), eq(false));
    verify(s3Operations).listObjects(any(ListObjectsRequest.class), any(AmazonS3.class));
    verifyNoMoreInteractionsHelper();
}
 
Example 14
Source File: S3FileSystemTest.java    From beam with Apache License 2.0 4 votes vote down vote up
@Test
public void matchGlob() throws IOException {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(s3Options());

  S3ResourceId path = S3ResourceId.fromUri("s3://testbucket/foo/bar*baz");

  ListObjectsV2Request firstRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken(null);

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary firstMatch = new S3ObjectSummary();
  firstMatch.setBucketName(path.getBucket());
  firstMatch.setKey("foo/bar0baz");
  firstMatch.setSize(100);
  firstMatch.setLastModified(new Date(1540000000001L));

  // Expected to not be returned; prefix matches, but substring after wildcard does not
  S3ObjectSummary secondMatch = new S3ObjectSummary();
  secondMatch.setBucketName(path.getBucket());
  secondMatch.setKey("foo/bar1qux");
  secondMatch.setSize(200);
  secondMatch.setLastModified(new Date(1540000000002L));

  // Expected first request returns continuation token
  ListObjectsV2Result firstResult = new ListObjectsV2Result();
  firstResult.setNextContinuationToken("token");
  firstResult.getObjectSummaries().add(firstMatch);
  firstResult.getObjectSummaries().add(secondMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(firstRequest))))
      .thenReturn(firstResult);

  // Expect second request with continuation token
  ListObjectsV2Request secondRequest =
      new ListObjectsV2Request()
          .withBucketName(path.getBucket())
          .withPrefix(path.getKeyNonWildcardPrefix())
          .withContinuationToken("token");

  // Expected to be returned; prefix and wildcard/regex match
  S3ObjectSummary thirdMatch = new S3ObjectSummary();
  thirdMatch.setBucketName(path.getBucket());
  thirdMatch.setKey("foo/bar2baz");
  thirdMatch.setSize(300);
  thirdMatch.setLastModified(new Date(1540000000003L));

  // Expected second request returns third prefix match and no continuation token
  ListObjectsV2Result secondResult = new ListObjectsV2Result();
  secondResult.setNextContinuationToken(null);
  secondResult.getObjectSummaries().add(thirdMatch);
  when(s3FileSystem
          .getAmazonS3Client()
          .listObjectsV2(argThat(new ListObjectsV2RequestArgumentMatches(secondRequest))))
      .thenReturn(secondResult);

  // Expect object metadata queries for content encoding
  ObjectMetadata metadata = new ObjectMetadata();
  metadata.setContentEncoding("");
  when(s3FileSystem.getAmazonS3Client().getObjectMetadata(anyObject())).thenReturn(metadata);

  assertThat(
      s3FileSystem.matchGlobPaths(ImmutableList.of(path)).get(0),
      MatchResultMatcher.create(
          ImmutableList.of(
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          firstMatch.getBucketName(), firstMatch.getKey()))
                  .setSizeBytes(firstMatch.getSize())
                  .setLastModifiedMillis(firstMatch.getLastModified().getTime())
                  .build(),
              MatchResult.Metadata.builder()
                  .setIsReadSeekEfficient(true)
                  .setResourceId(
                      S3ResourceId.fromComponents(
                          thirdMatch.getBucketName(), thirdMatch.getKey()))
                  .setSizeBytes(thirdMatch.getSize())
                  .setLastModifiedMillis(thirdMatch.getLastModified().getTime())
                  .build())));
}
 
Example 15
Source File: TestListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testListIgnoreByMinAge() throws IOException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.MIN_AGE, "30 sec");

    Date lastModifiedNow = new Date();
    Date lastModifiedMinus1Hour = DateUtils.addHours(lastModifiedNow, -1);
    Date lastModifiedMinus3Hour = DateUtils.addHours(lastModifiedNow, -3);
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("minus-3hour");
    objectSummary1.setLastModified(lastModifiedMinus3Hour);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("minus-1hour");
    objectSummary2.setLastModified(lastModifiedMinus1Hour);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("now");
    objectSummary3.setLastModified(lastModifiedNow);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    Map<String,String> stateMap = new HashMap<>();
    String previousTimestamp = String.valueOf(lastModifiedMinus3Hour.getTime());
    stateMap.put(ListS3.CURRENT_TIMESTAMP, previousTimestamp);
    stateMap.put(ListS3.CURRENT_KEY_PREFIX + "0", "minus-3hour");
    runner.getStateManager().setState(stateMap, Scope.CLUSTER);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "minus-1hour");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModifiedMinus1Hour.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 
Example 16
Source File: TestListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testListWithRecords() throws InitializationException {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");

    final MockRecordWriter recordWriter = new MockRecordWriter(null, false);
    runner.addControllerService("record-writer", recordWriter);
    runner.enableControllerService(recordWriter);
    runner.setProperty(ListS3.RECORD_WRITER, "record-writer");

    Date lastModified = new Date();
    ObjectListing objectListing = new ObjectListing();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjects(Mockito.any(ListObjectsRequest.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsRequest> captureRequest = ArgumentCaptor.forClass(ListObjectsRequest.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjects(captureRequest.capture());
    ListObjectsRequest request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 1);

    final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
    final String lastModifiedString = dateFormat.format(lastModified);

    final MockFlowFile flowFile = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS).get(0);
    flowFile.assertAttributeEquals("record.count", "3");
    flowFile.assertContentEquals("a,test-bucket,,," + lastModifiedString + ",0,,true,,,\n"
        + "b/c,test-bucket,,," + lastModifiedString + ",0,,true,,,\n"
        + "d/e,test-bucket,,," + lastModifiedString + ",0,,true,,,\n");
}
 
Example 17
Source File: S3ObjectsProviderTest.java    From pentaho-kettle with Apache License 2.0 4 votes vote down vote up
private static S3ObjectSummary buildObjectSummary( String bucketName, String key, String dataString ) {
  S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
  s3ObjectSummary.setKey( key );
  s3ObjectSummary.setBucketName( bucketName );
  return s3ObjectSummary;
}
 
Example 18
Source File: S3DaoImplTest.java    From herd with Apache License 2.0 4 votes vote down vote up
private void runTagObjectsTest()
{
    // Create an S3 file transfer request parameters DTO to access S3 objects.
    S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto();
    s3FileTransferRequestParamsDto.setS3BucketName(S3_BUCKET_NAME);

    // Create an S3 file transfer request parameters DTO to tag S3 objects.
    S3FileTransferRequestParamsDto s3ObjectTaggerParamsDto = new S3FileTransferRequestParamsDto();
    s3ObjectTaggerParamsDto.setAwsAccessKeyId(AWS_ASSUMED_ROLE_ACCESS_KEY);
    s3ObjectTaggerParamsDto.setAwsSecretKey(AWS_ASSUMED_ROLE_SECRET_KEY);
    s3ObjectTaggerParamsDto.setSessionToken(AWS_ASSUMED_ROLE_SESSION_TOKEN);

    // Create an S3 object summary.
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(S3_KEY);

    // Create an S3 object tag.
    Tag tag = new Tag(S3_OBJECT_TAG_KEY, S3_OBJECT_TAG_VALUE);

    // Create a retry policy.
    RetryPolicy retryPolicy =
        new RetryPolicy(PredefinedRetryPolicies.DEFAULT_RETRY_CONDITION, PredefinedRetryPolicies.DEFAULT_BACKOFF_STRATEGY, INTEGER_VALUE, true);

    // Create a get object tagging result.
    GetObjectTaggingResult getObjectTaggingResult = new GetObjectTaggingResult(null);

    // Create a set object tagging result.
    SetObjectTaggingResult setObjectTaggingResult = new SetObjectTaggingResult();

    // Mock the external calls.
    when(retryPolicyFactory.getRetryPolicy()).thenReturn(retryPolicy);
    when(s3Operations.getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(getObjectTaggingResult);
    when(s3Operations.setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class))).thenReturn(setObjectTaggingResult);

    // Call the method under test.
    s3DaoImpl.tagObjects(s3FileTransferRequestParamsDto, s3ObjectTaggerParamsDto, Collections.singletonList(s3ObjectSummary), tag);

    // Verify the external calls.
    verify(retryPolicyFactory, times(2)).getRetryPolicy();
    verify(s3Operations).getObjectTagging(any(GetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verify(s3Operations).setObjectTagging(any(SetObjectTaggingRequest.class), any(AmazonS3Client.class));
    verifyNoMoreInteractionsHelper();
}
 
Example 19
Source File: TestListS3.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Test
public void testListVersion2() {
    runner.setProperty(ListS3.REGION, "eu-west-1");
    runner.setProperty(ListS3.BUCKET, "test-bucket");
    runner.setProperty(ListS3.LIST_TYPE, "2");

    Date lastModified = new Date();
    ListObjectsV2Result objectListing = new ListObjectsV2Result();
    S3ObjectSummary objectSummary1 = new S3ObjectSummary();
    objectSummary1.setBucketName("test-bucket");
    objectSummary1.setKey("a");
    objectSummary1.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary1);
    S3ObjectSummary objectSummary2 = new S3ObjectSummary();
    objectSummary2.setBucketName("test-bucket");
    objectSummary2.setKey("b/c");
    objectSummary2.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary2);
    S3ObjectSummary objectSummary3 = new S3ObjectSummary();
    objectSummary3.setBucketName("test-bucket");
    objectSummary3.setKey("d/e");
    objectSummary3.setLastModified(lastModified);
    objectListing.getObjectSummaries().add(objectSummary3);
    Mockito.when(mockS3Client.listObjectsV2(Mockito.any(ListObjectsV2Request.class))).thenReturn(objectListing);

    runner.run();

    ArgumentCaptor<ListObjectsV2Request> captureRequest = ArgumentCaptor.forClass(ListObjectsV2Request.class);
    Mockito.verify(mockS3Client, Mockito.times(1)).listObjectsV2(captureRequest.capture());
    ListObjectsV2Request request = captureRequest.getValue();
    assertEquals("test-bucket", request.getBucketName());
    assertFalse(request.isRequesterPays());
    Mockito.verify(mockS3Client, Mockito.never()).listVersions(Mockito.any());

    runner.assertAllFlowFilesTransferred(ListS3.REL_SUCCESS, 3);
    List<MockFlowFile> flowFiles = runner.getFlowFilesForRelationship(ListS3.REL_SUCCESS);
    MockFlowFile ff0 = flowFiles.get(0);
    ff0.assertAttributeEquals("filename", "a");
    ff0.assertAttributeEquals("s3.bucket", "test-bucket");
    String lastModifiedTimestamp = String.valueOf(lastModified.getTime());
    ff0.assertAttributeEquals("s3.lastModified", lastModifiedTimestamp);
    flowFiles.get(1).assertAttributeEquals("filename", "b/c");
    flowFiles.get(2).assertAttributeEquals("filename", "d/e");
    runner.getStateManager().assertStateEquals(ListS3.CURRENT_TIMESTAMP, lastModifiedTimestamp, Scope.CLUSTER);
}
 
Example 20
Source File: StorageFileHelperTest.java    From herd with Apache License 2.0 3 votes vote down vote up
/**
 * Creates an S3 object summary.
 *
 * @param filePath the file path
 * @param fileSizeInBytes the file size in bytes
 *
 * @return the S3 object summary
 */
private S3ObjectSummary createS3ObjectSummary(String filePath, long fileSizeInBytes)
{
    S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
    s3ObjectSummary.setKey(filePath);
    s3ObjectSummary.setSize(fileSizeInBytes);
    return s3ObjectSummary;
}