com.amazonaws.services.dynamodbv2.document.TableKeysAndAttributes Java Examples

The following examples show how to use com.amazonaws.services.dynamodbv2.document.TableKeysAndAttributes. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

5 votes

@Before
public void setUp() {
    outcome = new BatchGetItemOutcome(result);
    KeysAndAttributes kaa = new KeysAndAttributes();
    Map<String,AttributeValue> map = new HashMap<>();
    map.put("hashS", new AttributeValue("h1"));
    map.put("rangeS", new AttributeValue("r1"));
    kaa.withKeys(map);
    unprocessed = new HashMap<>();
    unprocessed.put(stringHashStringRangeTableName, kaa);

    result.withUnprocessedKeys(unprocessed);

    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    responses.put("StringHashStringRangeTable", items);
    result.withResponses(responses);

    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return outcome;
        }

    };

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

}

Example #2

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetOnlyHashFailure() {
    // Inject a mock DynamoDB to create the exception condition
    final DynamoDB mockDynamoDb = Mockito.mock(DynamoDB.class);
    // When writing, mock thrown service exception from AWS
    Mockito.when(mockDynamoDb.batchGetItem(ArgumentMatchers.<TableKeysAndAttributes>any())).thenThrow(getSampleAwsServiceException());

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDb;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);

    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        ITAbstractDynamoDBTest.validateServiceExceptionAttribute(flowFile);
    }

}

Example #3

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetThrowsServiceException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            throw new AmazonServiceException("serviceException");
        }

    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("serviceException (Service: null; Status Code: 0; Error Code: null; Request ID: null)", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}

Example #4

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetThrowsRuntimeException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            throw new RuntimeException("runtimeException");
        }

    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("runtimeException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}

Example #5

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetThrowsClientException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            throw new AmazonClientException("clientException");
        }

    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("clientException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}

Example #6

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetOnlyHashFailure() {
    // Inject a mock DynamoDB to create the exception condition
    final DynamoDB mockDynamoDb = Mockito.mock(DynamoDB.class);
    // When writing, mock thrown service exception from AWS
    Mockito.when(mockDynamoDb.batchGetItem(Matchers.<TableKeysAndAttributes>anyVararg())).thenThrow(getSampleAwsServiceException());

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDb;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);

    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        ITAbstractDynamoDBTest.validateServiceExceptionAttribute(flowFile);
    }

}

Example #7

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetThrowsClientException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            throw new AmazonClientException("clientException");
        }

    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("clientException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}

Example #8

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

5 votes

@Before
public void setUp() {
    outcome = new BatchGetItemOutcome(result);
    KeysAndAttributes kaa = new KeysAndAttributes();
    Map<String,AttributeValue> map = new HashMap<>();
    map.put("hashS", new AttributeValue("h1"));
    map.put("rangeS", new AttributeValue("r1"));
    kaa.withKeys(map);
    unprocessed = new HashMap<>();
    unprocessed.put(stringHashStringRangeTableName, kaa);

    result.withUnprocessedKeys(unprocessed);

    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    responses.put("StringHashStringRangeTable", items);
    result.withResponses(responses);

    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return outcome;
        }

    };

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

}

Example #9

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetThrowsRuntimeException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            throw new RuntimeException("runtimeException");
        }

    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("runtimeException", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}

Example #10

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

5 votes

@Test
public void testStringHashStringRangeGetThrowsServiceException() {
    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            throw new AmazonServiceException("serviceException");
        }

    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };

    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_FAILURE, 1);
    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_FAILURE);
    for (MockFlowFile flowFile : flowFiles) {
        assertEquals("serviceException (Service: null; Status Code: 0; Error Code: null; Request ID: null)", flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_ERROR_EXCEPTION_MESSAGE));
    }

}

Example #11

Source File: DocumentAPIBatchGet.java From aws-dynamodb-examples with Apache License 2.0

4 votes

private static void retrieveMultipleItemsBatchGet() {        
    
    try {
        
        TableKeysAndAttributes forumTableKeysAndAttributes = new TableKeysAndAttributes(forumTableName);
        forumTableKeysAndAttributes.addHashOnlyPrimaryKeys("Name", "Amazon S3", "Amazon DynamoDB");
        
        TableKeysAndAttributes threadTableKeysAndAttributes = new TableKeysAndAttributes(threadTableName);
        threadTableKeysAndAttributes.addHashAndRangePrimaryKeys("ForumName", "Subject", 
            "Amazon DynamoDB","DynamoDB Thread 1",
            "Amazon DynamoDB","DynamoDB Thread 2",
            "Amazon S3","S3 Thread 1");
        
         Map<String, TableKeysAndAttributes> requestItems = new HashMap<String, TableKeysAndAttributes>();
         requestItems.put(forumTableName, forumTableKeysAndAttributes);
         requestItems.put(threadTableName, threadTableKeysAndAttributes);
         
         System.out.println("Making the request.");

         BatchGetItemOutcome outcome = dynamoDB.batchGetItem(forumTableKeysAndAttributes,
                        threadTableKeysAndAttributes);

        do {

            for (String tableName : outcome.getTableItems().keySet()) {
                System.out.println("Items in table " + tableName);
                List<Item> items = outcome.getTableItems().get(tableName);
                for (Item item : items) {
                    System.out.println(item.toJSONPretty());
                }
            }
            
            // Check for unprocessed keys which could happen if you exceed provisioned
            // throughput or reach the limit on response size.
            
            Map<String, KeysAndAttributes> unprocessedKeys = outcome.getUnprocessedKeys();
        
            if (outcome.getUnprocessedKeys().size() == 0) {
                System.out.println("No unprocessed keys found");
            } else {
                System.out.println("Retrieving the unprocessed keys");
                outcome = dynamoDB.batchGetItemUnprocessed(unprocessedKeys);
            }
            
        } while (outcome.getUnprocessedKeys().size() > 0);
 
 
    }  catch (Exception e) {
        System.err.println("Failed to retrieve items.");
        System.err.println(e.getMessage());
    }  

}

Example #12

Source File: DynamoSpaceConfigClient.java From xyz-hub with Apache License 2.0

4 votes

@Override
public void getSelectedSpaces(Marker marker, SpaceAuthorizationCondition authorizedCondition, SpaceSelectionCondition selectedCondition,
    Handler<AsyncResult<List<Space>>> handler) {
  logger.info(marker, "Getting selected spaces");

  if (authorizedCondition == null || selectedCondition == null) {
    throw new NullPointerException("authorizedCondition and selectedCondition are required");
  }

  final List<Space> result = new ArrayList<>();
  logger.debug(marker, "authorizedCondition: spaceIds: {}, ownerIds {}, packages: {}", authorizedCondition.spaceIds, authorizedCondition.ownerIds, authorizedCondition.packages);
  logger.debug(marker, "selectedCondition: spaceIds: {}, ownerIds {}, packages: {}, shared: {}, negateOwnerIds: {}", selectedCondition.spaceIds, selectedCondition.ownerIds, selectedCondition.packages, selectedCondition.shared, selectedCondition.negateOwnerIds);

  try {
    final Set<String> authorizedSpaces = getAuthorizedSpaces(marker, authorizedCondition);

    // get all shared spaces if the selection for shared spaces is enabled
    if (selectedCondition.shared) {
      spaces.getIndex("shared-index").query(new QuerySpec().withHashKey("shared", 1).withProjectionExpression("id")).pages()
          .forEach(p -> p.forEach(i -> {
            authorizedSpaces.add(i.getString("id"));
          }));
      logger.debug(marker, "Number of space IDs after addition of shared spaces: {}", authorizedSpaces.size());
    }

    // filter out the ones not present in the selectedCondition (null or empty represents 'do not filter')
    if (!CollectionUtils.isNullOrEmpty(selectedCondition.spaceIds)) {
      authorizedSpaces.removeIf(i -> !selectedCondition.spaceIds.contains(i));
      logger.debug(marker, "Number of space IDs after removal of the ones not selected by ID: {}", authorizedSpaces.size());
    }

    // now filter all spaceIds with the ones being selected in the selectedCondition (by checking the space's ownership) (
    if (!CollectionUtils.isNullOrEmpty(selectedCondition.ownerIds)) {
      final Set<String> ownersSpaces = new HashSet<>();
      selectedCondition.ownerIds.forEach(o ->
          spaces.getIndex("owner-index").query(new QuerySpec().withHashKey("owner", o).withProjectionExpression("id")).pages()
              .forEach(p -> p.forEach(i -> ownersSpaces.add(i.getString("id")))));

      // HINT: A ^ TRUE == !A (negateOwnerIds: keep or remove the spaces contained in the owner's spaces list)
      authorizedSpaces.removeIf(i -> !selectedCondition.negateOwnerIds ^ ownersSpaces.contains(i));
      logger.debug(marker, "Number of space IDs after removal of the ones not selected by owner: {}", authorizedSpaces.size());
    }

    // TODO selection per packages is not yet supported: selectedCondition.packages

    logger.info(marker, "Final number of space IDs to be retrieved from DynamoDB: {}", authorizedSpaces.size());
    if (!authorizedSpaces.isEmpty()) {
      int batches = (int) Math.ceil((double) authorizedSpaces.size()/100);
      for (int i=0; i<batches; i++) {
        final TableKeysAndAttributes keys = new TableKeysAndAttributes(dynamoClient.tableName);
        authorizedSpaces.stream().skip(i*100).limit(100).forEach(id -> keys.addHashOnlyPrimaryKey("id", id));

        BatchGetItemOutcome outcome = dynamoClient.db.batchGetItem(keys);
        processOutcome(outcome, result);

        while (!outcome.getUnprocessedKeys().isEmpty()) {
          outcome = dynamoClient.db.batchGetItemUnprocessed(outcome.getUnprocessedKeys());
          processOutcome(outcome, result);
        }
      }
    }

    logger.info(marker, "Number of spaces retrieved from DynamoDB: {}", result.size());
    handler.handle(Future.succeededFuture(result));
  } catch (AmazonDynamoDBException e) {
    logger.error(marker, "Failure getting authorized spaces", e);
    handler.handle(Future.failedFuture(e));
  }
}

Example #13

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

4 votes

@Test
public void testStringHashStringRangeGetNotFound() {
    result.clearResponsesEntries();
    result.clearUnprocessedKeysEntries();

    final BatchGetItemOutcome notFoundOutcome = new BatchGetItemOutcome(result);
    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    responses.put(stringHashStringRangeTableName, items);
    result.withResponses(responses);

    final DynamoDB notFoundMockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return notFoundOutcome;
        }
    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return notFoundMockDynamoDB;
        }
    };
    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(GetDynamoDB.REL_NOT_FOUND, 1);

    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_UNPROCESSED);
    for (MockFlowFile flowFile : flowFiles) {
        assertNotNull(flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND));
    }

}

Example #14

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

4 votes

@Test
public void testStringHashStringRangeGetJsonObjectValid() throws IOException {
    outcome = new BatchGetItemOutcome(result);
    KeysAndAttributes kaa = new KeysAndAttributes();
    Map<String,AttributeValue> map = new HashMap<>();
    map.put("hashS", new AttributeValue("h1"));
    map.put("rangeS", new AttributeValue("r1"));
    kaa.withKeys(map);
    unprocessed = new HashMap<>();
    result.withUnprocessedKeys(unprocessed);

    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    Map<String,AttributeValue> item = new HashMap<String,AttributeValue>();
    String jsonDocument = "{\"name\": \"john\"}";
    item.put("j1",new AttributeValue(jsonDocument));
    item.put("hashS", new AttributeValue("h1"));
    item.put("rangeS", new AttributeValue("r1"));
    items.add(item);
    responses.put("StringHashStringRangeTable", items);
    result.withResponses(responses);

    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return outcome;
        }

    };

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);
    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);
}

Example #15

Source File: GetDynamoDBTest.java From nifi with Apache License 2.0

4 votes

@Test
public void testStringHashStringRangeGetJsonObjectNull() {
    outcome = new BatchGetItemOutcome(result);
    KeysAndAttributes kaa = new KeysAndAttributes();
    Map<String,AttributeValue> map = new HashMap<>();
    map.put("hashS", new AttributeValue("h1"));
    map.put("rangeS", new AttributeValue("r1"));
    kaa.withKeys(map);
    unprocessed = new HashMap<>();
    result.withUnprocessedKeys(unprocessed);

    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    Map<String,AttributeValue> item = new HashMap<String,AttributeValue>();
    item.put("j1",null);
    item.put("hashS", new AttributeValue("h1"));
    item.put("rangeS", new AttributeValue("r1"));
    items.add(item);
    responses.put("StringHashStringRangeTable", items);
    result.withResponses(responses);

    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return outcome;
        }

    };

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
        assertNull(flowFile.getContentClaim());
    }

}

Example #16

Source File: DocumentAPIBatchGet.java From aws-doc-sdk-examples with Apache License 2.0

4 votes

private static void retrieveMultipleItemsBatchGet() {

        try {

            TableKeysAndAttributes forumTableKeysAndAttributes = new TableKeysAndAttributes(forumTableName);
            // Add a partition key
            forumTableKeysAndAttributes.addHashOnlyPrimaryKeys("Name", "Amazon S3", "Amazon DynamoDB");

            TableKeysAndAttributes threadTableKeysAndAttributes = new TableKeysAndAttributes(threadTableName);
            // Add a partition key and a sort key
            threadTableKeysAndAttributes.addHashAndRangePrimaryKeys("ForumName", "Subject", "Amazon DynamoDB",
                "DynamoDB Thread 1", "Amazon DynamoDB", "DynamoDB Thread 2", "Amazon S3", "S3 Thread 1");

            System.out.println("Making the request.");

            BatchGetItemOutcome outcome = dynamoDB.batchGetItem(forumTableKeysAndAttributes,
                threadTableKeysAndAttributes);

            Map<String, KeysAndAttributes> unprocessed = null;

            do {
                for (String tableName : outcome.getTableItems().keySet()) {
                    System.out.println("Items in table " + tableName);
                    List<Item> items = outcome.getTableItems().get(tableName);
                    for (Item item : items) {
                        System.out.println(item.toJSONPretty());
                    }
                }

                // Check for unprocessed keys which could happen if you exceed
                // provisioned
                // throughput or reach the limit on response size.
                unprocessed = outcome.getUnprocessedKeys();

                if (unprocessed.isEmpty()) {
                    System.out.println("No unprocessed keys found");
                }
                else {
                    System.out.println("Retrieving the unprocessed keys");
                    outcome = dynamoDB.batchGetItemUnprocessed(unprocessed);
                }

            } while (!unprocessed.isEmpty());

        }
        catch (Exception e) {
            System.err.println("Failed to retrieve items.");
            System.err.println(e.getMessage());
        }

    }

Example #17

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testStringHashStringRangeGetNotFound() {
    result.clearResponsesEntries();
    result.clearUnprocessedKeysEntries();

    final BatchGetItemOutcome notFoundOutcome = new BatchGetItemOutcome(result);
    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    responses.put(stringHashStringRangeTableName, items);
    result.withResponses(responses);

    final DynamoDB notFoundMockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {
        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return notFoundOutcome;
        }
    };

    final GetDynamoDB getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return notFoundMockDynamoDB;
        }
    };
    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(GetDynamoDB.REL_NOT_FOUND, 1);

    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_UNPROCESSED);
    for (MockFlowFile flowFile : flowFiles) {
        assertNotNull(flowFile.getAttribute(AbstractDynamoDBProcessor.DYNAMODB_KEY_ERROR_NOT_FOUND));
    }

}

Example #18

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testStringHashStringRangeGetJsonObjectValid() throws IOException {
    outcome = new BatchGetItemOutcome(result);
    KeysAndAttributes kaa = new KeysAndAttributes();
    Map<String,AttributeValue> map = new HashMap<>();
    map.put("hashS", new AttributeValue("h1"));
    map.put("rangeS", new AttributeValue("r1"));
    kaa.withKeys(map);
    unprocessed = new HashMap<>();
    result.withUnprocessedKeys(unprocessed);

    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    Map<String,AttributeValue> item = new HashMap<String,AttributeValue>();
    String jsonDocument = "{\"name\": \"john\"}";
    item.put("j1",new AttributeValue(jsonDocument));
    item.put("hashS", new AttributeValue("h1"));
    item.put("rangeS", new AttributeValue("r1"));
    items.add(item);
    responses.put("StringHashStringRangeTable", items);
    result.withResponses(responses);

    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return outcome;
        }

    };

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);
    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);
}

Example #19

Source File: GetDynamoDBTest.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testStringHashStringRangeGetJsonObjectNull() {
    outcome = new BatchGetItemOutcome(result);
    KeysAndAttributes kaa = new KeysAndAttributes();
    Map<String,AttributeValue> map = new HashMap<>();
    map.put("hashS", new AttributeValue("h1"));
    map.put("rangeS", new AttributeValue("r1"));
    kaa.withKeys(map);
    unprocessed = new HashMap<>();
    result.withUnprocessedKeys(unprocessed);

    Map<String,List<Map<String,AttributeValue>>> responses = new HashMap<>();
    List<Map<String,AttributeValue>> items = new ArrayList<>();
    Map<String,AttributeValue> item = new HashMap<String,AttributeValue>();
    item.put("j1",null);
    item.put("hashS", new AttributeValue("h1"));
    item.put("rangeS", new AttributeValue("r1"));
    items.add(item);
    responses.put("StringHashStringRangeTable", items);
    result.withResponses(responses);

    final DynamoDB mockDynamoDB = new DynamoDB(Regions.AP_NORTHEAST_1) {

        @Override
        public BatchGetItemOutcome batchGetItem(TableKeysAndAttributes... tableKeysAndAttributes) {
            return outcome;
        }

    };

    getDynamoDB = new GetDynamoDB() {
        @Override
        protected DynamoDB getDynamoDB() {
            return mockDynamoDB;
        }
    };
    final TestRunner getRunner = TestRunners.newTestRunner(getDynamoDB);

    getRunner.setProperty(AbstractDynamoDBProcessor.ACCESS_KEY,"abcd");
    getRunner.setProperty(AbstractDynamoDBProcessor.SECRET_KEY, "cdef");
    getRunner.setProperty(AbstractDynamoDBProcessor.REGION, REGION);
    getRunner.setProperty(AbstractDynamoDBProcessor.TABLE, stringHashStringRangeTableName);
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_NAME, "rangeS");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_NAME, "hashS");
    getRunner.setProperty(AbstractDynamoDBProcessor.RANGE_KEY_VALUE, "r1");
    getRunner.setProperty(AbstractDynamoDBProcessor.HASH_KEY_VALUE, "h1");
    getRunner.setProperty(AbstractDynamoDBProcessor.JSON_DOCUMENT, "j1");
    getRunner.enqueue(new byte[] {});

    getRunner.run(1);

    getRunner.assertAllFlowFilesTransferred(AbstractDynamoDBProcessor.REL_SUCCESS, 1);

    List<MockFlowFile> flowFiles = getRunner.getFlowFilesForRelationship(AbstractDynamoDBProcessor.REL_SUCCESS);
    for (MockFlowFile flowFile : flowFiles) {
        assertNull(flowFile.getContentClaim());
    }

}