Java Code Examples for com.amazonaws.services.dynamodbv2.model.ScanResult#getLastEvaluatedKey()

The following examples show how to use com.amazonaws.services.dynamodbv2.model.ScanResult#getLastEvaluatedKey() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ScanRecordReadRequest.java From emr-dynamodb-connector with Apache License 2.0

6 votes

@Override
protected PageResults<Map<String, AttributeValue>> fetchPage(RequestLimit lim) {
  // Read from DynamoDB
  RetryResult<ScanResult> retryResult = context.getClient().scanTable(tableName, null, segment,
      context.getSplit().getTotalSegments(), lastEvaluatedKey, lim.items, context.getReporter());

  ScanResult result = retryResult.result;
  int retries = retryResult.retries;

  double consumedCapacityUnits = 0.0;
  if (result.getConsumedCapacity() != null) {
    consumedCapacityUnits = result.getConsumedCapacity().getCapacityUnits();
  }
  return new PageResults<>(result.getItems(), result.getLastEvaluatedKey(), consumedCapacityUnits,
      retries);
}

Example 2

Source File: ScanSegmentWorker.java From dynamodb-janusgraph-storage-backend with Apache License 2.0

6 votes

@SuppressFBWarnings(value = "IT_NO_SUCH_ELEMENT",
    justification = "https://github.com/awslabs/dynamodb-janusgraph-storage-backend/issues/222")
@Override
public ScanResult next() {
    final Scan backoff = new Scan(request, delegate, lastConsumedCapacity);
    ScanResult result = null;
    try {
        result = backoff.runWithBackoff(); //this will be non-null or runWithBackoff throws
    } catch (BackendException e) {
        throw new BackendRuntimeException(e);
    }

    if (result.getConsumedCapacity() != null) {
        lastConsumedCapacity = result.getConsumedCapacity().getCapacityUnits().intValue();
    }

    if (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    return result;
}

Example 3

Source File: TransformerHolisticTests.java From aws-dynamodb-encryption-java with Apache License 2.0

6 votes

private void dumpTables() {
    for (String table : client.listTables().getTableNames()) {
        ScanResult scanResult;
        Map<String, AttributeValue> lastKey = null;
        do {
            scanResult = client.scan(new ScanRequest().withTableName(table).withExclusiveStartKey(lastKey));
            lastKey = scanResult.getLastEvaluatedKey();
            for (Map<String, AttributeValue> map : scanResult.getItems()) {
                for (Map.Entry<String, AttributeValue> item : map.entrySet()) {
                    System.out.print("item.put(\"");
                    System.out.print(item.getKey());
                    System.out.print("\", b642Av(\"");
                    System.out.print(Base64.encodeAsString(AttributeValueMarshaller.marshall(item.getValue()).array()));
                    System.out.println("\"));");
                }
                System.out.print("ddb.putItem(new PutItemRequest(\"");
                System.out.print(table);
                System.out.println("\", item));");
                System.out.println("item.clear();");
                System.out.println();
            }
        } while (lastKey != null);

    }
}

Example 4

Source File: DynamoDBOperations.java From geowave with Apache License 2.0

6 votes

public Iterator<GeoWaveRow> getRowsFromDataIndex(final short adapterId, final String typeName) {
  final List<GeoWaveRow> resultList = new ArrayList<>();
  // fill result list
  ScanResult result =
      getResults(
          typeName + "_" + getQualifiedTableName(DataIndexUtils.DATA_ID_INDEX.getName()),
          adapterId,
          resultList,
          null);
  while ((result.getLastEvaluatedKey() != null) && !result.getLastEvaluatedKey().isEmpty()) {
    result =
        getResults(
            typeName + "_" + getQualifiedTableName(DataIndexUtils.DATA_ID_INDEX.getName()),
            adapterId,
            resultList,
            result.getLastEvaluatedKey());
  }
  return resultList.iterator();
}

Example 5

Source File: GenericDynamoDB.java From strongbox with Apache License 2.0

5 votes

private Stream<Entry> scan(SecretEventStream.Filter<Entry> filter, Converters converters) {
    ScanRequest scanRequest = new ScanRequest();
    scanRequest.withConsistentRead(true);
    scanRequest.withTableName(tableName);

    FilterGenerator filterGenerator = new FilterGenerator();
    FilterGenerator.Filter generated = filterGenerator.process(filter.parsedAttributeCondition.get(), converters);

    if(!generated.expressionAttributeNames.isEmpty()) {
        scanRequest.withExpressionAttributeNames(generated.expressionAttributeNames);
    }

    if (!generated.expressionAttributeValues.isEmpty()) {
        scanRequest.withExpressionAttributeValues(generated.expressionAttributeValues);
    }

    scanRequest.withFilterExpression(generated.filterExpression);

    ScanResult result = client.scan(scanRequest);

    List<Map<String, AttributeValue>> results = new ArrayList<>();
    results.addAll(result.getItems());

    while (result.getLastEvaluatedKey() != null) {
        scanRequest = scanRequest.withExclusiveStartKey(result.getLastEvaluatedKey());

        result = client.scan(scanRequest);

        results.addAll(result.getItems());
    }

    Stream<Entry> typedResult = results.stream().map(this::fromMap);

    if (filter.reverse) {
        typedResult = Lists.reverse(typedResult.collect(Collectors.toCollection(LinkedList::new))).stream();
    }

    return typedResult;
}

Example 6

Source File: LowLevelParallelScan.java From aws-doc-sdk-examples with Apache License 2.0

5 votes

@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments
        + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while (true) {
            ScanRequest scanRequest = new ScanRequest().withTableName(tableName).withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey).withTotalSegments(totalSegments).withSegment(segment);

            ScanResult result = client.scan(scanRequest);

            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();

            // print items returned from scan request
            processScanResult(segment, result);

            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    }
    catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    }
    finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of "
            + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}

Example 7

Source File: ScanSegmentWorker.java From dynamodb-import-export-tool with Apache License 2.0

5 votes

@Override
public SegmentedScanResult call() {
    ScanResult result = null;
    result = runWithBackoff();

    final ConsumedCapacity cc = result.getConsumedCapacity();

    if (cc != null && cc.getCapacityUnits() != null) {
        lastConsumedCapacity = result.getConsumedCapacity()
                .getCapacityUnits().intValue();
    } else if (result.getScannedCount() != null && result.getCount() != null) {

        final boolean isConsistent = request.getConsistentRead();
        int itemSize = isConsistent ? BootstrapConstants.STRONGLY_CONSISTENT_READ_ITEM_SIZE
                : BootstrapConstants.EVENTUALLY_CONSISTENT_READ_ITEM_SIZE;

        lastConsumedCapacity = (result.getScannedCount() / (int) Math.max(1.0, result.getCount()))
                * (ItemSizeCalculator.calculateScanResultSizeInBytes(result) / itemSize);
    }

    if (result.getLastEvaluatedKey() != null
            && !result.getLastEvaluatedKey().isEmpty()) {
        hasNext = true;
        request.setExclusiveStartKey(result.getLastEvaluatedKey());
    } else {
        hasNext = false;
    }

    if (lastConsumedCapacity > 0) {
        rateLimiter.acquire(lastConsumedCapacity);
    }
    return new SegmentedScanResult(result, request.getSegment());
}

Example 8

Source File: VerifyIndex.java From aws-big-data-blog with Apache License 2.0

5 votes

private void fetchIndexList() {
   final ScanRequest req = new ScanRequest("s3-index-example.mikedeck-index").withAttributesToGet("Key");
   ScanResult result = DDB.scan(req);
   addIndexItems(result.getItems());
   while (result.getLastEvaluatedKey() != null && !result.getLastEvaluatedKey().isEmpty()) {
      req.setExclusiveStartKey(result.getLastEvaluatedKey());
      result = DDB.scan(req);
      addIndexItems(result.getItems());
   }
}

Example 9

Source File: DynamoDBService.java From Doradus with Apache License 2.0

5 votes

@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
    String tableName = storeToTableName(storeName);
    ScanRequest scanRequest = new ScanRequest(tableName);
    scanRequest.setAttributesToGet(Arrays.asList(ROW_KEY_ATTR_NAME)); // attributes to get
    if (continuationToken != null) {
        scanRequest.setExclusiveStartKey(makeDDBKey(continuationToken));
    }
    List<String> rowKeys = new ArrayList<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) {
            break;
        }
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get(ROW_KEY_ATTR_NAME);
            rowKeys.add(rowAttr.getS());
            if (rowKeys.size() >= count) {
                break;
            }
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) {
            break;
        }
        scanRequest.setExclusiveStartKey(lastEvaluatedKey);
    }
    return rowKeys;
}

Example 10

Source File: DynamoDBService2.java From Doradus with Apache License 2.0

5 votes

@Override
public List<String> getRows(String storeName, String continuationToken, int count) {
	Timer t = new Timer();
	ScanRequest scanRequest = new ScanRequest(getTenant().getName());
    scanRequest.setAttributesToGet(Arrays.asList("key")); // attributes to get
    //if (continuationToken != null) {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_" + continuationToken, "\u007F"));
    //} else {
    //	scanRequest.setExclusiveStartKey(getPrimaryKey(storeName + "_", "\0"));
    //}

    Set<String> rowKeys = new HashSet<>();
    while (rowKeys.size() < count) {
        ScanResult scanResult = m_client.scan(scanRequest);
        List<Map<String, AttributeValue>> itemList = scanResult.getItems();
        if (itemList.size() == 0) break;
        for (Map<String, AttributeValue> attributeMap : itemList) {
            AttributeValue rowAttr = attributeMap.get("key");
            if(!rowAttr.getS().startsWith(storeName)) continue;
            String name = rowAttr.getS().substring(storeName.length() + 1);
            if(continuationToken != null && continuationToken.compareTo(name) >= 0) continue;
            rowKeys.add(name);
        }
        Map<String, AttributeValue> lastEvaluatedKey = scanResult.getLastEvaluatedKey();
        if (lastEvaluatedKey == null) break;
    	scanRequest.setExclusiveStartKey(getPrimaryKey(lastEvaluatedKey.get("key").getS(), "\u007F"));
    }
    List<String> list = new ArrayList<>(rowKeys);
    Collections.sort(list);
	m_logger.debug("get rows in {} in {}", storeName, t);
    return list;
}

Example 11

Source File: LowLevelParallelScan.java From aws-dynamodb-examples with Apache License 2.0

5 votes

@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while(true) {
            ScanRequest scanRequest = new ScanRequest()
                .withTableName(tableName)
                .withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey)
                .withTotalSegments(totalSegments)
                .withSegment(segment);
            
            ScanResult result = client.scan(scanRequest);
            
            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();
            
            // print items returned from scan request
            processScanResult(segment, result);
            
            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    } catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    } finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of " + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}

Example 12

Source File: GenericDynamoDB.java From strongbox with Apache License 2.0

4 votes

private Stream<Entry> all() {
    ScanRequest scanRequest = new ScanRequest();
    scanRequest.withConsistentRead(true);
    scanRequest.withTableName(tableName);

    ScanResult result = client.scan(scanRequest);

    List<Map<String, AttributeValue>> results = new ArrayList<>();
    results.addAll(result.getItems());

    while (result.getLastEvaluatedKey() != null) {
        scanRequest = scanRequest.withExclusiveStartKey(result.getLastEvaluatedKey());

        result = client.scan(scanRequest);

        results.addAll(result.getItems());
    }


    return results.stream().map(this::fromMap);
}