com.amazonaws.services.dynamodbv2.model.ScanRequest Java Examples

/**
 * Returns the list of usernames stored in the identity table.
 * 
 * @return list of existing usernames in DynamoDB table
 */
public List<String> listUsers() {
    List<String> users = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(USER_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        users.add(s);
    }

    return users;
}
 

@Test
public void testMissingTotalSegments() {
  thrown.expectMessage("TotalSegments is required with withScanRequestFn()");
  pipeline.apply(
      DynamoDBIO.read()
          .withScanRequestFn(
              (SerializableFunction<Void, ScanRequest>) input -> new ScanRequest(tableName))
          .withAwsClientsProvider(
              AwsClientsProviderMock.of(DynamoDBIOTestHelper.getDynamoDBClient())));
  try {
    pipeline.run().waitUntilFinish();
    fail("TotalSegments is required with withScanRequestFn()");
  } catch (IllegalArgumentException ex) {
    assertEquals("TotalSegments is required with withScanRequestFn()", ex.getMessage());
  }
}
 

private void dumpTables() {
    for (String table : client.listTables().getTableNames()) {
        ScanResult scanResult;
        Map<String, AttributeValue> lastKey = null;
        do {
            scanResult = client.scan(new ScanRequest().withTableName(table).withExclusiveStartKey(lastKey));
            lastKey = scanResult.getLastEvaluatedKey();
            for (Map<String, AttributeValue> map : scanResult.getItems()) {
                for (Map.Entry<String, AttributeValue> item : map.entrySet()) {
                    System.out.print("item.put(\"");
                    System.out.print(item.getKey());
                    System.out.print("\", b642Av(\"");
                    System.out.print(Base64.encodeAsString(AttributeValueMarshaller.marshall(item.getValue()).array()));
                    System.out.println("\"));");
                }
                System.out.print("ddb.putItem(new PutItemRequest(\"");
                System.out.print(table);
                System.out.println("\", item));");
                System.out.println("item.clear();");
                System.out.println();
            }
        } while (lastKey != null);

    }
}
 

@Test
public void testNegativeTotalSegments() {
  thrown.expectMessage("TotalSegments is required with withScanRequestFn() and greater zero");
  pipeline.apply(
      DynamoDBIO.read()
          .withScanRequestFn(
              (SerializableFunction<Void, ScanRequest>)
                  input -> new ScanRequest(tableName).withTotalSegments(-1))
          .withAwsClientsProvider(
              AwsClientsProviderMock.of(DynamoDBIOTestHelper.getDynamoDBClient())));
  try {
    pipeline.run().waitUntilFinish();
    fail("withTotalSegments() is expected and greater than zero");
  } catch (IllegalArgumentException ex) {
    assertEquals(
        "TotalSegments is required with withScanRequestFn() and greater zero", ex.getMessage());
  }
}
 

@Override
public PCollection<T> expand(PBegin input) {
  checkArgument((getScanRequestFn() != null), "withScanRequestFn() is required");
  checkArgument((getAwsClientsProvider() != null), "withAwsClientsProvider() is required");
  ScanRequest scanRequest = getScanRequestFn().apply(null);
  checkArgument(
      (scanRequest.getTotalSegments() != null && scanRequest.getTotalSegments() > 0),
      "TotalSegments is required with withScanRequestFn() and greater zero");

  PCollection<Read<T>> splits =
      (PCollection<Read<T>>)
          input.apply("Create", Create.of(this)).apply("Split", ParDo.of(new SplitFn()));
  splits.setCoder(SerializableCoder.of(new TypeDescriptor<Read<T>>() {}));

  PCollection<T> output =
      (PCollection<T>)
          splits
              .apply("Reshuffle", Reshuffle.viaRandomKey())
              .apply("Read", ParDo.of(new ReadFn()));
  output.setCoder(getCoder());
  return output;
}
 

/**
 * Returns the list of usernames stored in the identity table.
 * 
 * @return list of existing usernames in DynamoDB table
 */
public List<String> listUsers() {
    List<String> users = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(USER_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        users.add(s);
    }

    return users;
}
 

private ScanResult getResults(
    final String tableName,
    final short adapterId,
    final List<GeoWaveRow> resultList,
    final Map<String, AttributeValue> lastEvaluatedKey) {
  final ScanRequest request = new ScanRequest(tableName);
  if ((lastEvaluatedKey != null) && !lastEvaluatedKey.isEmpty()) {
    request.setExclusiveStartKey(lastEvaluatedKey);
  }
  final ScanResult result = client.scan(request);
  result.getItems().forEach(objMap -> {
    final byte[] dataId = objMap.get(DynamoDBRow.GW_PARTITION_ID_KEY).getB().array();
    final AttributeValue valueAttr = objMap.get(DynamoDBRow.GW_VALUE_KEY);
    final byte[] value = valueAttr == null ? null : valueAttr.getB().array();
    final AttributeValue visAttr = objMap.get(DynamoDBRow.GW_VISIBILITY_KEY);
    final byte[] vis = visAttr == null ? new byte[0] : visAttr.getB().array();

    resultList.add(DataIndexUtils.deserializeDataIndexRow(dataId, adapterId, value, vis));
  });
  return result;
}
 

/**
 * @return the list of device ID (UID) stored in the identity table.
 */
public List<String> listDevices() {
    List<String> devices = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(DEVICE_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        devices.add(s);
    }

    return devices;
}
 

/**
 * This function copies a scan request for the number of segments and then
 * adds those workers to the executor service to begin scanning.
 * 
 * @param totalSections
 * @param section
 * 
 * @return <ParallelScanExecutor> the parallel scan executor to grab results
 *         when a segment is finished.
 */
public ParallelScanExecutor getParallelScanCompletionService(
        ScanRequest initialRequest, int numSegments, Executor executor,
        int section, int totalSections) {
    final int segments = Math.max(1, numSegments);
    final ParallelScanExecutor completion = new ParallelScanExecutor(
            executor, segments);

    int sectionSize = segments / totalSections;
    int start = sectionSize * section;
    int end = start + sectionSize;
    if (section + 1 == totalSections) {
        end = segments;
    }

    for (int segment = start; segment < end; segment++) {
        ScanRequest scanSegment = copyScanRequest(initialRequest)
                .withTotalSegments(segments).withSegment(segment);
        completion.addWorker(new ScanSegmentWorker(this.client,
                this.rateLimiter, scanSegment), segment);
    }

    return completion;
}
 

/**
 * @return the list of device ID (UID) stored in the identity table.
 */
public List<String> listDevices() {
    List<String> devices = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(DEVICE_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        devices.add(s);
    }

    return devices;
}
 

private static void findProductsForPriceLessThanZero() {

        Map<String, AttributeValue> expressionAttributeValues = new HashMap<String, AttributeValue>();
        expressionAttributeValues.put(":pr", new AttributeValue().withN("100"));

        ScanRequest scanRequest = new ScanRequest().withTableName(tableName).withFilterExpression("Price < :pr")
            .withExpressionAttributeValues(expressionAttributeValues)
            .withProjectionExpression("Id, Title, ProductCategory, Price");

        ScanResult result = client.scan(scanRequest);

        System.out.println("Scan of " + tableName + " for items with a price less than 100.");
        for (Map<String, AttributeValue> item : result.getItems()) {
            System.out.println("");
            printItem(item);
        }
    }
 

/**
 * Begins to pipe the log results by parallel scanning the table and the
 * consumer writing the results.
 */
public void pipe(final AbstractLogConsumer consumer)
        throws ExecutionException, InterruptedException {
    final DynamoDBTableScan scanner = new DynamoDBTableScan(rateLimit,
            client);

    final ScanRequest request = new ScanRequest().withTableName(tableName)
            .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL)
            .withLimit(BootstrapConstants.SCAN_LIMIT)
            .withConsistentRead(consistentScan);

    final ParallelScanExecutor scanService = scanner
            .getParallelScanCompletionService(request, numSegments,
                    threadPool, section, totalSections);

    while (!scanService.finished()) {
        SegmentedScanResult result = scanService.grab();
        consumer.writeResult(result);
    }

    shutdown(true);
    consumer.shutdown(true);
}
 

public RetryResult<ScanResult> scanTable(
    String tableName, DynamoDBQueryFilter dynamoDBQueryFilter, Integer segment, Integer
    totalSegments, Map<String, AttributeValue> exclusiveStartKey, long limit, Reporter reporter) {
  final ScanRequest scanRequest = new ScanRequest(tableName)
      .withExclusiveStartKey(exclusiveStartKey)
      .withLimit(Ints.checkedCast(limit))
      .withSegment(segment)
      .withTotalSegments(totalSegments)
      .withReturnConsumedCapacity(ReturnConsumedCapacity.TOTAL);

  if (dynamoDBQueryFilter != null) {
    Map<String, Condition> scanFilter = dynamoDBQueryFilter.getScanFilter();
    if (!scanFilter.isEmpty()) {
      scanRequest.setScanFilter(scanFilter);
    }
  }

  RetryResult<ScanResult> retryResult = getRetryDriver().runWithRetry(new Callable<ScanResult>() {
    @Override
    public ScanResult call() {
      log.debug("Executing DynamoDB scan: " + scanRequest);
      return dynamoDB.scan(scanRequest);
    }
  }, reporter, PrintCounter.DynamoDBReadThrottle);
  return retryResult;
}
 

/**
 * @return the list of device ID (UID) stored in the identity table.
 */
public List<String> listDevices() {
    List<String> devices = new ArrayList<String>(1000);

    ScanResult result = ddb.scan(new ScanRequest().withTableName(DEVICE_TABLE).withLimit(1000));

    for (Map<String, AttributeValue> item : result.getItems()) {
        String s = "";

        for (Entry<String, AttributeValue> entry : item.entrySet()) {
            s += " ** " + entry.getKey() + " = " + entry.getValue().getS();
        }

        devices.add(s);
    }

    return devices;
}
 

public ScanResult scan(final ScanRequest request, final int permitsToConsume) throws BackendException {
    setUserAgent(request);
    ScanResult result;
    timedReadThrottle(SCAN, request.getTableName(), permitsToConsume);

    final Timer.Context apiTimerContext = getTimerContext(SCAN, request.getTableName());
    try {
        result = client.scan(request);
    } catch (Exception e) {
        throw processDynamoDbApiException(e, SCAN, request.getTableName());
    } finally {
        apiTimerContext.stop();
    }
    meterConsumedCapacity(SCAN, result.getConsumedCapacity());
    measureItemCount(SCAN, request.getTableName(), result.getCount());
    return result;
}
 

private static void findProductsForPriceLessThanZero() {
    
    Map<String, AttributeValue> expressionAttributeValues = new HashMap<String, AttributeValue>();
    expressionAttributeValues.put(":pr", new AttributeValue().withN("100"));
    
    ScanRequest scanRequest = new ScanRequest()
        .withTableName(tableName)
        .withFilterExpression("Price < :pr")
        .withExpressionAttributeValues(expressionAttributeValues)
        .withProjectionExpression("Id, Title, ProductCategory, Price");

    ScanResult result = client.scan(scanRequest);
    
    System.out.println("Scan of " + tableName + " for items with a price less than 100.");
    for (Map<String, AttributeValue> item : result.getItems()) {
        System.out.println("");
        printItem(item);
    }
}
 

/**
 * This method gets a segmentedScanResult and submits the next scan request for that segment, if there is one.
 * @return the next available ScanResult
 * @throws ExecutionException if one of the segment pages threw while executing
 * @throws InterruptedException if one of the segment pages was interrupted while executing.
 */
private ScanContext grab() throws ExecutionException, InterruptedException {
    final Future<ScanContext> ret = exec.take();

    final ScanRequest originalRequest = ret.get().getScanRequest();
    final int segment = originalRequest.getSegment();

    final ScanSegmentWorker sw = workers[segment];

    if (sw.hasNext()) {
        currentFutures[segment] = exec.submit(sw);
    } else {
        finishSegment(segment);
        currentFutures[segment] = null;
    }

    //FYI, This might block if nothing is available.
    return ret.get();
}
 

@Override
public KeyIterator getKeys(final SliceQuery query, final StoreTransaction txh) throws BackendException {
    log.debug("Entering getKeys table:{} query:{} txh:{}", getTableName(), encodeForLog(query), txh);

    final ScanRequest scanRequest = super.createScanRequest();

    final Scanner scanner;
    if (client.isEnableParallelScan()) {
        scanner = client.getDelegate().getParallelScanCompletionService(scanRequest);
    } else {
        scanner = new SequentialScanner(client.getDelegate(), scanRequest);
    }
    // Because SINGLE records cannot be split across scan results, we can use the same interpreter for both
    // sequential and parallel scans.
    final KeyIterator result = new ScanBackedKeyIterator(scanner, new SingleRowScanInterpreter(query));

    log.debug("Exiting getKeys table:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(query), txh, result);
    return result;
}
 

@Override
public KeyIterator getKeys(final SliceQuery query, final StoreTransaction txh) throws BackendException {
    log.debug("Entering getKeys table:{} query:{} txh:{}", getTableName(), encodeForLog(query), txh);
    final Expression filterExpression = new FilterExpressionBuilder().rangeKey()
                                                                     .range(query)
                                                                     .build();

    final ScanRequest scanRequest = super.createScanRequest()
             .withFilterExpression(filterExpression.getConditionExpression())
             .withExpressionAttributeValues(filterExpression.getAttributeValues());

    final Scanner scanner;
    final ScanContextInterpreter interpreter;
    if (client.isEnableParallelScan()) {
        scanner = client.getDelegate().getParallelScanCompletionService(scanRequest);
        interpreter = new MultiRowParallelScanInterpreter(this, query);
    } else {
        scanner = new SequentialScanner(client.getDelegate(), scanRequest);
        interpreter = new MultiRowSequentialScanInterpreter(this, query);
    }

    final KeyIterator result = new ScanBackedKeyIterator(scanner, interpreter);
    log.debug("Exiting getKeys table:{} query:{} txh:{} returning:{}", getTableName(), encodeForLog(query), txh, result);
    return result;
}
 

@Override
public ScanResult scan(String tableName, Map<String, Condition> scanFilter)
        throws AmazonServiceException, AmazonClientException {
    ScanRequest request = new ScanRequest()
            .withTableName(tableName)
            .withScanFilter(scanFilter);
    return scan(request);
}
 

private void testScanContainsItem(
        final TransactionManagerDynamoDBFacade facade,
        final Map<String, AttributeValue> item,
        final boolean filterAttributes) {
    ScanRequest scanRequest = new ScanRequest()
            .withTableName(INTEG_HASH_TABLE_NAME);
    if (filterAttributes) {
        scanRequest.setAttributesToGet(attributesToGet);
    }
    ScanResult scanResult = facade.scan(scanRequest);
    assertEquals(1, scanResult.getItems().size());
    assertContainsNoTransactionAttributes(scanResult.getItems().get(0));
    assertEquals(item, scanResult.getItems().get(0));
}
 

public void sweepForStuckAndOldTransactions() {
    print("\n*** sweepForStuckAndOldTransactions() ***\n");
    
    // The scan should be done in a loop to follow the LastEvaluatedKey, and done with following the best practices for scanning a table.
    // This includes sleeping between pages, using Limit to limit the throughput of each operation to avoid hotspots,
    // and using parallel scan.
    print("Scanning one full page of the transactions table");
    ScanResult result = dynamodb.scan(new ScanRequest()
        .withTableName(TX_TABLE_NAME));
    
    // Pick some duration where transactions should be rolled back if they were sitting there PENDING.
    // 
    //long rollbackAfterDurationMills = 5 * 60 * 1000; // Must be idle and PENDING for 5 minutes to be rolled back
    //long deleteAfterDurationMillis = 24 * 60 * 60 * 1000; // Must be completed for 24 hours to be deleted
    long rollbackAfterDurationMills = 1;
    long deleteAfterDurationMillis = 1;
    for(Map<String, AttributeValue> txItem : result.getItems()) {
        print("Sweeping transaction " + txItem);
        try {
            if(TransactionManager.isTransactionItem(txItem)) {
                Transaction t = txManager.resumeTransaction(txItem);    
                t.sweep(rollbackAfterDurationMills, deleteAfterDurationMillis);
                print("  - Swept transaction (but it might have been skipped)");
            }
        } catch (TransactionException e) {
            // Log and report an error "unsticking" this transaction, but keep going.
            print("  - Error sweeping transaction " + txItem + " " + e);
        }
    }
    
}
 

@Override
public void run() {
    System.out.println("Scanning " + tableName + " segment " + segment + " out of " + totalSegments + " segments " + itemLimit + " items at a time...");
    Map<String, AttributeValue> exclusiveStartKey = null;
    int totalScannedItemCount = 0;
    int totalScanRequestCount = 0;
    try {
        while(true) {
            ScanRequest scanRequest = new ScanRequest()
                .withTableName(tableName)
                .withLimit(itemLimit)
                .withExclusiveStartKey(exclusiveStartKey)
                .withTotalSegments(totalSegments)
                .withSegment(segment);
            
            ScanResult result = client.scan(scanRequest);
            
            totalScanRequestCount++;
            totalScannedItemCount += result.getScannedCount();
            
            // print items returned from scan request
            processScanResult(segment, result);
            
            exclusiveStartKey = result.getLastEvaluatedKey();
            if (exclusiveStartKey == null) {
                break;
            }
        }
    } catch (AmazonServiceException ase) {
        System.err.println(ase.getMessage());
    } finally {
        System.out.println("Scanned " + totalScannedItemCount + " items from segment " + segment + " out of " + totalSegments + " of " + tableName + " with " + totalScanRequestCount + " scan requests");
    }
}
 

@Override
public ScanResult scan(ScanRequest request) throws AmazonServiceException,
        AmazonClientException {
    Collection<String> attributesToGet = addSpecialAttributes(request.getAttributesToGet());
    request.setAttributesToGet(attributesToGet);
    ScanResult result = txManager.getClient().scan(request);
    List<Map<String,AttributeValue>> items = handleItems(result.getItems(), request.getTableName(), request.getAttributesToGet());
    result.setItems(items);
    return result;
}
 

private ScanResult scan(ScanRequest scanRequest) {
    m_logger.debug("Performing scan() request on table {}", scanRequest.getTableName());
    
    Timer timer = new Timer();
    boolean bSuccess = false;
    ScanResult scanResult = null;
    for (int attempts = 1; !bSuccess; attempts++) {
        try {
            scanResult = m_ddbClient.scan(scanRequest);
            if (attempts > 1) {
                m_logger.info("scan() succeeded on attempt #{}", attempts);
            }
            bSuccess = true;
            m_logger.debug("Time to scan table {}: {}", scanRequest.getTableName(), timer.toString());
        } catch (ProvisionedThroughputExceededException e) {
            if (attempts >= m_max_read_attempts) {
                String errMsg = "All retries exceeded; abandoning scan() for table: " + scanRequest.getTableName();
                m_logger.error(errMsg, e);
                throw new RuntimeException(errMsg, e);
            }
            m_logger.warn("scan() attempt #{} failed: {}", attempts, e);
            try {
                Thread.sleep(attempts * m_retry_wait_millis);
            } catch (InterruptedException ex2) {
                // ignore
            }
        }
    }
    return scanResult;
}
 

/**
 * Test the parallel executor completion service with multiple segments and
 * make sure it creates the correct number of segments
 */
@Test
public void testGetParallelExecutorCompletionServiceWithVariousNumberOfSegments()
        throws Exception {
    int segments = 0;
    ExecutorService mockExec = createMock(ExecutorService.class);
    mockStatic(RateLimiter.class);
    AmazonDynamoDBClient mockClient = createMock(AmazonDynamoDBClient.class);
    RateLimiter mockRateLimiter = createMock(RateLimiter.class);
    expect(RateLimiter.create(rateLimit)).andReturn(mockRateLimiter);

    replay(RateLimiter.class);
    DynamoDBTableScan scanner = new DynamoDBTableScan(rateLimit, mockClient);
    ParallelScanExecutor mockScanExecutor = createMock(ParallelScanExecutor.class);
    ScanSegmentWorker mockSegmentWorker = createMock(ScanSegmentWorker.class);

    expectNew(ScanSegmentWorker.class, mockClient, mockRateLimiter, req)
            .andReturn(mockSegmentWorker);
    expectNew(ParallelScanExecutor.class, mockExec, 1).andReturn(
            mockScanExecutor);

    mockScanExecutor.addWorker(mockSegmentWorker, 0);

    int segments2 = 3;
    ScanRequest testReq = scanner.copyScanRequest(req).withTotalSegments(
            segments2);
    expectNew(ParallelScanExecutor.class, mockExec, segments2).andReturn(
            mockScanExecutor);
    for (int i = 0; i < segments2; i++) {
        expectNew(ScanSegmentWorker.class, mockClient, mockRateLimiter,
                scanner.copyScanRequest(testReq).withSegment(i)).andReturn(
                mockSegmentWorker);
        mockScanExecutor.addWorker(mockSegmentWorker, i);
    }

    replayAll();
    scanner.getParallelScanCompletionService(req, segments, mockExec, 0, 1);
    scanner.getParallelScanCompletionService(req, segments2, mockExec, 0, 1);
    verifyAll();
}
 

public ScanRequest copyScanRequest(ScanRequest request) {
    return new ScanRequest()
            .withTableName(request.getTableName())
            .withTotalSegments(request.getTotalSegments())
            .withSegment(request.getSegment())
            .withReturnConsumedCapacity(request.getReturnConsumedCapacity())
            .withLimit(request.getLimit())
            .withConsistentRead(request.getConsistentRead());
}
 

ScanSegmentWorker(final AmazonDynamoDBClient client,
        final RateLimiter rateLimiter, ScanRequest request) {
    this.request = request;
    this.client = client;
    this.rateLimiter = rateLimiter;
    this.hasNext = true;
    this.exponentialBackoffTime = BootstrapConstants.INITIAL_RETRY_TIME_MILLISECONDS;
    lastConsumedCapacity = 256;
}
 

public SequentialScanner(final DynamoDbDelegate dynamoDbDelegate, final ScanRequest request) {
    this.dynamoDbDelegate = dynamoDbDelegate;
    Preconditions.checkArgument(request.getExclusiveStartKey() == null || request.getExclusiveStartKey().isEmpty(),
                                "A scan worker should start with a fresh ScanRequest");
    this.request = DynamoDbDelegate.copyScanRequest(request);
    this.lastConsumedCapacity = dynamoDbDelegate.estimateCapacityUnits(DynamoDbDelegate.SCAN, request.getTableName());
    this.currentFuture = dynamoDbDelegate.scanAsync(request, lastConsumedCapacity);
}
 

@Override
public ScanContext call() throws Exception {
    try {
        final ScanRequest originalRequest = DynamoDbDelegate.copyScanRequest(request);
        final ScanResult result = next();
        return new ScanContext(originalRequest, result);
    } catch (BackendRuntimeException e) {
        throw e.getBackendException();
    }
}