Java Code Examples for org.apache.hadoop.hbase.client.Scan#setLoadColumnFamiliesOnDemand()

private void runScanner(Table table, boolean slow) throws Exception {
  long time = System.nanoTime();
  Scan scan = new Scan();
  scan.addColumn(cf_essential, col_name);
  scan.addColumn(cf_joined, col_name);

  SingleColumnValueFilter filter = new SingleColumnValueFilter(
      cf_essential, col_name, CompareOperator.EQUAL, flag_yes);
  filter.setFilterIfMissing(true);
  scan.setFilter(filter);
  scan.setLoadColumnFamiliesOnDemand(!slow);

  ResultScanner result_scanner = table.getScanner(scan);
  Result res;
  long rows_count = 0;
  while ((res = result_scanner.next()) != null) {
    rows_count++;
  }

  double timeSec = (System.nanoTime() - time) / 1000000000.0;
  result_scanner.close();
  LOG.info((slow ? "Slow" : "Joined") + " scanner finished in " + Double.toString(timeSec)
    + " seconds, got " + Long.toString(rows_count/2) + " rows");
}
 

private RegionScannerHolder newRegionScanner(ScanRequest request, ScanResponse.Builder builder)
    throws IOException {
  HRegion region = getRegion(request.getRegion());
  rejectIfInStandByState(region);
  ClientProtos.Scan protoScan = request.getScan();
  boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
  Scan scan = ProtobufUtil.toScan(protoScan);
  // if the request doesn't set this, get the default region setting.
  if (!isLoadingCfsOnDemandSet) {
    scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
  }

  if (!scan.hasFamilies()) {
    // Adding all families to scanner
    for (byte[] family : region.getTableDescriptor().getColumnFamilyNames()) {
      scan.addFamily(family);
    }
  }
  if (region.getCoprocessorHost() != null) {
    // preScannerOpen is not allowed to return a RegionScanner. Only post hook can create a
    // wrapper for the core created RegionScanner
    region.getCoprocessorHost().preScannerOpen(scan);
  }
  RegionScannerImpl coreScanner = region.getScanner(scan);
  Shipper shipper = coreScanner;
  RegionScanner scanner = coreScanner;
  if (region.getCoprocessorHost() != null) {
    scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
  }
  long scannerId = scannerIdGenerator.generateNewScannerId();
  builder.setScannerId(scannerId);
  builder.setMvccReadPoint(scanner.getMvccReadPoint());
  builder.setTtl(scannerLeaseTimeoutPeriod);
  String scannerName = String.valueOf(scannerId);
  return addScanner(scannerName, scanner, shipper, region, scan.isNeedCursorResult());
}
 

@Test(expected = DoNotRetryIOException.class)
public void testWithReverseScan() throws Exception {
  try (Connection con = TEST_UTIL.getConnection(); Admin admin = con.getAdmin()) {
    TableName tableName = TableName.valueOf(name.getMethodName());

    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1"))
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf2"))
        .build();
    admin.createTable(tableDescriptor);

    try (Table table = con.getTable(tableName)) {
      SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("cf1"),
        Bytes.toBytes("col"), CompareOperator.EQUAL, Bytes.toBytes("val"));
      filter.setFilterIfMissing(true);

      // Reverse scan with loading CFs on demand
      Scan scan = new Scan();
      scan.setFilter(filter);
      scan.setReversed(true);
      scan.setLoadColumnFamiliesOnDemand(true);

      try (ResultScanner scanner = table.getScanner(scan)) {
        // DoNotRetryIOException should occur
        scanner.next();
      }
    }
  }
}
 

@Test
public void testReadersAndWriters() throws Exception {
  Configuration conf = util.getConfiguration();
  String timeoutKey = String.format(TIMEOUT_KEY, this.getClass().getSimpleName());
  long maxRuntime = conf.getLong(timeoutKey, DEFAULT_TIMEOUT_MINUTES);
  long serverCount = util.getHBaseClusterInterface().getClusterMetrics()
    .getLiveServerMetrics().size();
  long keysToWrite = serverCount * KEYS_TO_WRITE_PER_SERVER;
  Connection connection = ConnectionFactory.createConnection(conf);
  Table table = connection.getTable(TABLE_NAME);

  // Create multi-threaded writer and start it. We write multiple columns/CFs and verify
  // their integrity, therefore multi-put is necessary.
  MultiThreadedWriter writer =
    new MultiThreadedWriter(dataGen, conf, TABLE_NAME);
  writer.setMultiPut(true);

  LOG.info("Starting writer; the number of keys to write is " + keysToWrite);
  // TODO : Need to see if tag support has to be given here in the integration test suite
  writer.start(1, keysToWrite, WRITER_THREADS);

  // Now, do scans.
  long now = EnvironmentEdgeManager.currentTime();
  long timeLimit = now + (maxRuntime * 60000);
  boolean isWriterDone = false;
  while (now < timeLimit && !isWriterDone) {
    LOG.info("Starting the scan; wrote approximately "
      + dataGen.getTotalNumberOfKeys() + " keys");
    isWriterDone = writer.isDone();
    if (isWriterDone) {
      LOG.info("Scanning full result, writer is done");
    }
    Scan scan = new Scan();
    for (byte[] cf : dataGen.getColumnFamilies()) {
      scan.addFamily(cf);
    }
    scan.setFilter(dataGen.getScanFilter());
    scan.setLoadColumnFamiliesOnDemand(true);
    // The number of keys we can expect from scan - lower bound (before scan).
    // Not a strict lower bound - writer knows nothing about filters, so we report
    // this from generator. Writer might have generated the value but not put it yet.
    long onesGennedBeforeScan = dataGen.getExpectedNumberOfKeys();
    long startTs = EnvironmentEdgeManager.currentTime();
    ResultScanner results = table.getScanner(scan);
    long resultCount = 0;
    Result result = null;
    // Verify and count the results.
    while ((result = results.next()) != null) {
      boolean isOk = writer.verifyResultAgainstDataGenerator(result, true, true);
      Assert.assertTrue("Failed to verify [" + Bytes.toString(result.getRow())+ "]", isOk);
      ++resultCount;
    }
    long timeTaken = EnvironmentEdgeManager.currentTime() - startTs;
    // Verify the result count.
    long onesGennedAfterScan = dataGen.getExpectedNumberOfKeys();
    Assert.assertTrue("Read " + resultCount + " keys when at most " + onesGennedAfterScan
      + " were generated ", onesGennedAfterScan >= resultCount);
    if (isWriterDone) {
      Assert.assertTrue("Read " + resultCount + " keys; the writer is done and "
        + onesGennedAfterScan + " keys were generated", onesGennedAfterScan == resultCount);
    } else if (onesGennedBeforeScan * 0.9 > resultCount) {
      LOG.warn("Read way too few keys (" + resultCount + "/" + onesGennedBeforeScan
        + ") - there might be a problem, or the writer might just be slow");
    }
    LOG.info("Scan took " + timeTaken + "ms");
    if (!isWriterDone) {
      Thread.sleep(WAIT_BETWEEN_SCANS_MS);
      now = EnvironmentEdgeManager.currentTime();
    }
  }
  Assert.assertEquals("There are write failures", 0, writer.getNumWriteFailures());
  Assert.assertTrue("Writer is not done", isWriterDone);
  // Assert.fail("Boom!");
  connection.close();
}
 

public static void setReversed(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.TRUE_BYTES);
    scan.setLoadColumnFamiliesOnDemand(false);
}
 

public static void unsetReversed(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES);
    scan.setLoadColumnFamiliesOnDemand(true);
}