Java Code Examples for org.apache.hadoop.hbase.client.Scan#setLoadColumnFamiliesOnDemand()

The following examples show how to use org.apache.hadoop.hbase.client.Scan#setLoadColumnFamiliesOnDemand() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestJoinedScanners.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void runScanner(Table table, boolean slow) throws Exception {
  long time = System.nanoTime();
  Scan scan = new Scan();
  scan.addColumn(cf_essential, col_name);
  scan.addColumn(cf_joined, col_name);

  SingleColumnValueFilter filter = new SingleColumnValueFilter(
      cf_essential, col_name, CompareOperator.EQUAL, flag_yes);
  filter.setFilterIfMissing(true);
  scan.setFilter(filter);
  scan.setLoadColumnFamiliesOnDemand(!slow);

  ResultScanner result_scanner = table.getScanner(scan);
  Result res;
  long rows_count = 0;
  while ((res = result_scanner.next()) != null) {
    rows_count++;
  }

  double timeSec = (System.nanoTime() - time) / 1000000000.0;
  result_scanner.close();
  LOG.info((slow ? "Slow" : "Joined") + " scanner finished in " + Double.toString(timeSec)
    + " seconds, got " + Long.toString(rows_count/2) + " rows");
}
 
Example 2
Source File: RSRpcServices.java    From hbase with Apache License 2.0 5 votes vote down vote up
private RegionScannerHolder newRegionScanner(ScanRequest request, ScanResponse.Builder builder)
    throws IOException {
  HRegion region = getRegion(request.getRegion());
  rejectIfInStandByState(region);
  ClientProtos.Scan protoScan = request.getScan();
  boolean isLoadingCfsOnDemandSet = protoScan.hasLoadColumnFamiliesOnDemand();
  Scan scan = ProtobufUtil.toScan(protoScan);
  // if the request doesn't set this, get the default region setting.
  if (!isLoadingCfsOnDemandSet) {
    scan.setLoadColumnFamiliesOnDemand(region.isLoadingCfsOnDemandDefault());
  }

  if (!scan.hasFamilies()) {
    // Adding all families to scanner
    for (byte[] family : region.getTableDescriptor().getColumnFamilyNames()) {
      scan.addFamily(family);
    }
  }
  if (region.getCoprocessorHost() != null) {
    // preScannerOpen is not allowed to return a RegionScanner. Only post hook can create a
    // wrapper for the core created RegionScanner
    region.getCoprocessorHost().preScannerOpen(scan);
  }
  RegionScannerImpl coreScanner = region.getScanner(scan);
  Shipper shipper = coreScanner;
  RegionScanner scanner = coreScanner;
  if (region.getCoprocessorHost() != null) {
    scanner = region.getCoprocessorHost().postScannerOpen(scan, scanner);
  }
  long scannerId = scannerIdGenerator.generateNewScannerId();
  builder.setScannerId(scannerId);
  builder.setMvccReadPoint(scanner.getMvccReadPoint());
  builder.setTtl(scannerLeaseTimeoutPeriod);
  String scannerName = String.valueOf(scannerId);
  return addScanner(scannerName, scanner, shipper, region, scan.isNeedCursorResult());
}
 
Example 3
Source File: TestJoinedScanners.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test(expected = DoNotRetryIOException.class)
public void testWithReverseScan() throws Exception {
  try (Connection con = TEST_UTIL.getConnection(); Admin admin = con.getAdmin()) {
    TableName tableName = TableName.valueOf(name.getMethodName());

    TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1"))
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf2"))
        .build();
    admin.createTable(tableDescriptor);

    try (Table table = con.getTable(tableName)) {
      SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("cf1"),
        Bytes.toBytes("col"), CompareOperator.EQUAL, Bytes.toBytes("val"));
      filter.setFilterIfMissing(true);

      // Reverse scan with loading CFs on demand
      Scan scan = new Scan();
      scan.setFilter(filter);
      scan.setReversed(true);
      scan.setLoadColumnFamiliesOnDemand(true);

      try (ResultScanner scanner = table.getScanner(scan)) {
        // DoNotRetryIOException should occur
        scanner.next();
      }
    }
  }
}
 
Example 4
Source File: IntegrationTestLazyCfLoading.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testReadersAndWriters() throws Exception {
  Configuration conf = util.getConfiguration();
  String timeoutKey = String.format(TIMEOUT_KEY, this.getClass().getSimpleName());
  long maxRuntime = conf.getLong(timeoutKey, DEFAULT_TIMEOUT_MINUTES);
  long serverCount = util.getHBaseClusterInterface().getClusterMetrics()
    .getLiveServerMetrics().size();
  long keysToWrite = serverCount * KEYS_TO_WRITE_PER_SERVER;
  Connection connection = ConnectionFactory.createConnection(conf);
  Table table = connection.getTable(TABLE_NAME);

  // Create multi-threaded writer and start it. We write multiple columns/CFs and verify
  // their integrity, therefore multi-put is necessary.
  MultiThreadedWriter writer =
    new MultiThreadedWriter(dataGen, conf, TABLE_NAME);
  writer.setMultiPut(true);

  LOG.info("Starting writer; the number of keys to write is " + keysToWrite);
  // TODO : Need to see if tag support has to be given here in the integration test suite
  writer.start(1, keysToWrite, WRITER_THREADS);

  // Now, do scans.
  long now = EnvironmentEdgeManager.currentTime();
  long timeLimit = now + (maxRuntime * 60000);
  boolean isWriterDone = false;
  while (now < timeLimit && !isWriterDone) {
    LOG.info("Starting the scan; wrote approximately "
      + dataGen.getTotalNumberOfKeys() + " keys");
    isWriterDone = writer.isDone();
    if (isWriterDone) {
      LOG.info("Scanning full result, writer is done");
    }
    Scan scan = new Scan();
    for (byte[] cf : dataGen.getColumnFamilies()) {
      scan.addFamily(cf);
    }
    scan.setFilter(dataGen.getScanFilter());
    scan.setLoadColumnFamiliesOnDemand(true);
    // The number of keys we can expect from scan - lower bound (before scan).
    // Not a strict lower bound - writer knows nothing about filters, so we report
    // this from generator. Writer might have generated the value but not put it yet.
    long onesGennedBeforeScan = dataGen.getExpectedNumberOfKeys();
    long startTs = EnvironmentEdgeManager.currentTime();
    ResultScanner results = table.getScanner(scan);
    long resultCount = 0;
    Result result = null;
    // Verify and count the results.
    while ((result = results.next()) != null) {
      boolean isOk = writer.verifyResultAgainstDataGenerator(result, true, true);
      Assert.assertTrue("Failed to verify [" + Bytes.toString(result.getRow())+ "]", isOk);
      ++resultCount;
    }
    long timeTaken = EnvironmentEdgeManager.currentTime() - startTs;
    // Verify the result count.
    long onesGennedAfterScan = dataGen.getExpectedNumberOfKeys();
    Assert.assertTrue("Read " + resultCount + " keys when at most " + onesGennedAfterScan
      + " were generated ", onesGennedAfterScan >= resultCount);
    if (isWriterDone) {
      Assert.assertTrue("Read " + resultCount + " keys; the writer is done and "
        + onesGennedAfterScan + " keys were generated", onesGennedAfterScan == resultCount);
    } else if (onesGennedBeforeScan * 0.9 > resultCount) {
      LOG.warn("Read way too few keys (" + resultCount + "/" + onesGennedBeforeScan
        + ") - there might be a problem, or the writer might just be slow");
    }
    LOG.info("Scan took " + timeTaken + "ms");
    if (!isWriterDone) {
      Thread.sleep(WAIT_BETWEEN_SCANS_MS);
      now = EnvironmentEdgeManager.currentTime();
    }
  }
  Assert.assertEquals("There are write failures", 0, writer.getNumWriteFailures());
  Assert.assertTrue("Writer is not done", isWriterDone);
  // Assert.fail("Boom!");
  connection.close();
}
 
Example 5
Source File: ScanUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public static void setReversed(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.TRUE_BYTES);
    scan.setLoadColumnFamiliesOnDemand(false);
}
 
Example 6
Source File: ScanUtil.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public static void unsetReversed(Scan scan) {
    scan.setAttribute(BaseScannerRegionObserver.REVERSE_SCAN, PDataType.FALSE_BYTES);
    scan.setLoadColumnFamiliesOnDemand(true);
}