Java Code Examples for org.apache.hadoop.hbase.client.Scan#setCaching()

private Scan createScan(String applicationName, Range range, boolean scanBackward) {
    Scan scan = new Scan();
    scan.setCaching(this.scanCacheSize);

    byte[] bApplicationName = Bytes.toBytes(applicationName);
    byte[] traceIndexStartKey = SpanUtils.getApplicationTraceIndexRowKey(bApplicationName, range.getFrom());
    byte[] traceIndexEndKey = SpanUtils.getApplicationTraceIndexRowKey(bApplicationName, range.getTo());

    if (scanBackward) {
        // start key is replaced by end key because key has been reversed
        scan.setStartRow(traceIndexEndKey);
        scan.setStopRow(traceIndexStartKey);
    } else {
        scan.setReversed(true);
        scan.setStartRow(traceIndexStartKey);
        scan.setStopRow(traceIndexEndKey);
    }

    scan.addFamily(descriptor.getColumnFamilyName());
    scan.setId("ApplicationTraceIndexScan");

    // toString() method of Scan converts a message to json format so it is slow for the first time.
    logger.trace("create scan:{}", scan);
    return scan;
}
 

private Scan createScan(Application application, Range range, byte[]... familyArgs) {
    range = rangeFactory.createStatisticsRange(range);

    if (logger.isDebugEnabled()) {
        logger.debug("scan Time:{}", range.prettyToString());
    }

    // start key is replaced by end key because timestamp has been reversed
    byte[] startKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getTo());
    byte[] endKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getFrom());

    Scan scan = new Scan();
    scan.setCaching(SCAN_CACHE_SIZE);
    scan.setStartRow(startKey);
    scan.setStopRow(endKey);
    for(byte[] family : familyArgs) {
        scan.addFamily(family);
    }
    scan.setId("ApplicationStatisticsScan");

    return scan;
}
 

private Scan createScan(Application application, Range range, byte[] family) {
    range = rangeFactory.createStatisticsRange(range);

    if (logger.isDebugEnabled()) {
        logger.debug("scan time:{} ", range.prettyToString());
    }

    // start key is replaced by end key because timestamp has been reversed
    byte[] startKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getTo());
    byte[] endKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getFrom());

    Scan scan = new Scan();
    scan.setCaching(SCAN_CACHE_SIZE);
    scan.setStartRow(startKey);
    scan.setStopRow(endKey);
    scan.addFamily(family);
    scan.setId("ApplicationStatisticsScan");

    return scan;
}
 

/**
 * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize
 * @param cachingRowLimit The row limit that will be enforced through caching
 */
public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
    throws Exception {
  Scan scan = new Scan();
  scan.setAllowPartialResults(true);

  // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to
  // ensure that the result size we specify is not an exact multiple of the number of cells
  // in a row. This ensures that partial results will be returned when the result size limit
  // is reached before the caching limit.
  int cellOffset = NUM_COLS / 3;
  long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
  scan.setMaxResultSize(maxResultSize);
  scan.setCaching(cachingRowLimit);

  try (ResultScanner scanner = TABLE.getScanner(scan)) {
    Result r = null;
    // Approximate the number of rows we expect will fit into the specified max rsult size. If
    // this approximation is less than caching, then we expect that the max result size limit will
    // be hit before the caching limit and thus partial results may be seen
    boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
    while ((r = scanner.next()) != null) {
      assertTrue(!r.mayHaveMoreCellsInRow() || expectToSeePartialResults);
    }
  }
}
 

/**
 * Read the first column of the first row and display it's AVRO content.
 * @param args
 * @throws IOException
 */
public static void main(String[] args) throws IOException {
  Configuration config = HBaseConfiguration.create();
  // tag::SETUP[]
  try (Connection connection = ConnectionFactory.createConnection(config);
       Table sensorsTable = connection.getTable(sensorsTableName)) {  // <1>
    Scan scan = new Scan ();
    scan.setCaching(1); // <2>

    ResultScanner scanner = sensorsTable.getScanner(scan);
    Result result = scanner.next(); // <3>
    if (result != null && !result.isEmpty()) {
      Event event = new Util().cellToEvent(result.listCells().get(0), null); // <4>
      LOG.info("Retrived AVRO content: " + event.toString());
    } else {
      LOG.error("Impossible to find requested cell");
    }
  }
  // end::SETUP[]
}
 

private Scan createScan(Application parentApplication, Range range) {
    Objects.requireNonNull(parentApplication, "parentApplication");

    if (logger.isDebugEnabled()) {
        logger.debug("scan parentApplication:{}, range:{}", parentApplication, range);
    }

    // TODO need common logic for creating scanner
    final long startTime = TimeUtils.reverseTimeMillis(timeSlot.getTimeSlot(range.getFrom()));
    final long endTime = TimeUtils.reverseTimeMillis(timeSlot.getTimeSlot(range.getTo()) + 1);
    // start key is replaced by end key because timestamp has been reversed
    final byte[] startKey = createKey(parentApplication, endTime);
    final byte[] endKey = createKey(parentApplication, startTime);

    Scan scan = new Scan();
    scan.setCaching(this.scanCacheSize);
    scan.setStartRow(startKey);
    scan.setStopRow(endKey);
    scan.setId("HostApplicationScan_Ver2");

    return scan;
}
 

private Scan createScan(StatType statType, String applicationId, Range range, int scanCacheSize) {
    Scan scan = this.operationFactory.createScan(applicationId, statType, range.getFrom(), range.getTo());
    scan.setCaching(scanCacheSize);
    scan.setId("ApplicationStat_" + statType);
    scan.addFamily(descriptor.getColumnFamilyName());
    return scan;
}
 

private void doScan(int n, boolean caching) throws IOException {
  Scan scan = new Scan();
  if (caching) {
    scan.setCaching(n);
  } else {
    scan.setCaching(1);
  }
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < n; i++) {
    Result res = scanner.next();
    LOG.debug("Result row: " + Bytes.toString(res.getRow()) + ", value: " +
        Bytes.toString(res.getValue(cf, qualifier)));
  }
}
 

@Test
public void testReadPt() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  setUp(0L, tableName);
  long ts = System.currentTimeMillis();
  byte[] value1 = Bytes.toBytes("value1");
  Put put1 = new Put(row1);
  put1.addColumn(family, qf1, ts, value1);
  table.put(put1);
  Put put2 = new Put(row2);
  byte[] value2 = Bytes.toBytes("value2");
  put2.addColumn(family, qf1, ts, value2);
  table.put(put2);

  Scan scan = new Scan();
  scan.setCaching(1);
  ResultScanner rs = table.getScanner(scan);
  Result result = rs.next();
  Put put3 = new Put(row1);
  byte[] value3 = Bytes.toBytes("value3");
  put3.addColumn(family, qf1, ts, value3);
  table.put(put3);
  Put put4 = new Put(row2);
  byte[] value4 = Bytes.toBytes("value4");
  put4.addColumn(family, qf1, ts, value4);
  table.put(put4);

  Cell cell = result.getColumnLatestCell(family, qf1);
  Assert.assertArrayEquals(value1, CellUtil.cloneValue(cell));

  admin.flush(tableName);
  result = rs.next();
  cell = result.getColumnLatestCell(family, qf1);
  Assert.assertArrayEquals(value2, CellUtil.cloneValue(cell));
}
 

@Override
public List<Object> getMetricData(String metricId, MetricType metricType, int win, long start, long end, int size) {
    long metricIdLong = Long.parseLong(metricId);
    byte[] startKey = MetricBaseData.makeKey(metricIdLong, win, start);
    byte[] endKey = MetricBaseData.makeKey(metricIdLong, win, end);

    HTableInterface table = getHTableInterface(TABLE_METRIC_DATA);
    Scan scan = new Scan(startKey, endKey);
    //scan.setBatch(10);
    scan.setCaching(CACHE_SIZE);
    ResultScanner scanner = null;
    try {
        scanner = getScanner(table, scan);
        Iterator<Result> rows = scanner.iterator();
        if (rows != null) {
            List<Object> ret = new ArrayList<>(size);
            while (rows.hasNext()) {
                Result row = rows.next();
                Object obj = parseMetricDataRow(row, metricType);
                if (obj != null) {
                    ret.add(obj);
                }
            }
            return ret;
        }
    } catch (Exception ex) {
        logger.error("Scan error, metric id:{}, metric type:{}", metricId, metricType, ex);
    } finally {
        if (scanner != null) {
            scanner.close();
        }
        closeTable(table);
    }

    return new ArrayList<>(0);
}
 

private Scan buildScan(HBaseKeyRange keyRange) {
    Scan scan = new Scan();
    scan.setCaching(SCAN_CACHE);
    scan.setCacheBlocks(true);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
    for (RowValueDecoder valueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn();
        byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
        byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
        scan.addColumn(byteFamily, byteQualifier);
    }
    scan.setStartRow(keyRange.getStartKey());
    scan.setStopRow(keyRange.getStopKey());
    return scan;
}
 

@Test
public void testReversedCompleteResultWhenRegionMove() throws IOException {
  Table table = createTestTable(TableName.valueOf(name.getMethodName()),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setCaching(1);
  scan.setReversed(true);
  ResultScanner scanner = table.getScanner(scan);

  Result result1 = scanner.next();
  assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result1.mayHaveMoreCellsInRow());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result2.mayHaveMoreCellsInRow());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[NUM_ROWS-3], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result3.mayHaveMoreCellsInRow());

}
 

public Result getClosestRowBefore(Region r, byte[] row, byte[] family) throws IOException {
  Scan scan = new Scan().withStartRow(row);
  scan.setSmall(true);
  scan.setCaching(1);
  scan.setReversed(true);
  scan.addFamily(family);
  try (RegionScanner scanner = r.getScanner(scan)) {
    List<Cell> cells = new ArrayList<>(1);
    scanner.next(cells);
    if (r.getRegionInfo().isMetaRegion() && !isTargetTable(row, cells.get(0))) {
      return null;
    }
    return Result.create(cells);
  }
}
 

/**
 * Delete all rows from the given table. This method is intended only for development and testing use.
 * @param tableString
 * @param timestamp
 * @throws IOException
 */
@Override
public void clearTable(String tableString, long timestamp) throws IOException
{
    TableName tableName = TableName.valueOf(tableString);

    if (!adm.tableExists(tableName)) {
        log.debug("Attempted to clear table {} before it exists (noop)", tableString);
        return;
    }

    // Unfortunately, linear scanning and deleting rows is faster in HBase when running integration tests than
    // disabling and deleting/truncating tables.
    final Scan scan = new Scan();
    scan.setCacheBlocks(false);
    scan.setCaching(2000);
    scan.setTimeRange(0, Long.MAX_VALUE);
    scan.readVersions(1);

    try (final Table table = adm.getConnection().getTable(tableName);
         final ResultScanner scanner = table.getScanner(scan)) {
        final Iterator<Result> iterator = scanner.iterator();
        final int batchSize = 1000;
        final List<Delete> deleteList = new ArrayList<>();
        while (iterator.hasNext()) {
            deleteList.add(new Delete(iterator.next().getRow(), timestamp));
            if (!iterator.hasNext() || deleteList.size() == batchSize) {
                table.delete(deleteList);
                deleteList.clear();
            }
        }
    }
}
 

public Scanner(Scan scan) throws IOException {
  if (scan.getBatch() > 0) {
    throw new IOException("Batch is not supported in Scanner");
  }
  if (scan.getCaching() <= 0) {
    scan.setCaching(scannerCaching);
  } else if (scan.getCaching() == 1 && scan.isReversed()){
    // for reverse scan, we need to pass the last row to the next scanner
    // we need caching number bigger than 1
    scan.setCaching(scan.getCaching() + 1);
  }
  this.scan = ThriftUtilities.scanFromHBase(scan);
}
 

public SlicedRowFilterGTSDecoderIterator(long now, long timespan, List<Metadata> metadatas, Connection conn, TableName tableName, byte[] colfam, boolean writeTimestamp, KeyStore keystore, boolean useBlockCache) {
    
  this.keystore = keystore;
  this.now = now;
  this.timespan = timespan;
  this.hbaseAESKey = keystore.getKey(KeyStore.AES_HBASE_DATA);
  this.writeTimestamp = writeTimestamp;
  
  //
  // Check that if 'timespan' is < 0 then 'now' is either Long.MAX_VALUE or congruent to 0 modulo DEFAULT_MODULUS
  //
  
  if (timespan < 0) {
    if (Long.MAX_VALUE != now && 0 != (now % Constants.DEFAULT_MODULUS)) {
      throw new RuntimeException("Incompatible 'timespan' (" + timespan + ") and 'now' (" + now + ")");
    }
  }
  
  //
  // Create a SlicedRowFilter for the prefix, class id, labels id and ts
  // We include the prefix so we exit the filter early when the last
  // matching row has been reached
  //
  
  // 128BITS
  
  int[] bounds = { 0, 24 };
  
  //
  // Create singleton for each classId/labelsId combo
  //
  // TODO(hbs): we should really create multiple scanner, one per class Id for example,
  // 
  
  List<Pair<byte[], byte[]>> ranges = new ArrayList<Pair<byte[], byte[]>>();
  
  for (Metadata metadata: metadatas) {
    byte[][] keys = getKeys(metadata, now, timespan);
    byte[] lower = keys[0];
    byte[] upper = keys[1];
    
    this.metadatas.put(new String(Arrays.copyOfRange(lower, prefix.length, prefix.length + 16), StandardCharsets.ISO_8859_1), metadata);
    
    Pair<byte[],byte[]> range = new Pair<byte[],byte[]>(lower, upper);
    
    ranges.add(range);
  }
              
  SlicedRowFilter filter = new SlicedRowFilter(bounds, ranges, timespan < 0 ? -timespan : Long.MAX_VALUE);

  //
  // Create scanner. The start key is the lower bound of the first range
  //
  
  Scan scan = new Scan();
  scan.addFamily(colfam); // (HBaseStore.GTS_COLFAM, Longs.toByteArray(Long.MAX_VALUE - modulus));
  scan.setStartRow(filter.getStartKey());
  byte[] filterStopKey = filter.getStopKey();
  // Add one byte at the end (we can do that because we know the slice is the whole key)
  byte[] stopRow = Arrays.copyOf(filterStopKey, filterStopKey.length + 1);
  scan.setStopRow(stopRow);
  scan.setFilter(filter);
  
  scan.setMaxResultSize(1000000L);
  scan.setBatch(50000);
  scan.setCaching(50000);
  
  scan.setCacheBlocks(useBlockCache);

  Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_HBASE_CLIENT_FILTERED_SCANNERS, Sensision.EMPTY_LABELS, 1);
  Sensision.update(SensisionConstants.SENSISION_CLASS_CONTINUUM_HBASE_CLIENT_FILTERED_SCANNERS_RANGES, Sensision.EMPTY_LABELS, ranges.size());

  try {
    this.htable = conn.getTable(tableName);
    this.scanner = this.htable.getScanner(scan);
    iter = scanner.iterator();          
  } catch (IOException ioe) {
    LOG.error("",ioe);
    this.iter = null;
  }
}
 

public int run(Path outputDir, int numReducers) throws Exception {
  LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);

  job = Job.getInstance(getConf());

  job.setJobName("Link Verifier");
  job.setNumReduceTasks(numReducers);
  job.setJarByClass(getClass());

  setJobScannerConf(job);

  Scan scan = new Scan();
  scan.addColumn(FAMILY_NAME, COLUMN_PREV);
  scan.setCaching(10000);
  scan.setCacheBlocks(false);
  if (isMultiUnevenColumnFamilies(getConf())) {
    scan.addColumn(BIG_FAMILY_NAME, BIG_FAMILY_NAME);
    scan.addColumn(TINY_FAMILY_NAME, TINY_FAMILY_NAME);
  }

  TableMapReduceUtil.initTableMapperJob(getTableName(getConf()).getName(), scan,
      VerifyMapper.class, BytesWritable.class, BytesWritable.class, job);
  TableMapReduceUtil.addDependencyJarsForClasses(job.getConfiguration(),
                                                 AbstractHBaseTool.class);

  job.getConfiguration().setBoolean("mapreduce.map.speculative", false);

  job.setReducerClass(VerifyReducer.class);
  job.setOutputFormatClass(SequenceFileAsBinaryOutputFormat.class);
  job.setOutputKeyClass(BytesWritable.class);
  job.setOutputValueClass(BytesWritable.class);
  TextOutputFormat.setOutputPath(job, outputDir);

  boolean success = job.waitForCompletion(true);

  if (success) {
    Counters counters = job.getCounters();
    if (null == counters) {
      LOG.warn("Counters were null, cannot verify Job completion."
          + " This is commonly a result of insufficient YARN configuration.");
      // We don't have access to the counters to know if we have "bad" counts
      return 0;
    }

    // If we find no unexpected values, the job didn't outright fail
    if (verifyUnexpectedValues(counters)) {
      // We didn't check referenced+unreferenced counts, leave that to visual inspection
      return 0;
    }
  }

  // We failed
  return 1;
}
 

public ScannerResultGenerator(final String tableName, final RowSpec rowspec,
  final Filter filter, final int caching ,final boolean cacheBlocks, int limit) throws IOException {
  Table table = RESTServlet.getInstance().getTable(tableName);
  try {
    Scan scan;
    if (rowspec.hasEndRow()) {
      scan = new Scan().withStartRow(rowspec.getStartRow()).withStopRow(rowspec.getEndRow());
    } else {
      scan = new Scan().withStartRow(rowspec.getStartRow());
    }
    if (rowspec.hasColumns()) {
      byte[][] columns = rowspec.getColumns();
      for (byte[] column: columns) {
        byte[][] split = CellUtil.parseColumn(column);
        if (split.length == 1) {
          scan.addFamily(split[0]);
        } else if (split.length == 2) {
          scan.addColumn(split[0], split[1]);
        } else {
          throw new IllegalArgumentException("Invalid familyAndQualifier provided.");
        }
      }
    }
    scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
    scan.readVersions(rowspec.getMaxVersions());
    if (filter != null) {
      scan.setFilter(filter);
    }
    if (caching > 0 ) {
      scan.setCaching(caching);
    }
    if (limit > 0) {
      scan.setLimit(limit);
    }
    scan.setCacheBlocks(cacheBlocks);
    if (rowspec.hasLabels()) {
      scan.setAuthorizations(new Authorizations(rowspec.getLabels()));
    }
    scanner = table.getScanner(scan);
    cached = null;
    id = Long.toString(System.currentTimeMillis()) +
           Integer.toHexString(scanner.hashCode());
  } finally {
    table.close();
  }
}
 

public static Scan scanFromThrift(TScan in) throws IOException {
  Scan out = new Scan();

  if (in.isSetStartRow()) {
    out.withStartRow(in.getStartRow());
  }
  if (in.isSetStopRow()) {
    out.withStopRow(in.getStopRow());
  }
  if (in.isSetCaching()) {
    out.setCaching(in.getCaching());
  }
  if (in.isSetMaxVersions()) {
    out.readVersions(in.getMaxVersions());
  }

  if (in.isSetColumns()) {
    for (TColumn column : in.getColumns()) {
      if (column.isSetQualifier()) {
        out.addColumn(column.getFamily(), column.getQualifier());
      } else {
        out.addFamily(column.getFamily());
      }
    }
  }

  TTimeRange timeRange = in.getTimeRange();
  if (timeRange != null &&
      timeRange.isSetMinStamp() && timeRange.isSetMaxStamp()) {
    out.setTimeRange(timeRange.getMinStamp(), timeRange.getMaxStamp());
  }

  if (in.isSetBatchSize()) {
    out.setBatch(in.getBatchSize());
  }

  if (in.isSetFilterString()) {
    ParseFilter parseFilter = new ParseFilter();
    out.setFilter(parseFilter.parseFilterString(in.getFilterString()));
  }

  if (in.isSetAttributes()) {
    addAttributes(out,in.getAttributes());
  }

  if (in.isSetAuthorizations()) {
    out.setAuthorizations(new Authorizations(in.getAuthorizations().getLabels()));
  }

  if (in.isSetReversed()) {
    out.setReversed(in.isReversed());
  }

  if (in.isSetCacheBlocks()) {
    out.setCacheBlocks(in.isCacheBlocks());
  }

  if (in.isSetColFamTimeRangeMap()) {
    Map<ByteBuffer, TTimeRange> colFamTimeRangeMap = in.getColFamTimeRangeMap();
    if (MapUtils.isNotEmpty(colFamTimeRangeMap)) {
      for (Map.Entry<ByteBuffer, TTimeRange> entry : colFamTimeRangeMap.entrySet()) {
        out.setColumnFamilyTimeRange(Bytes.toBytes(entry.getKey()),
            entry.getValue().getMinStamp(), entry.getValue().getMaxStamp());
      }
    }
  }

  if (in.isSetReadType()) {
    out.setReadType(readTypeFromThrift(in.getReadType()));
  }

  if (in.isSetLimit()) {
    out.setLimit(in.getLimit());
  }

  if (in.isSetConsistency()) {
    out.setConsistency(consistencyFromThrift(in.getConsistency()));
  }

  if (in.isSetTargetReplicaId()) {
    out.setReplicaId(in.getTargetReplicaId());
  }

  if (in.isSetFilterBytes()) {
    out.setFilter(filterFromThrift(in.getFilterBytes()));
  }

  return out;
}
 

@Test
public void testDistributedScan() {
  Configuration conf = htu.getConfiguration();
  
  JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);

  Scan scan = new Scan();
  scan.setCaching(100);
  
  JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd = hbaseContext.hbaseRDD(tableName, scan);
  
  List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect();
  
  results.size();
}