Java Code Examples for org.apache.hadoop.hbase.client.Scan.setCaching()

The following are Jave code examples for showing how to use setCaching() of the org.apache.hadoop.hbase.client.Scan class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License Vote up 6 votes
/**
 * 使用行过滤器 选择大于rowKey的行
 *
 * @param tableName 表名
 * @param rowKey    行健
 * @param count     数量
 */
public void rowFilter(String tableName, String rowKey, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    //使用行过滤器 选择大于 rowkey的行
    //scan.setFilter(new RowFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator(Bytes.toBytes(rowKey))));//直接行健
    //scan.setFilter(new RowFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new RegexStringComparator("row.*")));//正则表达式
    //scan.setFilter(new RowFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new SubstringComparator("row")));//字符串包含
    scan.setFilter(new RowFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryPrefixComparator("row".getBytes())));//字符串前缀
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 2
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License Vote up 6 votes
/**
 * 列族过滤器
 *
 * @param tableName 表名
 * @param rowFamily 列族
 * @param count     数量
 */
public void familyFilter(String tableName, String rowFamily, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    //使用列族过滤器
    //scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator(Bytes.toBytes(rowFamily))));//直接行健
    //scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new RegexStringComparator("row.*")));//正则表达式
    //scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new SubstringComparator("row")));//字符串包含
    scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryPrefixComparator("mm".getBytes())));//字符串前缀
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 3
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License Vote up 6 votes
/**
 * 列限定符过滤器
 *
 * @param tableName  表名
 * @param columnName 列限定符
 * @param count      数量
 */
public void qualifierFilter(String tableName, String columnName, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    //使用列族过滤器
    scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(columnName))));//直接行健
    //scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator("row.*")));//正则表达式
    //scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("row")));//字符串包含
    //scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator("m".getBytes())));//字符串前缀
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 4
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License Vote up 6 votes
/**
 * 参考列过滤器(获取相同时间戳的列)
 *
 * @param tableName    表名
 * @param columnFamily 列族
 * @param qualifier    列限定符
 * @param columnValue  列值
 * @param count        数量
 */
public void dependentColumnFilter(String tableName, String columnFamily, String qualifier, String columnValue, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    scan.setFilter(new PrefixFilter(Bytes.toBytes("")));
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 5
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License Vote up 6 votes
/**
 * 单列排除过滤器(返回的列 不包含参考列)
 *
 * @param tableName    表名
 * @param columnFamily 列族
 * @param qualifier    列限定符
 * @param columnValue  列值
 * @param count        数量
 */
public void SingleColumnValueExcludeFilter(String tableName, String columnFamily, String qualifier, String columnValue, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    SingleColumnValueExcludeFilter singleColumnValueFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(columnFamily), Bytes.toBytes(qualifier), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(columnValue));
    //singleColumnValueFilter.setFilterIfMissing(true);//当不存在这列的行 默认不过滤
    singleColumnValueFilter.setLatestVersionOnly(true);//获取最新版本
    scan.setFilter(singleColumnValueFilter);
    scan.setCaching(10);
    //scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 6
Project: ditb   File: MDIndex.java   Source Code and License Vote up 6 votes
public Result getRowOrBefore(Table table, byte[] row, byte[] family) throws IOException {
  long start = System.currentTimeMillis();
  Scan scan = new Scan();
  scan.addFamily(family);
  scan.setReversed(true);
  scan.setStartRow(row);
  scan.setCacheBlocks(false);
  scan.setCaching(1);
  scan.setSmall(true);
  ResultScanner scanner = table.getScanner(scan);
  Result ret = scanner.next();
  scanner.close();
  prevRowTotalTime += System.currentTimeMillis() - start;
  prevRowTotalCount++;
  return ret;
}
 
Example 7
Project: ditb   File: TestScannerHeartbeatMessages.java   Source Code and License Vote up 6 votes
/**
 * Test the case that the time limit for the scan is reached after each full row of cells is
 * fetched.
 * @throws Exception
 */
public Callable<Void> testHeartbeatBetweenRows() throws Exception {
  return new Callable<Void>() {

    @Override
    public Void call() throws Exception {
      // Configure the scan so that it can read the entire table in a single RPC. We want to test
      // the case where a scan stops on the server side due to a time limit
      Scan scan = new Scan();
      scan.setMaxResultSize(Long.MAX_VALUE);
      scan.setCaching(Integer.MAX_VALUE);

      testEquivalenceOfScanWithHeartbeats(scan, DEFAULT_ROW_SLEEP_TIME, -1, false);
      return null;
    }
  };
}
 
Example 8
Project: ditb   File: LocalScanner.java   Source Code and License Vote up 6 votes
private ResultScanner getNextScanner() throws IOException {
  if (INIT_REGION_SIZE != getRegionNumber()) {
    throw new IOException(
        "region number changed from " + INIT_REGION_SIZE + " to " + getRegionNumber());
  }
  if (regionLocationQueue.isEmpty()) return null;
  HRegionLocation regionLocation = regionLocationQueue.poll();

  Scan newScan = new Scan(rawScan);
  byte[] key = regionLocation.getRegionInfo().getStartKey();
  if (key != null && key.length > 0) newScan.setStartRow(key);
  key = regionLocation.getRegionInfo().getEndKey();
  if (key != null && key.length > 0) newScan.setStopRow(key);
  newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi"));
  newScan.setId(rawScan.getId());
  newScan.setCacheBlocks(rawScan.getCacheBlocks());
  newScan.setCaching(rawScan.getCaching());
  return table.getScanner(newScan);
}
 
Example 9
Project: ditb   File: PerformanceEvaluation.java   Source Code and License Vote up 6 votes
protected Scan constructScan(byte[] valuePrefix) throws IOException {
  FilterList list = new FilterList();
  Filter filter = new SingleColumnValueFilter(
      FAMILY_NAME, COLUMN_ZERO, CompareFilter.CompareOp.EQUAL,
      new BinaryComparator(valuePrefix)
  );
  list.addFilter(filter);
  if(opts.filterAll) {
    list.addFilter(new FilterAllFilter());
  }
  Scan scan = new Scan();
  scan.setCaching(opts.caching);
  if (opts.addColumns) {
    scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
  } else {
    scan.addFamily(FAMILY_NAME);
  }
  scan.setFilter(list);
  return scan;
}
 
Example 10
Project: ditb   File: IntegrationTestBigLinkedListWithVisibility.java   Source Code and License Vote up 5 votes
private int doVerify(Path outputDir, int numReducers) throws IOException, InterruptedException,
    ClassNotFoundException {
  job = new Job(getConf());

  job.setJobName("Link Verifier");
  job.setNumReduceTasks(numReducers);
  job.setJarByClass(getClass());

  setJobScannerConf(job);

  Scan scan = new Scan();
  scan.addColumn(FAMILY_NAME, COLUMN_PREV);
  scan.setCaching(10000);
  scan.setCacheBlocks(false);
  String[] split = labels.split(COMMA);

  scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2],
      split[(this.labelIndex * 2) + 1]));

  TableMapReduceUtil.initTableMapperJob(tableName.getName(), scan, VerifyMapper.class,
      BytesWritable.class, BytesWritable.class, job);
  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);

  job.getConfiguration().setBoolean("mapreduce.map.speculative", false);

  job.setReducerClass(VerifyReducer.class);
  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, outputDir);
  boolean success = job.waitForCompletion(true);

  return success ? 0 : 1;
}
 
Example 11
Project: QDrill   File: HBasePStore.java   Source Code and License Vote up 5 votes
Iter() {
  try {
    Scan scan = new Scan(tableNameStartKey, tableNameStopKey);
    scan.addColumn(FAMILY, QUALIFIER);
    scan.setCaching(config.getMaxIteratorSize() > 100 ? 100 : config.getMaxIteratorSize());
    scanner = table.getScanner(scan);
  } catch (IOException e) {
    throw new DrillRuntimeException("Caught error while creating HBase scanner for table:" + Bytes.toString(table.getTableName()), e);
  }
}
 
Example 12
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License Vote up 5 votes
@Test
public void testBatchingResultWhenRegionMove() throws IOException {
  Table table =
      createTestTable(TableName.valueOf("testBatchingResultWhenRegionMove"), ROWS, FAMILIES,
          QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setCaching(1);
  scan.setBatch(1);

  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
}
 
Example 13
Project: ditb   File: UDGScanner.java   Source Code and License Vote up 5 votes
private GCScanner createGCScanner(ScanRange selectedRange) throws IOException {
  List<ScanRange> list = new ArrayList<>(rangeList.getRanges());
  list.remove(selectedRange);
  Scan scan = new Scan();
  scan.setStartRow(selectedRange.getStart());
  scan.setStopRow(selectedRange.getStop());
  scan.setCaching(rawScan.getCaching());
  scan.setCacheBlocks(rawScan.getCacheBlocks());
  scan.setFilter(new ScanRange.ScanRangeList(list).toFilterList());
  Table table = conn.getTable(
      relation.getIndexTableName(selectedRange.getFamily(), selectedRange.getQualifier()));
  ResultScanner scanner = table.getScanner(scan);
  return new GCScanner(this, scanner, selectedRange.getFamily(), selectedRange.getQualifier());
}
 
Example 14
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License Vote up 5 votes
/**
 * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize
 * @param cachingRowLimit The row limit that will be enforced through caching
 * @throws Exception
 */
public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
    throws Exception {
  Scan scan = new Scan();
  scan.setAllowPartialResults(true);

  // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to
  // ensure that the result size we specify is not an exact multiple of the number of cells
  // in a row. This ensures that partial results will be returned when the result size limit
  // is reached before the caching limit.
  int cellOffset = NUM_COLS / 3;
  long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
  scan.setMaxResultSize(maxResultSize);
  scan.setCaching(cachingRowLimit);

  ResultScanner scanner = TABLE.getScanner(scan);
  ClientScanner clientScanner = (ClientScanner) scanner;
  Result r = null;

  // Approximate the number of rows we expect will fit into the specified max rsult size. If this
  // approximation is less than caching, then we expect that the max result size limit will be
  // hit before the caching limit and thus partial results may be seen
  boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
  while ((r = clientScanner.next()) != null) {
    assertTrue(!r.isPartial() || expectToSeePartialResults);
  }

  scanner.close();
}
 
Example 15
Project: ditb   File: PerfScanBase.java   Source Code and License Vote up 5 votes
private OpResult executeScan() throws IOException, ParseException {
  if (!hasScan()) {
    return new OpResult("scan not supported", 1, 1);
  }
  Table table = conn.getTable(opTblName);
  BufferedReader br = new BufferedReader(new FileReader(scanFilePath));
  String line;
  long totalTime = 0;
  int counter = 0;
  Result[] results;
  while ((line = br.readLine()) != null) {
    Scan scan = new Scan(getIndexTableScanStartKey(line));
    scan.setCaching(workload.getScanCacheSize());
    scan.setCacheBlocks(false);
    long startTime = System.currentTimeMillis();
    ResultScanner scanner = table.getScanner(scan);
    int wantedRecords = sizeScanCovering;
    while (true) {
      results = scanner.next(Math.min(wantedRecords, workload.getScanCacheSize()));
      if (results == null || results.length == 0) break;
      for (Result result : results) {
        int k = recordsInOneResult(result);
        wantedRecords -= k;
        counter += k;
      }
      if (wantedRecords <= 0) break;
    }
    scanner.close();
    totalTime += System.currentTimeMillis() - startTime;
  }
  OpResult ret = new OpResult("scan", counter, totalTime);
  br.close();
  table.close();
  return ret;
}
 
Example 16
Project: ditb   File: PerfMD.java   Source Code and License Vote up 5 votes
@Override protected void insertOneRecord(AbstractDITBRecord record) throws IOException {
  // note, MD-HBase get before update, we summarize the time in get seperately
  MDPoint point = record.toMDPoint();
  byte[] row = MDUtils.bitwiseZip(point.values, mdAdmin.getDimensions());
  // get before row
  long startTime = System.currentTimeMillis();
  Scan scan = new Scan();
  scan.addFamily(MDHBaseAdmin.BUCKET_FAMILY);
  scan.setReversed(true);
  scan.setStartRow(row);
  scan.setCacheBlocks(false);
  scan.setCaching(1);
  scan.setSmall(true);
  ResultScanner scanner = table.getScanner(scan);
  Result result = scanner.next();
  scanner.close();
  gbuTime += System.currentTimeMillis() - startTime;
  gbuCount++;
  // default scan
  if (result == null) {
    row = mdAdmin.getBucketSuffixRow(point);
  } else {
    row = result.getRow();
  }
  table.incrementColumnValue(row, MDHBaseAdmin.BUCKET_FAMILY,
      MDHBaseAdmin.BUCKET_SIZE_QUALIFIER, 1);
}
 
Example 17
Project: ditb   File: PerfMD.java   Source Code and License Vote up 5 votes
@Override protected Result processGet(Table table, Get get) throws IOException {
  Scan scan = new Scan();
  scan.addFamily(MDHBaseAdmin.BUCKET_FAMILY);
  scan.setReversed(true);
  scan.setStartRow(get.getRow());
  scan.setCacheBlocks(false);
  scan.setCaching(1);
  scan.setSmall(true);
  ResultScanner scanner = table.getScanner(scan);
  Result ret = scanner.next();
  scanner.close();
  return ret;
}
 
Example 18
Project: ditb   File: GCScanner.java   Source Code and License Vote up 5 votes
/**
 * init selected range and scanner
 *
 * @throws IOException
 */
private void initScanner() throws IOException {
  ScanRange selectedRange = null;
  int selectedRegionNumber = Integer.MAX_VALUE;
  for (ScanRange range : rangeList.getRanges()) {
    int cover = countCoveringRegions(conn,
        relation.getIndexTableName(range.getFamily(), range.getQualifier()), range.getStart(),
        range.getStop());
    LOG.info("LCDBG, " + cover + " regions are covered by range " + range);
    if (selectedRegionNumber > cover) {
      selectedRegionNumber = cover;
      selectedRange = range;
    }
  }
  LOG.info("LCDBG, GC Scanner using range " + selectedRange + " with " + selectedRegionNumber
      + " regions for scan id= " + rawScan.getId());
  indexFamily = selectedRange.getFamily();
  indexQualifier = selectedRange.getQualifier();
  List<ScanRange> list = new ArrayList<>(rangeList.getRanges());
  list.remove(selectedRange);
  Scan scan = new Scan();
  scan.setStartRow(selectedRange.getStart());
  scan.setStopRow(selectedRange.getStop());
  scan.setFamilyMap(rawScan.getFamilyMap());
  scan.setCaching(rawScan.getCaching());
  scan.setCacheBlocks(rawScan.getCacheBlocks());
  scan.setId(rawScan.getId());
  scan.setFilter(new ScanRange.ScanRangeList(list).toFilterList());
  Table table = conn.getTable(
      relation.getIndexTableName(selectedRange.getFamily(), selectedRange.getQualifier()));
  scanner = table.getScanner(scan);
}
 
Example 19
Project: ditb   File: ThriftServerRunner.java   Source Code and License Vote up 4 votes
@Override
public int scannerOpenWithScan(ByteBuffer tableName, TScan tScan,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan();
    addAttributes(scan, attributes);
    if (tScan.isSetStartRow()) {
      scan.setStartRow(tScan.getStartRow());
    }
    if (tScan.isSetStopRow()) {
      scan.setStopRow(tScan.getStopRow());
    }
    if (tScan.isSetTimestamp()) {
      scan.setTimeRange(0, tScan.getTimestamp());
    }
    if (tScan.isSetCaching()) {
      scan.setCaching(tScan.getCaching());
    }
    if (tScan.isSetBatchSize()) {
      scan.setBatch(tScan.getBatchSize());
    }
    if (tScan.isSetColumns() && tScan.getColumns().size() != 0) {
      for(ByteBuffer column : tScan.getColumns()) {
        byte [][] famQf = KeyValue.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    if (tScan.isSetFilterString()) {
      ParseFilter parseFilter = new ParseFilter();
      scan.setFilter(
          parseFilter.parseFilterString(tScan.getFilterString()));
    }
    if (tScan.isSetReversed()) {
      scan.setReversed(tScan.isReversed());
    }
    return addScanner(table.getScanner(scan), tScan.sortColumns);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw new IOError(Throwables.getStackTraceAsString(e));
  } finally{
    closeTable(table);
  }
}
 
Example 20
Project: ditb   File: MultiThreadedClientExample.java   Source Code and License Vote up 4 votes
@Override
public Boolean call() throws Exception {

  // total length in bytes of all read rows.
  int result = 0;

  // Number of rows the scan will read before being considered done.
  int toRead = 100;
  try (Table t = connection.getTable(tableName)) {
    byte[] rk = Bytes.toBytes(ThreadLocalRandom.current().nextLong());
    Scan s = new Scan(rk);

    // This filter will keep the values from being sent accross the wire.
    // This is good for counting or other scans that are checking for
    // existence and don't rely on the value.
    s.setFilter(new KeyOnlyFilter());

    // Don't go back to the server for every single row.
    // We know these rows are small. So ask for 20 at a time.
    // This would be application specific.
    //
    // The goal is to reduce round trips but asking for too
    // many rows can lead to GC problems on client and server sides.
    s.setCaching(20);

    // Don't use the cache. While this is a silly test program it's still good to be
    // explicit that scans normally don't use the block cache.
    s.setCacheBlocks(false);

    // Open up the scanner and close it automatically when done.
    try (ResultScanner rs = t.getScanner(s)) {

      // Now go through rows.
      for (Result r : rs) {
        // Keep track of things size to simulate doing some real work.
        result += r.getRow().length;
        toRead -= 1;

        // Most online applications won't be
        // reading the entire table so this break
        // simulates small to medium size scans,
        // without needing to know an end row.
        if (toRead <= 0)  {
          break;
        }
      }
    }
  }
  return result > 0;
}