Java Code Examples for org.apache.hadoop.hbase.client.Scan.setCacheBlocks()

The following are Jave code examples for showing how to use setCacheBlocks() of the org.apache.hadoop.hbase.client.Scan class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: SparkIsax   File: HBaseUtils.java   Source Code and License Vote up 6 votes
/**
 * 拷贝表
 * 
 * @throws IOException
 */
public static void copyTable(String oldTableName, String newTableName,String ColumnFamily, String ColumnName)throws IOException {
	if(CreateNewTable(newTableName))
		logger.info("创建表"+newTableName+"表成功");
	else{
		logger.info("创建表"+newTableName+"表失败");
	}
	Scan s = new Scan();
	s.addColumn(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName));
	s.setMaxVersions(1);
	s.setCacheBlocks(false);
	ResultScanner rs = hbase_table.getScanner(s);
	
	HTableInterface hbase_table_new = conn.getTable(newTableName);
	for (Result r : rs) {
		byte[] key = r.getRow();
		byte[] value = r.getValue(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName));
		Put put = new Put(key);
		put.add(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName), value);
		hbase_table_new.put(put);
	}
	rs.close();
	hbase_table_new.close();
}
 
Example 2
Project: ditb   File: CellCounter.java   Source Code and License Vote up 6 votes
private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
  Scan s = new Scan();
  // Set Scan Versions
  s.setMaxVersions(Integer.MAX_VALUE);
  s.setCacheBlocks(false);
  // Set Scan Column Family
  if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
    s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
  }
  // Set RowFilter or Prefix Filter if applicable.
  Filter rowFilter = getRowFilter(args);
  if (rowFilter!= null) {
    LOG.info("Setting Row Filter for counter.");
    s.setFilter(rowFilter);
  }
  // Set TimeRange if defined
  long timeRange[] = getTimeRange(args);
  if (timeRange != null) {
    LOG.info("Setting TimeRange for counter.");
    s.setTimeRange(timeRange[0], timeRange[1]);
  }
  return s;
}
 
Example 3
Project: ditb   File: HashTable.java   Source Code and License Vote up 6 votes
Scan initScan() throws IOException {
  Scan scan = new Scan();
  scan.setCacheBlocks(false);
  if (startTime != 0 || endTime != 0) {
    scan.setTimeRange(startTime, endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
  }
  if (scanBatch > 0) {
    scan.setBatch(scanBatch);
  }
  if (versions >= 0) {
    scan.setMaxVersions(versions);
  }
  if (!isTableStartRow(startRow)) {
    scan.setStartRow(startRow);
  }
  if (!isTableEndRow(stopRow)) {
    scan.setStopRow(stopRow);
  }
  if(families != null) {
    for(String fam : families.split(",")) {
      scan.addFamily(Bytes.toBytes(fam));
    }
  }
  return scan;
}
 
Example 4
Project: ditb   File: LocalScanner.java   Source Code and License Vote up 6 votes
private ResultScanner getNextScanner() throws IOException {
  if (INIT_REGION_SIZE != getRegionNumber()) {
    throw new IOException(
        "region number changed from " + INIT_REGION_SIZE + " to " + getRegionNumber());
  }
  if (regionLocationQueue.isEmpty()) return null;
  HRegionLocation regionLocation = regionLocationQueue.poll();

  Scan newScan = new Scan(rawScan);
  byte[] key = regionLocation.getRegionInfo().getStartKey();
  if (key != null && key.length > 0) newScan.setStartRow(key);
  key = regionLocation.getRegionInfo().getEndKey();
  if (key != null && key.length > 0) newScan.setStopRow(key);
  newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi"));
  newScan.setId(rawScan.getId());
  newScan.setCacheBlocks(rawScan.getCacheBlocks());
  newScan.setCaching(rawScan.getCaching());
  return table.getScanner(newScan);
}
 
Example 5
Project: ditb   File: IRScanner.java   Source Code and License Vote up 6 votes
private ResultScanner getNextScanner() throws IOException {
  if (INIT_REGION_SIZE != getRegionNumber()) {
    throw new IOException(
        "region number changed from " + INIT_REGION_SIZE + " to " + getRegionNumber());
  }
  if (regionLocationQueue.isEmpty()) return null;
  HRegionLocation regionLocation = regionLocationQueue.poll();
  Scan newScan = new Scan(rawScan);
  if (regionLocation.getRegionInfo().getStartKey() != null)
    newScan.setStartRow(regionLocation.getRegionInfo().getStartKey());
  if (regionLocation.getRegionInfo().getEndKey() != null)
    newScan.setStopRow(regionLocation.getRegionInfo().getEndKey());
  newScan.setAttribute(IndexConstants.SCAN_WITH_INDEX, Bytes.toBytes("Hi"));
  newScan.setFilter(rangeList.toFilterList());
  newScan.setAttribute(IndexConstants.MAX_SCAN_SCALE, Bytes.toBytes(1.0f));
  newScan.setId(rawScan.getId());
  newScan.setCacheBlocks(rawScan.getCacheBlocks());
  newScan.setCaching(rawScan.getCaching());
  return table.getScanner(newScan);
}
 
Example 6
Project: ditb   File: IndexResultScanner.java   Source Code and License Vote up 5 votes
public SingleScanner(Scan scan, byte[][] resultColumns, Table table) throws IOException {
  this.scan = scan;
  this.resultColumns = resultColumns;
  this.indexTable = table;
  this.currentStartKey = scan.getStartRow();
  this.scanning = true;
  this.finished = false;
  scan.setCacheBlocks(false);
  this.resultScanner = table.getScanner(scan);
  LOG.debug("scan caching:" + scan.getCaching());
}
 
Example 7
Project: ditb   File: BaseRunner.java   Source Code and License Vote up 5 votes
protected Scan getScan() {
  Scan scan = new Scan();
  scan.setAttribute(ScanRange.SCAN_RANGE_ATTRIBUTE_STR, scanRangeList.toBytesAttribute());
  scan.setFilter(scanRangeList.toFilterList());
  scan.setCacheBlocks(false);
  return scan;
}
 
Example 8
Project: ditb   File: PerfScanBase.java   Source Code and License Vote up 5 votes
private OpResult executeScan() throws IOException, ParseException {
  if (!hasScan()) {
    return new OpResult("scan not supported", 1, 1);
  }
  Table table = conn.getTable(opTblName);
  BufferedReader br = new BufferedReader(new FileReader(scanFilePath));
  String line;
  long totalTime = 0;
  int counter = 0;
  Result[] results;
  while ((line = br.readLine()) != null) {
    Scan scan = new Scan(getIndexTableScanStartKey(line));
    scan.setCaching(workload.getScanCacheSize());
    scan.setCacheBlocks(false);
    long startTime = System.currentTimeMillis();
    ResultScanner scanner = table.getScanner(scan);
    int wantedRecords = sizeScanCovering;
    while (true) {
      results = scanner.next(Math.min(wantedRecords, workload.getScanCacheSize()));
      if (results == null || results.length == 0) break;
      for (Result result : results) {
        int k = recordsInOneResult(result);
        wantedRecords -= k;
        counter += k;
      }
      if (wantedRecords <= 0) break;
    }
    scanner.close();
    totalTime += System.currentTimeMillis() - startTime;
  }
  OpResult ret = new OpResult("scan", counter, totalTime);
  br.close();
  table.close();
  return ret;
}
 
Example 9
Project: ditb   File: UDGScanner.java   Source Code and License Vote up 5 votes
private GCScanner createGCScanner(ScanRange selectedRange) throws IOException {
  List<ScanRange> list = new ArrayList<>(rangeList.getRanges());
  list.remove(selectedRange);
  Scan scan = new Scan();
  scan.setStartRow(selectedRange.getStart());
  scan.setStopRow(selectedRange.getStop());
  scan.setCaching(rawScan.getCaching());
  scan.setCacheBlocks(rawScan.getCacheBlocks());
  scan.setFilter(new ScanRange.ScanRangeList(list).toFilterList());
  Table table = conn.getTable(
      relation.getIndexTableName(selectedRange.getFamily(), selectedRange.getQualifier()));
  ResultScanner scanner = table.getScanner(scan);
  return new GCScanner(this, scanner, selectedRange.getFamily(), selectedRange.getQualifier());
}
 
Example 10
Project: ditb   File: PerfMD.java   Source Code and License Vote up 5 votes
@Override protected Result processGet(Table table, Get get) throws IOException {
  Scan scan = new Scan();
  scan.addFamily(MDHBaseAdmin.BUCKET_FAMILY);
  scan.setReversed(true);
  scan.setStartRow(get.getRow());
  scan.setCacheBlocks(false);
  scan.setCaching(1);
  scan.setSmall(true);
  ResultScanner scanner = table.getScanner(scan);
  Result ret = scanner.next();
  scanner.close();
  return ret;
}
 
Example 11
Project: ditb   File: HRegion.java   Source Code and License Vote up 5 votes
/**
 * create scan on memstore
 */
private Scan createScanOnMemstore(Scan rawScan, ScanRange.ScanRangeList allRangeList)
    throws IOException {
  Scan scan = new Scan(rawScan);
  scan.setCacheBlocks(false);
  scan.setFilter(allRangeList.toFilterList());
  return scan;
}
 
Example 12
Project: ditb   File: TestCorruptedRegionStoreFile.java   Source Code and License Vote up 5 votes
private int fullScanAndCount(final TableName tableName, final ScanInjector injector)
    throws Exception {
  Table table = UTIL.getConnection().getTable(tableName);
  int count = 0;
  try {
    Scan scan = new Scan();
    scan.setCaching(1);
    scan.setCacheBlocks(false);
    injector.beforeScan(table, scan);
    ResultScanner scanner = table.getScanner(scan);
    try {
      while (true) {
        injector.beforeScanNext(table);
        Result result = scanner.next();
        injector.afterScanNext(table, result);
        if (result == null) break;
        if ((count++ % (ROW_PER_FILE / 2)) == 0) {
          LOG.debug("scan next " + count);
        }
      }
    } finally {
      scanner.close();
      injector.afterScan(table);
    }
  } finally {
    table.close();
  }
  return count;
}
 
Example 13
Project: ditb   File: ScanPerformanceEvaluation.java   Source Code and License Vote up 5 votes
private Scan getScan() {
  Scan scan = new Scan(); // default scan settings
  scan.setCacheBlocks(false);
  scan.setMaxVersions(1);
  scan.setScanMetricsEnabled(true);
  if (caching != null) {
    scan.setCaching(Integer.parseInt(caching));
  }

  return scan;
}
 
Example 14
Project: ditb   File: TestSeekBeforeWithReverseScan.java   Source Code and License Vote up 5 votes
@Test
public void testReverseScanWithPadding() throws Exception {
  byte[] terminator = new byte[] { -1 };
  byte[] row1 = Bytes.add(invert(Bytes.toBytes("a")), terminator);
  byte[] row2 = Bytes.add(invert(Bytes.toBytes("ab")), terminator);
  byte[] row3 = Bytes.add(invert(Bytes.toBytes("b")), terminator);

  Put put1 = new Put(row1);
  put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY);
  Put put2 = new Put(row2);
  put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY);
  Put put3 = new Put(row3);
  put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY);

  region.put(put1);
  region.put(put2);
  region.put(put3);
  region.flush(true);
  Scan scan = new Scan();
  scan.setCacheBlocks(false);
  scan.setReversed(true);
  scan.setFilter(new FirstKeyOnlyFilter());
  scan.addFamily(cfName);
  RegionScanner scanner = region.getScanner(scan);
  List<Cell> res = new ArrayList<Cell>();
  int count = 1;
  while (scanner.next(res)) {
    count++;
  }
  assertEquals(3, count);
}
 
Example 15
Project: ditb   File: IntegrationTestBulkLoad.java   Source Code and License Vote up 5 votes
/**
 * After adding data to the table start a mr job to
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void runCheck() throws IOException, ClassNotFoundException, InterruptedException {
  LOG.info("Running check");
  Configuration conf = getConf();
  String jobName = getTablename() + "_check" + EnvironmentEdgeManager.currentTime();
  Path p = util.getDataTestDirOnTestFS(jobName);

  Job job = new Job(conf);
  job.setJarByClass(getClass());
  job.setJobName(jobName);

  job.setPartitionerClass(NaturalKeyPartitioner.class);
  job.setGroupingComparatorClass(NaturalKeyGroupingComparator.class);
  job.setSortComparatorClass(CompositeKeyComparator.class);

  Scan scan = new Scan();
  scan.addFamily(CHAIN_FAM);
  scan.addFamily(SORT_FAM);
  scan.setMaxVersions(1);
  scan.setCacheBlocks(false);
  scan.setBatch(1000);

  int replicaCount = conf.getInt(NUM_REPLICA_COUNT_KEY, NUM_REPLICA_COUNT_DEFAULT);
  if (replicaCount != NUM_REPLICA_COUNT_DEFAULT) {
    scan.setConsistency(Consistency.TIMELINE);
  }

  TableMapReduceUtil.initTableMapperJob(
      getTablename().getName(),
      scan,
      LinkedListCheckingMapper.class,
      LinkKey.class,
      LinkChain.class,
      job
  );

  job.setReducerClass(LinkedListCheckingReducer.class);
  job.setOutputKeyClass(NullWritable.class);
  job.setOutputValueClass(NullWritable.class);

  FileOutputFormat.setOutputPath(job, p);

  assertEquals(true, job.waitForCompletion(true));

  // Delete the files.
  util.getTestFileSystem().delete(p, true);
}
 
Example 16
Project: ditb   File: IntegrationTestBigLinkedListWithVisibility.java   Source Code and License Vote up 5 votes
private int doVerify(Path outputDir, int numReducers) throws IOException, InterruptedException,
    ClassNotFoundException {
  job = new Job(getConf());

  job.setJobName("Link Verifier");
  job.setNumReduceTasks(numReducers);
  job.setJarByClass(getClass());

  setJobScannerConf(job);

  Scan scan = new Scan();
  scan.addColumn(FAMILY_NAME, COLUMN_PREV);
  scan.setCaching(10000);
  scan.setCacheBlocks(false);
  String[] split = labels.split(COMMA);

  scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2],
      split[(this.labelIndex * 2) + 1]));

  TableMapReduceUtil.initTableMapperJob(tableName.getName(), scan, VerifyMapper.class,
      BytesWritable.class, BytesWritable.class, job);
  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);

  job.getConfiguration().setBoolean("mapreduce.map.speculative", false);

  job.setReducerClass(VerifyReducer.class);
  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, outputDir);
  boolean success = job.waitForCompletion(true);

  return success ? 0 : 1;
}
 
Example 17
Project: ditb   File: MultiThreadedClientExample.java   Source Code and License Vote up 4 votes
@Override
public Boolean call() throws Exception {

  // total length in bytes of all read rows.
  int result = 0;

  // Number of rows the scan will read before being considered done.
  int toRead = 100;
  try (Table t = connection.getTable(tableName)) {
    byte[] rk = Bytes.toBytes(ThreadLocalRandom.current().nextLong());
    Scan s = new Scan(rk);

    // This filter will keep the values from being sent accross the wire.
    // This is good for counting or other scans that are checking for
    // existence and don't rely on the value.
    s.setFilter(new KeyOnlyFilter());

    // Don't go back to the server for every single row.
    // We know these rows are small. So ask for 20 at a time.
    // This would be application specific.
    //
    // The goal is to reduce round trips but asking for too
    // many rows can lead to GC problems on client and server sides.
    s.setCaching(20);

    // Don't use the cache. While this is a silly test program it's still good to be
    // explicit that scans normally don't use the block cache.
    s.setCacheBlocks(false);

    // Open up the scanner and close it automatically when done.
    try (ResultScanner rs = t.getScanner(s)) {

      // Now go through rows.
      for (Result r : rs) {
        // Keep track of things size to simulate doing some real work.
        result += r.getRow().length;
        toRead -= 1;

        // Most online applications won't be
        // reading the entire table so this break
        // simulates small to medium size scans,
        // without needing to know an end row.
        if (toRead <= 0)  {
          break;
        }
      }
    }
  }
  return result > 0;
}
 
Example 18
Project: ditb   File: DITBScan.java   Source Code and License Vote up 4 votes
private ScanResult runOneTime(IndexType indexType, String fileName, Configuration conf,
    int cacheSize, AbstractWorkload workload) throws IOException {
  ScanRange.ScanRangeList rangeList = ScanRange.ScanRangeList.getScanRangeList(fileName);
  System.out.println("scan range in file " + fileName + " is: " + rangeList);
  DITBScanBase scanner;
  if (indexType == IndexType.MDIndex) {
    scanner = new DITBMDIndexScanner(workload.getTableName(), indexType, workload);
  } else {
    scanner = new DITBNormalScanner(workload.getTableName(), indexType, workload);
  }
  Scan scan = ScanRange.ScanRangeList.getScan(fileName);
  scan.setCacheBlocks(false);
  scan.setCaching(cacheSize);
  scan.setId(fileName);
  System.out.println("scan filter " + scan.getFilter());
  //    ScanResult scanResult = new ScanResult(fileName + ": " + rangeList);
  ScanResult scanResult = new ScanResult();
  int reportCount = 0;
  Result[] results;
  long timeStart = System.currentTimeMillis();
  ResultScanner resultScanner = scanner.getScanner(scan);
  try {
    while (true) {
      results = resultScanner.next(cacheSize);
      if (scanResult.firstLatency == -1)
        scanResult.firstLatency = System.currentTimeMillis() - timeStart;
      if (results == null || results.length == 0) break;
      if (PRINT_RESULT) {
        for (Result result : results)
          System.out.println(workload.parseResult(result));
      }
      scanResult.nbResults += results.length;
      reportCount += results.length;
      if (reportCount >= reportInterval) {
        System.out.println(
            "finish scan for " + results.length + " records, " + scanResult.nbResults
                + " in total");
        reportCount = 0;
      }
    }
  } catch (IOException e) {
    e.printStackTrace();
  }
  scanResult.totalTime = System.currentTimeMillis() - timeStart;
  if (scanResult.nbResults == 0) scanResult.nbResults = 1;
  return scanResult;
}
 
Example 19
Project: ditb   File: Export.java   Source Code and License Vote up 4 votes
private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
  Scan s = new Scan();
  // Optional arguments.
  // Set Scan Versions
  int versions = args.length > 2? Integer.parseInt(args[2]): 1;
  s.setMaxVersions(versions);
  // Set Scan Range
  long startTime = args.length > 3? Long.parseLong(args[3]): 0L;
  long endTime = args.length > 4? Long.parseLong(args[4]): Long.MAX_VALUE;
  s.setTimeRange(startTime, endTime);
  // Set cache blocks
  s.setCacheBlocks(false);
  // set Start and Stop row
  if (conf.get(TableInputFormat.SCAN_ROW_START) != null) {
    s.setStartRow(Bytes.toBytes(conf.get(TableInputFormat.SCAN_ROW_START)));
  }
  if (conf.get(TableInputFormat.SCAN_ROW_STOP) != null) {
    s.setStopRow(Bytes.toBytes(conf.get(TableInputFormat.SCAN_ROW_STOP)));
  }
  // Set Scan Column Family
  boolean raw = Boolean.parseBoolean(conf.get(RAW_SCAN));
  if (raw) {
    s.setRaw(raw);
  }
  
  if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
    s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
  }
  // Set RowFilter or Prefix Filter if applicable.
  Filter exportFilter = getExportFilter(args);
  if (exportFilter!= null) {
      LOG.info("Setting Scan Filter for Export.");
    s.setFilter(exportFilter);
  }

  int batching = conf.getInt(EXPORT_BATCHING, -1);
  if (batching !=  -1){
    try {
      s.setBatch(batching);
    } catch (IncompatibleFilterException e) {
      LOG.error("Batching could not be set", e);
    }
  }
  LOG.info("versions=" + versions + ", starttime=" + startTime +
    ", endtime=" + endTime + ", keepDeletedCells=" + raw);
  return s;
}
 
Example 20
Project: ditb   File: TestBlocksRead.java   Source Code and License Vote up 4 votes
/**
 * Test # of blocks read to ensure disabling cache-fill on Scan works.
 * @throws Exception
 */
@Test
public void testBlocksStoredWhenCachingDisabled() throws Exception {
  byte [] TABLE = Bytes.toBytes("testBlocksReadWhenCachingDisabled");
  String FAMILY = "cf1";

  HBaseConfiguration conf = getConf();
  this.region = initHRegion(TABLE, getName(), conf, FAMILY);

  try {
    putData(FAMILY, "row", "col1", 1);
    putData(FAMILY, "row", "col2", 2);
    region.flush(true);

    // Execute a scan with caching turned off
    // Expected blocks stored: 0
    long blocksStart = getBlkCount();
    Scan scan = new Scan();
    scan.setCacheBlocks(false);
    RegionScanner rs = region.getScanner(scan);
    List<Cell> result = new ArrayList<Cell>(2);
    rs.next(result);
    assertEquals(2 * BLOOM_TYPE.length, result.size());
    rs.close();
    long blocksEnd = getBlkCount();

    assertEquals(blocksStart, blocksEnd);

    // Execute with caching turned on
    // Expected blocks stored: 2
    blocksStart = blocksEnd;
    scan.setCacheBlocks(true);
    rs = region.getScanner(scan);
    result = new ArrayList<Cell>(2);
    rs.next(result);
    assertEquals(2 * BLOOM_TYPE.length, result.size());
    rs.close();
    blocksEnd = getBlkCount();
  
    assertEquals(2 * BLOOM_TYPE.length, blocksEnd - blocksStart);
  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }
}