Java Code Examples for org.apache.hadoop.hbase.client.Result.rawCells()

The following are Jave code examples for showing how to use rawCells() of the org.apache.hadoop.hbase.client.Result class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
+ Save this method
Example 1
Project: ditb   File: ThriftUtilities.java   View Source Code Vote up 6 votes
/**
 * Creates a {@link TResult} (Thrift) from a {@link Result} (HBase).
 *
 * @param in the <code>Result</code> to convert
 *
 * @return converted result, returns an empty result if the input is <code>null</code>
 */
public static TResult resultFromHBase(Result in) {
  Cell[] raw = in.rawCells();
  TResult out = new TResult();
  byte[] row = in.getRow();
  if (row != null) {
    out.setRow(in.getRow());
  }
  List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
  for (Cell kv : raw) {
    TColumnValue col = new TColumnValue();
    col.setFamily(CellUtil.cloneFamily(kv));
    col.setQualifier(CellUtil.cloneQualifier(kv));
    col.setTimestamp(kv.getTimestamp());
    col.setValue(CellUtil.cloneValue(kv));
    if (kv.getTagsLength() > 0) {
      col.setTags(CellUtil.getTagArray(kv));
    }
    columnValues.add(col);
  }
  out.setColumnValues(columnValues);
  return out;
}
 
Example 2
Project: ditb   File: AccessControlLists.java   View Source Code Vote up 6 votes
private static ListMultimap<String, TablePermission> parsePermissions(
    byte[] entryName, Result result) {
  ListMultimap<String, TablePermission> perms = ArrayListMultimap.create();
  if (result != null && result.size() > 0) {
    for (Cell kv : result.rawCells()) {

      Pair<String,TablePermission> permissionsOfUserOnTable =
          parsePermissionRecord(entryName, kv);

      if (permissionsOfUserOnTable != null) {
        String username = permissionsOfUserOnTable.getFirst();
        TablePermission permissions = permissionsOfUserOnTable.getSecond();
        perms.put(username, permissions);
      }
    }
  }
  return perms;
}
 
Example 3
Project: ditb   File: HashTable.java   View Source Code Vote up 6 votes
public void hashResult(Result result) {
  if (!batchStarted) {
    throw new RuntimeException("Cannot add to batch that has not been started.");
  }
  for (Cell cell : result.rawCells()) {
    int rowLength = cell.getRowLength();
    int familyLength = cell.getFamilyLength();
    int qualifierLength = cell.getQualifierLength();
    int valueLength = cell.getValueLength();
    digest.update(cell.getRowArray(), cell.getRowOffset(), rowLength);
    digest.update(cell.getFamilyArray(), cell.getFamilyOffset(), familyLength);
    digest.update(cell.getQualifierArray(), cell.getQualifierOffset(), qualifierLength);
    long ts = cell.getTimestamp();
    for (int i = 8; i > 0; i--) {
      digest.update((byte) ts);
      ts >>>= 8;
    }
    digest.update(cell.getValueArray(), cell.getValueOffset(), valueLength);
    
    batchSize += rowLength + familyLength + qualifierLength + 8 + valueLength;
  }
}
 
Example 4
Project: ditb   File: Import.java   View Source Code Vote up 6 votes
/**
 * @param row  The current table row key.
 * @param value  The columns.
 * @param context  The current context.
 * @throws IOException When something is broken with the data.
 */
@Override
public void map(ImmutableBytesWritable row, Result value,
  Context context)
throws IOException {
  try {
    if (LOG.isTraceEnabled()) {
      LOG.trace("Considering the row."
          + Bytes.toString(row.get(), row.getOffset(), row.getLength()));
    }
    if (filter == null || !filter.filterRowKey(row.get(), row.getOffset(), row.getLength())) {
      for (Cell kv : value.rawCells()) {
        kv = filterKv(filter, kv);
        // skip if we filtered it out
        if (kv == null) continue;
        // TODO get rid of ensureKeyValue
        context.write(row, KeyValueUtil.ensureKeyValueTypeForMR(convertKv(kv, cfRenameMap)));
      }
    }
  } catch (InterruptedException e) {
    e.printStackTrace();
  }
}
 
Example 5
Project: ditb   File: RSRpcServices.java   View Source Code Vote up 6 votes
/**
 * Method to account for the size of retained cells and retained data blocks.
 * @return an object that represents the last referenced block from this response.
 */
Object addSize(RpcCallContext context, Result r, Object lastBlock) {
  if (context != null && !r.isEmpty()) {
    for (Cell c : r.rawCells()) {
      context.incrementResponseCellSize(CellUtil.estimatedHeapSizeOf(c));
      // We're using the last block being the same as the current block as
      // a proxy for pointing to a new block. This won't be exact.
      // If there are multiple gets that bounce back and forth
      // Then it's possible that this will over count the size of
      // referenced blocks. However it's better to over count and
      // use two RPC's than to OOME the RegionServer.
      byte[] valueArray = c.getValueArray();
      if (valueArray != lastBlock) {
        context.incrementResponseBlockSize(valueArray.length);
        lastBlock = valueArray;
      }
    }
  }
  return lastBlock;
}
 
Example 6
Project: ditb   File: TestHRegion.java   View Source Code Vote up 6 votes
static void verifyData(HRegion newReg, int startRow, int numRows, byte[] qf, byte[]... families)
    throws IOException {
  for (int i = startRow; i < startRow + numRows; i++) {
    byte[] row = Bytes.toBytes("" + i);
    Get get = new Get(row);
    for (byte[] family : families) {
      get.addColumn(family, qf);
    }
    Result result = newReg.get(get);
    Cell[] raw = result.rawCells();
    assertEquals(families.length, result.size());
    for (int j = 0; j < families.length; j++) {
      assertTrue(CellUtil.matchingRow(raw[j], row));
      assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
      assertTrue(CellUtil.matchingQualifier(raw[j], qf));
    }
  }
}
 
Example 7
Project: ditb   File: TestAtomicOperation.java   View Source Code Vote up 6 votes
private void assertICV(byte [] row,
                       byte [] familiy,
                       byte[] qualifier,
                       long amount) throws IOException {
  // run a get and see?
  Get get = new Get(row);
  get.addColumn(familiy, qualifier);
  Result result = region.get(get);
  assertEquals(1, result.size());

  Cell kv = result.rawCells()[0];
  long r = Bytes.toLong(CellUtil.cloneValue(kv));
  assertEquals(amount, r);
}
 
Example 8
Project: ditb   File: TestPartialResultsFromClientSide.java   View Source Code Vote up 6 votes
/**
 * Verifies that result contains all the key values within expKvList. Fails the test otherwise
 * @param result
 * @param expKvList
 * @param msg
 */
static void verifyResult(Result result, List<Cell> expKvList, String msg) {
  if (LOG.isInfoEnabled()) {
    LOG.info(msg);
    LOG.info("Expected count: " + expKvList.size());
    LOG.info("Actual count: " + result.size());
  }

  if (expKvList.size() == 0) return;

  int i = 0;
  for (Cell kv : result.rawCells()) {
    if (i >= expKvList.size()) {
      break; // we will check the size later
    }

    Cell kvExp = expKvList.get(i++);
    assertTrue("Not equal. get kv: " + kv.toString() + " exp kv: " + kvExp.toString(),
        kvExp.equals(kv));
  }

  assertEquals(expKvList.size(), result.size());
}
 
Example 9
Project: mumu-hbase   File: HBaseResultUtil.java   View Source Code Vote up 5 votes
/**
 * 格式化输出
 *
 * @param results
 */
public static void print(Result... results) {
    log.info("查询结果:");
    for (Result result : results) {
        Cell[] cells = result.rawCells();
        for (Cell cell : cells) {
            log.info("rowkey:" + new String(CellUtil.cloneRow(cell)) + ","
                    + "columnFamily:" + new String(CellUtil.cloneFamily(cell)) + ","
                    + "qualifier:" + new String(CellUtil.cloneQualifier(cell)) + ","
                    + "Timetamp:" + cell.getTimestamp() + ","
                    + "value:" + new String(CellUtil.cloneValue(cell)) + ",");
        }
    }
}
 
Example 10
Project: ditb   File: TestPartialResultsFromClientSide.java   View Source Code Vote up 5 votes
@Test
public void testReversedPartialResultWhenRegionMove() throws IOException {
  Table table=createTestTable(TableName.valueOf("testReversedPartialResultWhenRegionMove"),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  scan.setReversed(true);
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS-1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
  assertFalse(result1.isPartial());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]);
  assertTrue(result2.isPartial());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[1]);
  assertTrue(result3.isPartial());

}
 
Example 11
Project: ditb   File: QuotaUtil.java   View Source Code Vote up 5 votes
public static long calculateResultSize(final Result result) {
  long size = 0;
  for (Cell cell : result.rawCells()) {
    size += KeyValueUtil.length(cell);
  }
  return size;
}
 
Example 12
Project: ditb   File: QuotaUtil.java   View Source Code Vote up 5 votes
public static long calculateResultSize(final List<Result> results) {
  long size = 0;
  for (Result result : results) {
    for (Cell cell : result.rawCells()) {
      size += KeyValueUtil.length(cell);
    }
  }
  return size;
}
 
Example 13
Project: ditb   File: TestPartialResultsFromClientSide.java   View Source Code Vote up 5 votes
@Test
public void testReversedCompleteResultWhenRegionMove() throws IOException {
  Table table=createTestTable(TableName.valueOf("testReversedCompleteResultWhenRegionMove"),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setCaching(1);
  scan.setReversed(true);
  ResultScanner scanner = table.getScanner(scan);

  Result result1 = scanner.next();
  assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[NUM_ROWS-1], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result1.isPartial());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[NUM_ROWS-2], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result2.isPartial());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(NUM_FAMILIES*NUM_QUALIFIERS, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[NUM_ROWS-3], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result3.isPartial());

}
 
Example 14
Project: ditb   File: TestParallelPut.java   View Source Code Vote up 5 votes
private static void assertGet(final HRegion region, byte [] row, byte [] familiy,
    byte[] qualifier, byte[] value) throws IOException {
  // run a get and see if the value matches
  Get get = new Get(row);
  get.addColumn(familiy, qualifier);
  Result result = region.get(get);
  assertEquals(1, result.size());

  Cell kv = result.rawCells()[0];
  byte[] r = CellUtil.cloneValue(kv);
  assertTrue(Bytes.compareTo(r, value) == 0);
}
 
Example 15
Project: ditb   File: TestPartialResultsFromClientSide.java   View Source Code Vote up 5 votes
/**
 * Exhausts the scanner by calling next repetitively. Once completely exhausted, close scanner and
 * return total cell count
 * @param scanner
 * @return
 * @throws Exception
 */
private int countCellsFromScanner(ResultScanner scanner) throws Exception {
  Result result = null;
  int numCells = 0;
  while ((result = scanner.next()) != null) {
    numCells += result.rawCells().length;
  }

  scanner.close();
  return numCells;
}
 
Example 16
Project: ditb   File: TestResettingCounters.java   View Source Code Vote up 5 votes
@Test
public void testResettingCounters() throws Exception {

  HBaseTestingUtility htu = new HBaseTestingUtility();
  Configuration conf = htu.getConfiguration();
  FileSystem fs = FileSystem.get(conf);
  byte [] table = Bytes.toBytes("table");
  byte [][] families = new byte [][] {
      Bytes.toBytes("family1"),
      Bytes.toBytes("family2"),
      Bytes.toBytes("family3")
  };
  int numQualifiers = 10;
  byte [][] qualifiers = new byte [numQualifiers][];
  for (int i=0; i<numQualifiers; i++) qualifiers[i] = Bytes.toBytes("qf" + i);
  int numRows = 10;
  byte [][] rows = new byte [numRows][];
  for (int i=0; i<numRows; i++) rows[i] = Bytes.toBytes("r" + i);

  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(table));
  for (byte [] family : families) htd.addFamily(new HColumnDescriptor(family));

  HRegionInfo hri = new HRegionInfo(htd.getTableName(), null, null, false);
  String testDir = htu.getDataTestDir() + "/TestResettingCounters/";
  Path path = new Path(testDir);
  if (fs.exists(path)) {
    if (!fs.delete(path, true)) {
      throw new IOException("Failed delete of " + path);
    }
  }
  HRegion region = HRegion.createHRegion(hri, path, conf, htd);
  try {
    Increment odd = new Increment(rows[0]);
    odd.setDurability(Durability.SKIP_WAL);
    Increment even = new Increment(rows[0]);
    even.setDurability(Durability.SKIP_WAL);
    Increment all = new Increment(rows[0]);
    all.setDurability(Durability.SKIP_WAL);
    for (int i=0;i<numQualifiers;i++) {
      if (i % 2 == 0) even.addColumn(families[0], qualifiers[i], 1);
      else odd.addColumn(families[0], qualifiers[i], 1);
      all.addColumn(families[0], qualifiers[i], 1);
    }

    // increment odd qualifiers 5 times and flush
    for (int i=0;i<5;i++) region.increment(odd, HConstants.NO_NONCE, HConstants.NO_NONCE);
    region.flush(true);

    // increment even qualifiers 5 times
    for (int i=0;i<5;i++) region.increment(even, HConstants.NO_NONCE, HConstants.NO_NONCE);

    // increment all qualifiers, should have value=6 for all
    Result result = region.increment(all, HConstants.NO_NONCE, HConstants.NO_NONCE);
    assertEquals(numQualifiers, result.size());
    Cell [] kvs = result.rawCells();
    for (int i=0;i<kvs.length;i++) {
      System.out.println(kvs[i].toString());
      assertTrue(CellUtil.matchingQualifier(kvs[i], qualifiers[i]));
      assertEquals(6, Bytes.toLong(CellUtil.cloneValue(kvs[i])));
    }
  } finally {
    HRegion.closeHRegion(region);
  }
  HRegion.closeHRegion(region);
}
 
Example 17
Project: ditb   File: TestPartialResultsFromClientSide.java   View Source Code Vote up 5 votes
public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult)
    throws Exception {
  if (LOG.isInfoEnabled()) {
    LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult);
  }

  Scan scan = new Scan();
  scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult));
  scan.setBatch(batch);
  ResultScanner scanner = TABLE.getScanner(scan);
  Result result = scanner.next();
  int repCount = 0;

  while ((result = scanner.next()) != null) {
    assertTrue(result.rawCells() != null);

    if (result.isPartial()) {
      final String error =
          "Cells:" + result.rawCells().length + " Batch size:" + batch
              + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount;
      assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult));
    } else {
      assertTrue(result.rawCells().length <= batch);
    }
    repCount++;
  }

  scanner.close();
}
 
Example 18
Project: ditb   File: ScanPerformanceEvaluation.java   View Source Code Vote up 4 votes
public void testScan() throws IOException {
  Stopwatch tableOpenTimer = new Stopwatch();
  Stopwatch scanOpenTimer = new Stopwatch();
  Stopwatch scanTimer = new Stopwatch();

  tableOpenTimer.start();
  Table table = new HTable(getConf(), TableName.valueOf(tablename));
  tableOpenTimer.stop();

  Scan scan = getScan();
  scanOpenTimer.start();
  ResultScanner scanner = table.getScanner(scan);
  scanOpenTimer.stop();

  long numRows = 0;
  long numCells = 0;
  scanTimer.start();
  while (true) {
    Result result = scanner.next();
    if (result == null) {
      break;
    }
    numRows++;

    numCells += result.rawCells().length;
  }
  scanTimer.stop();
  scanner.close();
  table.close();

  ScanMetrics metrics = scan.getScanMetrics();
  long totalBytes = metrics.countOfBytesInResults.get();
  double throughput = (double)totalBytes / scanTimer.elapsedTime(TimeUnit.SECONDS);
  double throughputRows = (double)numRows / scanTimer.elapsedTime(TimeUnit.SECONDS);
  double throughputCells = (double)numCells / scanTimer.elapsedTime(TimeUnit.SECONDS);

  System.out.println("HBase scan: ");
  System.out.println("total time to open table: " + tableOpenTimer.elapsedMillis() + " ms");
  System.out.println("total time to open scanner: " + scanOpenTimer.elapsedMillis() + " ms");
  System.out.println("total time to scan: " + scanTimer.elapsedMillis() + " ms");

  System.out.println("Scan metrics:\n" + metrics.getMetricsMap());

  System.out.println("total bytes: " + totalBytes + " bytes ("
      + StringUtils.humanReadableInt(totalBytes) + ")");
  System.out.println("throughput  : " + StringUtils.humanReadableInt((long)throughput) + "B/s");
  System.out.println("total rows  : " + numRows);
  System.out.println("throughput  : " + StringUtils.humanReadableInt((long)throughputRows) + " rows/s");
  System.out.println("total cells : " + numCells);
  System.out.println("throughput  : " + StringUtils.humanReadableInt((long)throughputCells) + " cells/s");
}
 
Example 19
Project: ditb   File: ScanPerformanceEvaluation.java   View Source Code Vote up 4 votes
public void testSnapshotScan() throws IOException {
  Stopwatch snapshotRestoreTimer = new Stopwatch();
  Stopwatch scanOpenTimer = new Stopwatch();
  Stopwatch scanTimer = new Stopwatch();

  Path restoreDir = new Path(this.restoreDir);

  snapshotRestoreTimer.start();
  restoreDir.getFileSystem(conf).delete(restoreDir, true);
  snapshotRestoreTimer.stop();

  Scan scan = getScan();
  scanOpenTimer.start();
  TableSnapshotScanner scanner = new TableSnapshotScanner(conf, restoreDir, snapshotName, scan);
  scanOpenTimer.stop();

  long numRows = 0;
  long numCells = 0;
  scanTimer.start();
  while (true) {
    Result result = scanner.next();
    if (result == null) {
      break;
    }
    numRows++;

    numCells += result.rawCells().length;
  }
  scanTimer.stop();
  scanner.close();

  ScanMetrics metrics = scanner.getScanMetrics();
  long totalBytes = metrics.countOfBytesInResults.get();
  double throughput = (double)totalBytes / scanTimer.elapsedTime(TimeUnit.SECONDS);
  double throughputRows = (double)numRows / scanTimer.elapsedTime(TimeUnit.SECONDS);
  double throughputCells = (double)numCells / scanTimer.elapsedTime(TimeUnit.SECONDS);

  System.out.println("HBase scan snapshot: ");
  System.out.println("total time to restore snapshot: " + snapshotRestoreTimer.elapsedMillis() + " ms");
  System.out.println("total time to open scanner: " + scanOpenTimer.elapsedMillis() + " ms");
  System.out.println("total time to scan: " + scanTimer.elapsedMillis() + " ms");

  System.out.println("Scan metrics:\n" + metrics.getMetricsMap());

  System.out.println("total bytes: " + totalBytes + " bytes ("
      + StringUtils.humanReadableInt(totalBytes) + ")");
  System.out.println("throughput  : " + StringUtils.humanReadableInt((long)throughput) + "B/s");
  System.out.println("total rows  : " + numRows);
  System.out.println("throughput  : " + StringUtils.humanReadableInt((long)throughputRows) + " rows/s");
  System.out.println("total cells : " + numCells);
  System.out.println("throughput  : " + StringUtils.humanReadableInt((long)throughputCells) + " cells/s");

}
 
Example 20
Project: ditb   File: TestPartialResultsFromClientSide.java   View Source Code Vote up 4 votes
public void testOrderingOfCellsInPartialResults(final Scan basePartialScan) throws Exception {
  // Scan that retrieves results in pieces (partials). By setting allowPartialResults to be true
  // the results will NOT be reconstructed and instead the caller will see the partial results
  // returned by the server
  Scan partialScan = new Scan(basePartialScan);
  partialScan.setAllowPartialResults(true);
  ResultScanner partialScanner = TABLE.getScanner(partialScan);

  // Scan that retrieves all table results in single RPC request
  Scan oneShotScan = new Scan(basePartialScan);
  oneShotScan.setMaxResultSize(Long.MAX_VALUE);
  oneShotScan.setCaching(ROWS.length);
  ResultScanner oneShotScanner = TABLE.getScanner(oneShotScan);

  Result oneShotResult = oneShotScanner.next();
  Result partialResult = null;
  int iterationCount = 0;

  while (oneShotResult != null && oneShotResult.rawCells() != null) {
    List<Cell> aggregatePartialCells = new ArrayList<Cell>();
    do {
      partialResult = partialScanner.next();
      assertTrue("Partial Result is null. iteration: " + iterationCount, partialResult != null);
      assertTrue("Partial cells are null. iteration: " + iterationCount,
          partialResult.rawCells() != null);

      for (Cell c : partialResult.rawCells()) {
        aggregatePartialCells.add(c);
      }
    } while (partialResult.isPartial());

    assertTrue("Number of cells differs. iteration: " + iterationCount,
        oneShotResult.rawCells().length == aggregatePartialCells.size());
    final Cell[] oneShotCells = oneShotResult.rawCells();
    for (int cell = 0; cell < oneShotCells.length; cell++) {
      Cell oneShotCell = oneShotCells[cell];
      Cell partialCell = aggregatePartialCells.get(cell);

      assertTrue("One shot cell was null", oneShotCell != null);
      assertTrue("Partial cell was null", partialCell != null);
      assertTrue("Cell differs. oneShotCell:" + oneShotCell + " partialCell:" + partialCell,
          oneShotCell.equals(partialCell));
    }

    oneShotResult = oneShotScanner.next();
    iterationCount++;
  }

  assertTrue(partialScanner.next() == null);

  partialScanner.close();
  oneShotScanner.close();
}