Java Code Examples for org.apache.hadoop.hbase.client.ResultScanner

The following are top voted examples for showing how to use org.apache.hadoop.hbase.client.ResultScanner. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ditb   File: TestTimeRangeMapRed.java   Source Code and License 8 votes vote down vote up
private void verify(final Table table) throws IOException {
  Scan scan = new Scan();
  scan.addColumn(FAMILY_NAME, COLUMN_NAME);
  scan.setMaxVersions(1);
  ResultScanner scanner = table.getScanner(scan);
  for (Result r: scanner) {
    for (Cell kv : r.listCells()) {
      log.debug(Bytes.toString(r.getRow()) + "\t" + Bytes.toString(CellUtil.cloneFamily(kv))
          + "\t" + Bytes.toString(CellUtil.cloneQualifier(kv))
          + "\t" + kv.getTimestamp() + "\t" + Bytes.toBoolean(CellUtil.cloneValue(kv)));
      org.junit.Assert.assertEquals(TIMESTAMP.get(kv.getTimestamp()),
        (Boolean)Bytes.toBoolean(CellUtil.cloneValue(kv)));
    }
  }
  scanner.close();
}
 
Example 2
Project: SparkDemo   File: MyClass.java   Source Code and License 6 votes vote down vote up
public static void QueryAll(String tableName) {
    try {
        HTableInterface table = conn.getTable(tableName);

        ResultScanner rs = table.getScanner(new Scan());
        for (Result r : rs) {
            System.out.println("rowkey:" + new String(r.getRow()));
            for (KeyValue keyValue : r.raw()) {
                System.out.println("column:" + new String(keyValue.getFamily())
                        + "====value:" + new String(keyValue.getValue()));
            }
        }
        table.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 3
Project: SparkDemo   File: MyClass.java   Source Code and License 6 votes vote down vote up
public static void QueryByCondition2(String tableName) {

        try {
            HTablePool pool = new HTablePool(configuration, 1000);
            HTable table = (HTable) pool.getTable(tableName);
            Filter filter = new SingleColumnValueFilter(Bytes
                    .toBytes("column1"), null, CompareOp.EQUAL, Bytes
                    .toBytes("aaa")); // 当列column1的值为aaa时进行查询
            Scan s = new Scan();
            s.setFilter(filter);
            ResultScanner rs = table.getScanner(s);
            for (Result r : rs) {
                System.out.println("获得到rowkey:" + new String(r.getRow()));
                for (KeyValue keyValue : r.raw()) {
                    System.out.println("列:" + new String(keyValue.getFamily())
                            + "====值:" + new String(keyValue.getValue()));
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
 
Example 4
Project: ditb   File: MetaTableAccessor.java   Source Code and License 6 votes vote down vote up
/**
 * Performs a full scan of a catalog table.
 * @param connection connection we're using
 * @param visitor Visitor invoked against each row.
 * @param startrow Where to start the scan. Pass null if want to begin scan
 * at first row.
 * <code>hbase:meta</code>, the default (pass false to scan hbase:meta)
 * @throws IOException
 */
public static void fullScan(Connection connection,
  final Visitor visitor, final byte [] startrow)
throws IOException {
  Scan scan = new Scan();
  if (startrow != null) scan.setStartRow(startrow);
  if (startrow == null) {
    int caching = connection.getConfiguration()
        .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
    scan.setCaching(caching);
  }
  scan.addFamily(HConstants.CATALOG_FAMILY);
  Table metaTable = getMetaHTable(connection);
  ResultScanner scanner = null;
  try {
    scanner = metaTable.getScanner(scan);
    Result data;
    while((data = scanner.next()) != null) {
      if (data.isEmpty()) continue;
      // Break if visit returns false.
      if (!visitor.visit(data)) break;
    }
  } finally {
    if (scanner != null) scanner.close();
    metaTable.close();
  }
}
 
Example 5
Project: ditb   File: TestZKBasedOpenCloseRegion.java   Source Code and License 6 votes vote down vote up
private static void waitUntilAllRegionsAssigned()
throws IOException {
  HTable meta = new HTable(TEST_UTIL.getConfiguration(), TableName.META_TABLE_NAME);
  while (true) {
    int rows = 0;
    Scan scan = new Scan();
    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
    ResultScanner s = meta.getScanner(scan);
    for (Result r = null; (r = s.next()) != null;) {
      byte [] b =
        r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
      if (b == null || b.length <= 0) {
        break;
      }
      rows++;
    }
    s.close();
    // If I get to here and all rows have a Server, then all have been assigned.
    if (rows >= countOfRegions) {
      break;
    }
    LOG.info("Found=" + rows);
    Threads.sleep(1000);
  }
  meta.close();
}
 
Example 6
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 6 votes vote down vote up
public void testEquivalenceOfScanResults(Table table, Scan scan1, Scan scan2) throws Exception {
  ResultScanner scanner1 = table.getScanner(scan1);
  ResultScanner scanner2 = table.getScanner(scan2);

  Result r1 = null;
  Result r2 = null;
  int count = 0;

  while ((r1 = scanner1.next()) != null) {
    r2 = scanner2.next();

    assertTrue(r2 != null);
    compareResults(r1, r2, "Comparing result #" + count);
    count++;
  }

  r2 = scanner2.next();
  assertTrue("r2: " + r2 + " Should be null", r2 == null);

  scanner1.close();
  scanner2.close();
}
 
Example 7
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License 6 votes vote down vote up
/**
 * 列族过滤器
 *
 * @param tableName 表名
 * @param rowFamily 列族
 * @param count     数量
 */
public void familyFilter(String tableName, String rowFamily, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    //使用列族过滤器
    //scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER, new BinaryComparator(Bytes.toBytes(rowFamily))));//直接行健
    //scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new RegexStringComparator("row.*")));//正则表达式
    //scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new SubstringComparator("row")));//字符串包含
    scan.setFilter(new FamilyFilter(CompareFilter.CompareOp.GREATER_OR_EQUAL, new BinaryPrefixComparator("mm".getBytes())));//字符串前缀
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 8
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License 6 votes vote down vote up
/**
 * 列限定符过滤器
 *
 * @param tableName  表名
 * @param columnName 列限定符
 * @param count      数量
 */
public void qualifierFilter(String tableName, String columnName, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    //使用列族过滤器
    scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes(columnName))));//直接行健
    //scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new RegexStringComparator("row.*")));//正则表达式
    //scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("row")));//字符串包含
    //scan.setFilter(new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryPrefixComparator("m".getBytes())));//字符串前缀
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 9
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License 6 votes vote down vote up
/**
 * 参考列过滤器(获取相同时间戳的列)
 *
 * @param tableName    表名
 * @param columnFamily 列族
 * @param qualifier    列限定符
 * @param columnValue  列值
 * @param count        数量
 */
public void dependentColumnFilter(String tableName, String columnFamily, String qualifier, String columnValue, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    scan.setFilter(new PrefixFilter(Bytes.toBytes("")));
    scan.setCaching(10);
    scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 10
Project: mumu-hbase   File: HBaseFilterOperation.java   Source Code and License 6 votes vote down vote up
/**
 * 单列排除过滤器(返回的列 不包含参考列)
 *
 * @param tableName    表名
 * @param columnFamily 列族
 * @param qualifier    列限定符
 * @param columnValue  列值
 * @param count        数量
 */
public void SingleColumnValueExcludeFilter(String tableName, String columnFamily, String qualifier, String columnValue, int count) {
    HBaseConfiguration hBaseConfiguration = new HBaseConfiguration();
    Table table = hBaseConfiguration.table(tableName);
    Scan scan = new Scan();
    SingleColumnValueExcludeFilter singleColumnValueFilter = new SingleColumnValueExcludeFilter(Bytes.toBytes(columnFamily), Bytes.toBytes(qualifier), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(columnValue));
    //singleColumnValueFilter.setFilterIfMissing(true);//当不存在这列的行 默认不过滤
    singleColumnValueFilter.setLatestVersionOnly(true);//获取最新版本
    scan.setFilter(singleColumnValueFilter);
    scan.setCaching(10);
    //scan.setBatch(10);
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result[] results = scanner.next(count);
        HBaseResultUtil.print(results);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 11
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 6 votes vote down vote up
/**
 * When a scan has a filter where {@link org.apache.hadoop.hbase.filter.Filter#hasFilterRow()} is
 * true, the scanner should not return partial results. The scanner cannot return partial results
 * because the entire row needs to be read for the include/exclude decision to be made
 */
@Test
public void testNoPartialResultsWhenRowFilterPresent() throws Exception {
  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  // If a filter hasFilter() is true then partial results should not be returned else filter
  // application server side would break.
  scan.setFilter(new RandomRowFilter(1.0f));
  ResultScanner scanner = TABLE.getScanner(scan);

  Result r = null;
  while ((r = scanner.next()) != null) {
    assertFalse(r.isPartial());
  }

  scanner.close();
}
 
Example 12
Project: SparkIsax   File: HBaseUtils.java   Source Code and License 6 votes vote down vote up
/**
 * 拷贝表
 * 
 * @throws IOException
 */
public static void copyTable(String oldTableName, String newTableName,String ColumnFamily, String ColumnName)throws IOException {
	if(CreateNewTable(newTableName))
		logger.info("创建表"+newTableName+"表成功");
	else{
		logger.info("创建表"+newTableName+"表失败");
	}
	Scan s = new Scan();
	s.addColumn(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName));
	s.setMaxVersions(1);
	s.setCacheBlocks(false);
	ResultScanner rs = hbase_table.getScanner(s);
	
	HTableInterface hbase_table_new = conn.getTable(newTableName);
	for (Result r : rs) {
		byte[] key = r.getRow();
		byte[] value = r.getValue(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName));
		Put put = new Put(key);
		put.add(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName), value);
		hbase_table_new.put(put);
	}
	rs.close();
	hbase_table_new.close();
}
 
Example 13
Project: ditb   File: TestRegionObserverInterface.java   Source Code and License 6 votes vote down vote up
@Test(timeout = 300000)
public void testHBASE14489() throws IOException {
  TableName tableName = TableName.valueOf("testHBASE14489");
  HTable table = util.createTable(tableName, new byte[][] { A });
  Put put = new Put(ROW);
  put.addColumn(A, A, A);
  table.put(put);

  Scan s = new Scan();
  s.setFilter(new FilterAllFilter());
  ResultScanner scanner = table.getScanner(s);
  try {
    for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
    }
  } finally {
    scanner.close();
  }
  verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" },
    tableName, new Boolean[] { true });
  util.deleteTable(tableName);
  table.close();
}
 
Example 14
Project: easyhbase   File: HbaseTemplate2.java   Source Code and License 6 votes vote down vote up
@Override
public <T> List<T> find(TableName tableName, final List<Scan> scanList, final
ResultsExtractor<T> action) {
    assertAccessAvailable();
    return execute(tableName, new TableCallback<List<T>>() {
        @Override
        public List<T> doInTable(Table table) throws Throwable {
            List<T> result = new ArrayList<>(scanList.size());
            for (Scan scan : scanList) {
                final ResultScanner scanner = table.getScanner(scan);
                try {
                    T t = action.extractData(scanner);
                    result.add(t);
                } finally {
                    scanner.close();
                }
            }
            return result;
        }
    });
}
 
Example 15
Project: easyhbase   File: HbaseTemplate2.java   Source Code and License 6 votes vote down vote up
private ResultScanner[] splitScan(Table table, Scan originalScan, AbstractRowKeyDistributor
        rowKeyDistributor) throws IOException {
    Scan[] scans = rowKeyDistributor.getDistributedScans(originalScan);
    final int length = scans.length;
    for (int i = 0; i < length; i++) {
        Scan scan = scans[i];
        // other properties are already set upon construction
        scan.setId(scan.getId() + "-" + i);
    }

    ResultScanner[] scanners = new ResultScanner[length];
    boolean success = false;
    try {
        for (int i = 0; i < length; i++) {
            scanners[i] = table.getScanner(scans[i]);
        }
        success = true;
    } finally {
        if (!success) {
            closeScanner(scanners);
        }
    }
    return scanners;
}
 
Example 16
Project: ditb   File: LMDTester.java   Source Code and License 6 votes vote down vote up
private void doRawScan() throws IOException {
  FilterList filterList = new FilterList();
  CompareFilter.CompareOp startOp = CompareFilter.CompareOp.GREATER_OR_EQUAL;
  CompareFilter.CompareOp stopOp = CompareFilter.CompareOp.LESS_OR_EQUAL;
  for (int i = 0; i < indexColumnNames.length && i < scanValues.length; i++) {
    filterList.addFilter(
        new SingleColumnValueFilter(familyName, Bytes.toBytes(indexColumnNames[i]), startOp,
            Bytes.toBytes(scanValues[i][0])));
    filterList.addFilter(
        new SingleColumnValueFilter(familyName, Bytes.toBytes(indexColumnNames[i]), stopOp,
            Bytes.toBytes(scanValues[i][1])));
  }
  Scan scan = new Scan();
  scan.setFilter(filterList);
  scan.setId("raw-scan");
  Table table = conn.getTable(tableName);
  ResultScanner scanner = table.getScanner(scan);
  Result result;
  int count = 0;
  while ((result = scanner.next()) != null) {
    ++count;
    if (PRINT_RESULT) printResult(result);
  }
  scanner.close();
  System.out.println("raw scan has " + count + " records");
}
 
Example 17
Project: ditb   File: TestScanRowPrefix.java   Source Code and License 6 votes vote down vote up
private void verifyScanResult(Table table, Scan scan, List<byte[]> expectedKeys, String message) {
  List<byte[]> actualKeys = new ArrayList<>();
  try {
    ResultScanner scanner = table.getScanner(scan);
    for (Result result : scanner) {
      actualKeys.add(result.getRow());
    }

    String fullMessage = message;
    if (LOG.isDebugEnabled()) {
      fullMessage = message + "\n" + tableOfTwoListsOfByteArrays(
              "Expected", expectedKeys,
              "Actual  ", actualKeys);
    }

    Assert.assertArrayEquals(
            fullMessage,
            expectedKeys.toArray(),
            actualKeys.toArray());
  } catch (IOException e) {
    e.printStackTrace();
    Assert.fail();
  }
}
 
Example 18
Project: ditb   File: ThriftHBaseServiceHandler.java   Source Code and License 6 votes vote down vote up
@Override
public List<TResult> getScannerResults(ByteBuffer table, TScan scan, int numRows)
    throws TIOError, TException {
  Table htable = getTable(table);
  List<TResult> results = null;
  ResultScanner scanner = null;
  try {
    scanner = htable.getScanner(scanFromThrift(scan));
    results = resultsFromHBase(scanner.next(numRows));
  } catch (IOException e) {
    throw getTIOError(e);
  } finally {
    if (scanner != null) {
      scanner.close();
    }
    closeTable(htable);
  }
  return results;
}
 
Example 19
Project: ditb   File: AccessControlLists.java   Source Code and License 6 votes vote down vote up
/**
 * Load all permissions from the region server holding {@code _acl_},
 * primarily intended for testing purposes.
 */
static Map<byte[], ListMultimap<String,TablePermission>> loadAll(
    Configuration conf) throws IOException {
  Map<byte[], ListMultimap<String,TablePermission>> allPerms =
      new TreeMap<byte[], ListMultimap<String,TablePermission>>(Bytes.BYTES_RAWCOMPARATOR);

  // do a full scan of _acl_, filtering on only first table region rows

  Scan scan = new Scan();
  scan.addFamily(ACL_LIST_FAMILY);

  ResultScanner scanner = null;
  // TODO: Pass in a Connection rather than create one each time.
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    try (Table table = connection.getTable(ACL_TABLE_NAME)) {
      scanner = table.getScanner(scan);
      try {
        for (Result row : scanner) {
          ListMultimap<String,TablePermission> resultPerms = parsePermissions(row.getRow(), row);
          allPerms.put(row.getRow(), resultPerms);
        }
      } finally {
        if (scanner != null) scanner.close();
      }
    }
  }

  return allPerms;
}
 
Example 20
Project: ditb   File: TestVisibilityLabelsWithDefaultVisLabelService.java   Source Code and License 6 votes vote down vote up
@Test(timeout = 60 * 1000)
public void testVisibilityLabelsOnWALReplay() throws Exception {
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = createTableAndWriteDataWithLabels(tableName,
      "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE);) {
    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
        .getRegionServerThreads();
    for (RegionServerThread rsThread : regionServerThreads) {
      rsThread.getRegionServer().abort("Aborting ");
    }
    // Start one new RS
    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
    waitForLabelsRegionAvailability(rs.getRegionServer());
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(SECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 1);
  }
}
 
Example 21
Project: ditb   File: IntegrationTestBigLinkedList.java   Source Code and License 6 votes vote down vote up
private static CINode findStartNode(Table table, byte[] startKey) throws IOException {
  Scan scan = new Scan();
  scan.setStartRow(startKey);
  scan.setBatch(1);
  scan.addColumn(FAMILY_NAME, COLUMN_PREV);

  long t1 = System.currentTimeMillis();
  ResultScanner scanner = table.getScanner(scan);
  Result result = scanner.next();
  long t2 = System.currentTimeMillis();
  scanner.close();

  if ( result != null) {
    CINode node = getCINode(result, new CINode());
    System.out.printf("FSR %d %s\n", t2 - t1, Bytes.toStringBinary(node.key));
    return node;
  }

  System.out.println("FSR " + (t2 - t1));

  return null;
}
 
Example 22
Project: ditb   File: TestReplicationSink.java   Source Code and License 6 votes vote down vote up
/**
 * Insert a mix of puts and deletes
 * @throws Exception
 */
@Test
public void testMixedPutDelete() throws Exception {
  List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE/2);
  List<Cell> cells = new ArrayList<Cell>();
  for(int i = 0; i < BATCH_SIZE/2; i++) {
    entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
  }
  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells));

  entries = new ArrayList<WALEntry>(BATCH_SIZE);
  cells = new ArrayList<Cell>();
  for(int i = 0; i < BATCH_SIZE; i++) {
    entries.add(createEntry(TABLE_NAME1, i,
        i % 2 != 0 ? KeyValue.Type.Put: KeyValue.Type.DeleteColumn, cells));
  }

  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
  Scan scan = new Scan();
  ResultScanner scanRes = table1.getScanner(scan);
  assertEquals(BATCH_SIZE/2, scanRes.next(BATCH_SIZE).length);
}
 
Example 23
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 6 votes vote down vote up
public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception {
  Scan partialScan = new Scan();
  partialScan.setMaxVersions();
  // Max result size of 1 ensures that each RPC request will return a single cell. The scanner
  // will need to reconstruct the results into a complete result before returning to the caller
  partialScan.setMaxResultSize(1);
  partialScan.setReversed(reversed);
  ResultScanner partialScanner = TABLE.getScanner(partialScan);

  final int startRow = reversed ? ROWS.length - 1 : 0;
  final int endRow = reversed ? -1 : ROWS.length;
  final int loopDelta = reversed ? -1 : 1;
  String message;

  for (int row = startRow; row != endRow; row = row + loopDelta) {
    message = "Ensuring the expected keyValues are present for row " + row;
    List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE);
    Result result = partialScanner.next();
    assertFalse(result.isPartial());
    verifyResult(result, expectedKeyValues, message);
  }

  partialScanner.close();
}
 
Example 24
Project: ditb   File: TestReplicationSink.java   Source Code and License 6 votes vote down vote up
/**
 * Insert then do different types of deletes
 * @throws Exception
 */
@Test
public void testMixedDeletes() throws Exception {
  List<WALEntry> entries = new ArrayList<WALEntry>(3);
  List<Cell> cells = new ArrayList<Cell>();
  for(int i = 0; i < 3; i++) {
    entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
  }
  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
  entries = new ArrayList<WALEntry>(3);
  cells = new ArrayList<Cell>();
  entries.add(createEntry(TABLE_NAME1, 0, KeyValue.Type.DeleteColumn, cells));
  entries.add(createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily, cells));
  entries.add(createEntry(TABLE_NAME1, 2, KeyValue.Type.DeleteColumn, cells));

  SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));

  Scan scan = new Scan();
  ResultScanner scanRes = table1.getScanner(scan);
  assertEquals(0, scanRes.next(3).length);
}
 
Example 25
Project: ditb   File: TestVisibilityLabels.java   Source Code and License 6 votes vote down vote up
@Test(timeout = 60 * 1000)
public void testVisibilityLabelsOnRSRestart() throws Exception {
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
      .getRegionServerThreads();
  for (RegionServerThread rsThread : regionServerThreads) {
    rsThread.getRegionServer().abort("Aborting ");
  }
  // Start one new RS
  RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
  waitForLabelsRegionAvailability(rs.getRegionServer());
  try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
      + ")", PRIVATE);) {
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(SECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 1);
  }
}
 
Example 26
Project: ditb   File: HFileTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * This verifies that each cell has a tag that is equal to its rowkey name.  For this to work
 * the hbase instance must have HConstants.RPC_CODEC_CONF_KEY set to
 * KeyValueCodecWithTags.class.getCanonicalName());
 * @param table table containing tagged cells
 * @throws IOException if problems reading table
 */
public static void verifyTags(Table table) throws IOException {
  ResultScanner s = table.getScanner(new Scan());
  for (Result r : s) {
    for (Cell c : r.listCells()) {
      byte[] ta = c.getTagsArray();
      int toff = c.getTagsOffset();
      int tlen = c.getTagsLength();
      Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE);
      if (t == null) {
        fail(c.toString() + " has null tag");
        continue;
      }
      byte[] tval = t.getValue();
      assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval),
          r.getRow(), tval);
    }
  }
}
 
Example 27
Project: ditb   File: TestServerSideScanMetricsFromClientSide.java   Source Code and License 6 votes vote down vote up
/**
 * Run the scan to completetion and check the metric against the specified value
 * @param scan
 * @param metricKey
 * @param expectedValue
 * @throws Exception
 */
public void testMetric(Scan scan, String metricKey, long expectedValue) throws Exception {
  assertTrue("Scan should be configured to record metrics", scan.isScanMetricsEnabled());
  ResultScanner scanner = TABLE.getScanner(scan);

  // Iterate through all the results
  for (Result r : scanner) {
  }
  scanner.close();
  ScanMetrics metrics = scan.getScanMetrics();
  assertTrue("Metrics are null", metrics != null);
  assertTrue("Metric : " + metricKey + " does not exist", metrics.hasCounter(metricKey));
  final long actualMetricValue = metrics.getCounter(metricKey).get();
  assertEquals("Metric: " + metricKey + " Expected: " + expectedValue + " Actual: "
      + actualMetricValue, expectedValue, actualMetricValue);

}
 
Example 28
Project: ditb   File: TestVisibilityLabels.java   Source Code and License 6 votes vote down vote up
@Test
public void testSimpleVisibilityLabels() throws Exception {
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
      PRIVATE + "|" + CONFIDENTIAL)) {
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);

    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));
  }
}
 
Example 29
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 5 votes vote down vote up
public void testPartialResultsAndBatch(final int batch, final int cellsPerPartialResult)
    throws Exception {
  if (LOG.isInfoEnabled()) {
    LOG.info("batch: " + batch + " cellsPerPartialResult: " + cellsPerPartialResult);
  }

  Scan scan = new Scan();
  scan.setMaxResultSize(getResultSizeForNumberOfCells(cellsPerPartialResult));
  scan.setBatch(batch);
  ResultScanner scanner = TABLE.getScanner(scan);
  Result result = scanner.next();
  int repCount = 0;

  while ((result = scanner.next()) != null) {
    assertTrue(result.rawCells() != null);

    if (result.isPartial()) {
      final String error =
          "Cells:" + result.rawCells().length + " Batch size:" + batch
              + " cellsPerPartialResult:" + cellsPerPartialResult + " rep:" + repCount;
      assertTrue(error, result.rawCells().length <= Math.min(batch, cellsPerPartialResult));
    } else {
      assertTrue(result.rawCells().length <= batch);
    }
    repCount++;
  }

  scanner.close();
}
 
Example 30
Project: ditb   File: TestTableInputFormat.java   Source Code and License 5 votes vote down vote up
/**
 * Create a table that IOE's on first scanner next call
 *
 * @throws IOException
 */
static Table createIOEScannerTable(byte[] name, final int failCnt)
    throws IOException {
  // build up a mock scanner stuff to fail the first time
  Answer<ResultScanner> a = new Answer<ResultScanner>() {
    int cnt = 0;

    @Override
    public ResultScanner answer(InvocationOnMock invocation) throws Throwable {
      // first invocation return the busted mock scanner
      if (cnt++ < failCnt) {
        // create mock ResultScanner that always fails.
        Scan scan = mock(Scan.class);
        doReturn("bogus".getBytes()).when(scan).getStartRow(); // avoid npe
        ResultScanner scanner = mock(ResultScanner.class);
        // simulate TimeoutException / IOException
        doThrow(new IOException("Injected exception")).when(scanner).next();
        return scanner;
      }

      // otherwise return the real scanner.
      return (ResultScanner) invocation.callRealMethod();
    }
  };

  Table htable = spy(createTable(name));
  doAnswer(a).when(htable).getScanner((Scan) anyObject());
  return htable;
}
 
Example 31
Project: stroom-stats   File: StatisticsTestService.java   Source Code and License 5 votes vote down vote up
private void deleteAllRows(final Table table) throws IOException {
    final Scan scan = new Scan();
    final List<Delete> deleteList = new ArrayList<>();
    final ResultScanner results = table.getScanner(scan);
    for (final Result result : results) {
        deleteList.add(new Delete(result.getRow()));
    }
    results.close();
    table.delete(deleteList);
}
 
Example 32
Project: stroom-stats   File: StatisticsTestService.java   Source Code and License 5 votes vote down vote up
private int countRows(final Table table) throws IOException {
    int count = 0;
    final Scan scan = new Scan();
    scan.addFamily(EventStoreColumnFamily.COUNTS.asByteArray());
    scan.addFamily(EventStoreColumnFamily.VALUES.asByteArray());
    try (final ResultScanner results = table.getScanner(scan)) {
        count = Iterables.size(results);
    }
    return count;
}
 
Example 33
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 5 votes vote down vote up
/**
 * @param resultSizeRowLimit The row limit that will be enforced through maxResultSize
 * @param cachingRowLimit The row limit that will be enforced through caching
 * @throws Exception
 */
public void testPartialResultsAndCaching(int resultSizeRowLimit, int cachingRowLimit)
    throws Exception {
  Scan scan = new Scan();
  scan.setAllowPartialResults(true);

  // The number of cells specified in the call to getResultSizeForNumberOfCells is offset to
  // ensure that the result size we specify is not an exact multiple of the number of cells
  // in a row. This ensures that partial results will be returned when the result size limit
  // is reached before the caching limit.
  int cellOffset = NUM_COLS / 3;
  long maxResultSize = getResultSizeForNumberOfCells(resultSizeRowLimit * NUM_COLS + cellOffset);
  scan.setMaxResultSize(maxResultSize);
  scan.setCaching(cachingRowLimit);

  ResultScanner scanner = TABLE.getScanner(scan);
  ClientScanner clientScanner = (ClientScanner) scanner;
  Result r = null;

  // Approximate the number of rows we expect will fit into the specified max rsult size. If this
  // approximation is less than caching, then we expect that the max result size limit will be
  // hit before the caching limit and thus partial results may be seen
  boolean expectToSeePartialResults = resultSizeRowLimit < cachingRowLimit;
  while ((r = clientScanner.next()) != null) {
    assertTrue(!r.isPartial() || expectToSeePartialResults);
  }

  scanner.close();
}
 
Example 34
Project: hbase-client   File: BatchIt.java   Source Code and License 5 votes vote down vote up
@Test( description = "Create valid batch instance and call batch method",
        dataProvider = "objCollection" )
public void testCreateValidBatchInstanceAndCallBatch( List<String> strCollection )
        throws Exception {

    final byte[] valQualifier = Bytes.toBytes("val");
    Batch.<String>newBuilder()
            .withBatchSize(3)
            .withObjectCollection(strCollection)
            .withMapper(string -> {
                final byte[] key = Bytes.toBytes(string);
                return new Put(key).addImmutable(TEST_CF_BYTES,
                                                 valQualifier,
                                                 key);
            })
            .withTable(testTable)
            .build()
            .call();

    final ResultScanner scanner = testTable.getScanner(TEST_CF_BYTES, valQualifier);
    final Integer rowCount
            = StreamSupport.stream(scanner.spliterator(), false)
                           .reduce(0,
                                   ( curVal, hresult ) -> curVal + hresult.size(),
                                   ( val1, val2 ) -> val1 + val2);
    assertThat(rowCount).isEqualTo(strCollection.size());
}
 
Example 35
Project: hbase-client   File: BatchIt.java   Source Code and License 5 votes vote down vote up
@Test( description = "Create valid batch instance and call batch method on Iterator",
        dataProvider = "objCollection" )
public void testCreateValidBatchInstanceAndCallBatchOnIterator( List<String> strCollection )
        throws Exception {

    final byte[] valQualifier = Bytes.toBytes("val");
    final Batch<String> batch = Batch.<String>newBuilder()
                                        .withBatchSize(3)
                                        .withObjectCollection(strCollection.iterator())
                                        .withMapper(string -> {
                                            final byte[] key = Bytes.toBytes(string);
                                            return new Put(key).addImmutable(TEST_CF_BYTES,
                                                                             valQualifier,
                                                                             key);
                                        })
                                        .withTable(testTable)
                                        .build();
    // batch on iterator instance perform only one time
    // other calls must have no effect(and must not fail)
    batch.call();
    batch.call();
    batch.call();

    final ResultScanner scanner = testTable.getScanner(TEST_CF_BYTES, valQualifier);
    final Integer rowCount
            = StreamSupport.stream(scanner.spliterator(), false)
                           .reduce(0,
                                   ( curVal, hresult ) -> curVal + hresult.size(),
                                   ( val1, val2 ) -> val1 + val2);
    assertThat(rowCount).isEqualTo(strCollection.size());
}
 
Example 36
Project: easyhbase   File: RowMapperResultsExtractor.java   Source Code and License 5 votes vote down vote up
public List<T> extractData(ResultScanner results) throws Exception {
    List<T> rs = new ArrayList<>();
    int rowNum = 0;
    for (Result result : results) {
        rs.add(this.rowMapper.mapRow(result, rowNum++));
    }
    return rs;
}
 
Example 37
Project: easyhbase   File: HbaseTemplate2.java   Source Code and License 5 votes vote down vote up
protected final <T> T executeDistributedScan(TableName tableName, final Scan scan, final
AbstractRowKeyDistributor rowKeyDistributor, final ResultsExtractor<T> action) {
    assertAccessAvailable();
    return execute(tableName, new TableCallback<T>() {
        @Override
        public T doInTable(Table table) throws Throwable {
            StopWatch watch = null;
            if (debugEnabled) {
                watch = new StopWatch();
                watch.start();
            }
            final ResultScanner[] splitScanners = splitScan(table, scan, rowKeyDistributor);
            final ResultScanner scanner = new DistributedScanner(rowKeyDistributor,
                    splitScanners);
            if (debugEnabled) {
                logger.debug("DistributedScanner createTime: {}ms", watch.stop());
                watch.start();
            }
            try {
                return action.extractData(scanner);
            } finally {
                scanner.close();
                if (debugEnabled) {
                    logger.debug("DistributedScanner scanTime: {}ms", watch.stop());
                }
            }
        }
    });
}
 
Example 38
Project: easyhbase   File: HbaseTemplate2.java   Source Code and License 5 votes vote down vote up
private void closeScanner(ResultScanner[] scannerList) {
    for (ResultScanner scanner : scannerList) {
        if (scanner != null) {
            try {
                scanner.close();
            } catch (Exception e) {
                logger.warn("Scanner.close() error Caused:{}", e.getMessage(), e);
            }
        }
    }
}
 
Example 39
Project: easyhbase   File: LimitRowMapperResultsExtractor.java   Source Code and License 5 votes vote down vote up
public List<T> extractData(ResultScanner results) throws Exception {
    final List<T> rs = new ArrayList<>();
    int rowNum = 0;
    Result lastResult = null;

    for (Result result : results) {
        final T t = this.rowMapper.mapRow(result, rowNum);
        lastResult = result;
        if (t instanceof Collection) {
            rowNum += ((Collection<?>) t).size();
        } else if (t instanceof Map) {
            rowNum += ((Map<?, ?>) t).size();
        } else if (t == null) {
            // empty
        } else if (t.getClass().isArray()) {
            rowNum += Array.getLength(t);
        } else {
            rowNum++;
        }
        rs.add(t);
        if (rowNum >= limit) {
            break;
        }
    }

    eventHandler.handleLastResult(lastResult);
    return rs;
}
 
Example 40
Project: ditb   File: PerformanceEvaluation.java   Source Code and License 5 votes vote down vote up
@Override
void testRow(int i) throws IOException {
  byte[] value = generateValue(this.rand);
  Scan scan = constructScan(value);
  ResultScanner scanner = null;
  try {
    scanner = this.table.getScanner(scan);
    while (scanner.next() != null) {
    }
  } finally {
    if (scanner != null) scanner.close();
  }
}
 
Example 41
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testPartialResultWhenRegionMove() throws IOException {
  Table table=createTestTable(TableName.valueOf("testPartialResultWhenRegionMove"),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
  assertFalse(result1.isPartial());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
  assertTrue(result2.isPartial());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
  assertTrue(result3.isPartial());

}
 
Example 42
Project: ditb   File: TestLoadAndSwitchEncodeOnDisk.java   Source Code and License 5 votes vote down vote up
private void assertAllOnLine(final HTable t) throws IOException {
  NavigableMap<HRegionInfo, ServerName> regions = t.getRegionLocations();
  for (Map.Entry<HRegionInfo, ServerName> e: regions.entrySet()) {
    byte [] startkey = e.getKey().getStartKey();
    Scan s = new Scan(startkey);
    ResultScanner scanner = t.getScanner(s);
    Result r = scanner.next();
    org.junit.Assert.assertTrue(r != null && r.size() > 0);
    scanner.close();
  }
}
 
Example 43
Project: ditb   File: GCScanner.java   Source Code and License 5 votes vote down vote up
public GCScanner(BaseIndexScanner base, ResultScanner scanner, byte[] indexFamily,
    byte[] indexQualifier) throws IOException {
  super(base);
  this.scanner = scanner;
  this.indexFamily = indexFamily;
  this.indexQualifier = indexQualifier;
}
 
Example 44
Project: ditb   File: HybridNormalWorker.java   Source Code and License 5 votes vote down vote up
@Override protected int executeScan(Operation operation) throws IOException {
  Scan scan = operation.getScan();
  scan.setCaching(workload.getScanCacheSize());
  scan.setCacheBlocks(true);
  ResultScanner scanner = BaseIndexScanner.getIndexScanner(conn, relation, scan);
  int resultCount = 0;
  while (true) {
    Result[] results = scanner.next(workload.getScanCacheSize());
    if (results == null || results.length == 0) break;
    resultCount += results.length;
  }
  return resultCount;
}
 
Example 45
Project: ditb   File: HBaseTestingUtility.java   Source Code and License 5 votes vote down vote up
public int countRows(final Table table, final byte[]... families) throws IOException {
  Scan scan = new Scan();
  for (byte[] family: families) {
    scan.addFamily(family);
  }
  ResultScanner results = table.getScanner(scan);
  int count = 0;
  for (@SuppressWarnings("unused") Result res : results) {
    count++;
  }
  results.close();
  return count;
}
 
Example 46
Project: ditb   File: PerfScanBase.java   Source Code and License 5 votes vote down vote up
private OpResult executeScan() throws IOException, ParseException {
  if (!hasScan()) {
    return new OpResult("scan not supported", 1, 1);
  }
  Table table = conn.getTable(opTblName);
  BufferedReader br = new BufferedReader(new FileReader(scanFilePath));
  String line;
  long totalTime = 0;
  int counter = 0;
  Result[] results;
  while ((line = br.readLine()) != null) {
    Scan scan = new Scan(getIndexTableScanStartKey(line));
    scan.setCaching(workload.getScanCacheSize());
    scan.setCacheBlocks(false);
    long startTime = System.currentTimeMillis();
    ResultScanner scanner = table.getScanner(scan);
    int wantedRecords = sizeScanCovering;
    while (true) {
      results = scanner.next(Math.min(wantedRecords, workload.getScanCacheSize()));
      if (results == null || results.length == 0) break;
      for (Result result : results) {
        int k = recordsInOneResult(result);
        wantedRecords -= k;
        counter += k;
      }
      if (wantedRecords <= 0) break;
    }
    scanner.close();
    totalTime += System.currentTimeMillis() - startTime;
  }
  OpResult ret = new OpResult("scan", counter, totalTime);
  br.close();
  table.close();
  return ret;
}
 
Example 47
Project: ditb   File: PerfMD.java   Source Code and License 5 votes vote down vote up
@Override protected Result processGet(Table table, Get get) throws IOException {
  Scan scan = new Scan();
  scan.addFamily(MDHBaseAdmin.BUCKET_FAMILY);
  scan.setReversed(true);
  scan.setStartRow(get.getRow());
  scan.setCacheBlocks(false);
  scan.setCaching(1);
  scan.setSmall(true);
  ResultScanner scanner = table.getScanner(scan);
  Result ret = scanner.next();
  scanner.close();
  return ret;
}
 
Example 48
Project: ditb   File: TestMasterTransitions.java   Source Code and License 5 votes vote down vote up
private static int addToEachStartKey(final int expected) throws IOException {
  Table t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
  Table meta = new HTable(TEST_UTIL.getConfiguration(),
      TableName.META_TABLE_NAME);
  int rows = 0;
  Scan scan = new Scan();
  scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  ResultScanner s = meta.getScanner(scan);
  for (Result r = null; (r = s.next()) != null;) {
    HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
    if (hri == null) break;
    if (!hri.getTable().equals(TABLENAME)) {
      continue;
    }

    // If start key, add 'aaa'.
    if(!hri.getTable().equals(TABLENAME)) {
      continue;
    }
    byte [] row = getStartKey(hri);
    Put p = new Put(row);
    p.setDurability(Durability.SKIP_WAL);
    p.add(getTestFamily(), getTestQualifier(), row);
    t.put(p);
    rows++;
  }
  s.close();
  Assert.assertEquals(expected, rows);
  t.close();
  meta.close();
  return rows;
}
 
Example 49
Project: ditb   File: TestZKBasedOpenCloseRegion.java   Source Code and License 5 votes vote down vote up
private static int addToEachStartKey(final int expected) throws IOException {
  HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
  HTable meta = new HTable(TEST_UTIL.getConfiguration(),
      TableName.META_TABLE_NAME);
  int rows = 0;
  Scan scan = new Scan();
  scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  ResultScanner s = meta.getScanner(scan);
  for (Result r = null; (r = s.next()) != null;) {
    HRegionInfo hri = HRegionInfo.getHRegionInfo(r);
    if (hri == null) break;
    if(!hri.getTable().equals(TABLENAME)) {
      continue;
    }
    // If start key, add 'aaa'.
    byte [] row = getStartKey(hri);
    Put p = new Put(row);
    p.setDurability(Durability.SKIP_WAL);
    p.add(getTestFamily(), getTestQualifier(), row);
    t.put(p);
    rows++;
  }
  s.close();
  Assert.assertEquals(expected, rows);
  t.close();
  meta.close();
  return rows;
}
 
Example 50
Project: ditb   File: TestImportExport.java   Source Code and License 5 votes vote down vote up
/**
 * Count the number of keyvalues in the specified table for the given timerange
 * @param start
 * @param end
 * @param table
 * @return
 * @throws IOException
 */
private int getCount(Table table, Filter filter) throws IOException {
  Scan scan = new Scan();
  scan.setFilter(filter);
  ResultScanner results = table.getScanner(scan);
  int count = 0;
  for (Result res : results) {
    count += res.size();
  }
  results.close();
  return count;
}
 
Example 51
Project: ditb   File: ThriftHBaseServiceHandler.java   Source Code and License 5 votes vote down vote up
@Override
public void closeScanner(int scannerId) throws TIOError, TIllegalArgument, TException {
  LOG.debug("scannerClose: id=" + scannerId);
  ResultScanner scanner = getScanner(scannerId);
  if (scanner == null) {
    String message = "scanner ID is invalid";
    LOG.warn(message);
    TIllegalArgument ex = new TIllegalArgument();
    ex.setMessage("Invalid scanner Id");
    throw ex;
  }
  scanner.close();
  removeScanner(scannerId);
}
 
Example 52
Project: ditb   File: TestVisibilityLabels.java   Source Code and License 5 votes vote down vote up
@Test
public void testVisibilityLabelsWithComplexLabels() throws Exception {
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
      + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET, "(" + PRIVATE + "&" + CONFIDENTIAL + "&"
      + SECRET + ")", "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "(" + PRIVATE
      + "&" + CONFIDENTIAL + "&" + SECRET + ")")) {
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(4);
    assertEquals(3, next.length);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row3, 0, row3.length));
    cellScanner = next[2].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row4, 0, row4.length));
  }
}
 
Example 53
Project: ditb   File: ThriftServerRunner.java   Source Code and License 5 votes vote down vote up
private Result getRowOrBefore(byte[] tableName, byte[] row, byte[] family) throws IOException {
  Scan scan = new Scan(row);
  scan.setReversed(true);
  scan.addFamily(family);
  scan.setStartRow(row);
  Table table = getTable(tableName);
  try (ResultScanner scanner = table.getScanner(scan)) {
    return scanner.next();
  } finally{
    if(table != null){
      table.close();
    }
  }
}
 
Example 54
Project: ditb   File: TestVisibilityLabels.java   Source Code and License 5 votes vote down vote up
@Test
public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
  TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try ( Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|"
      + CONFIDENTIAL + ")", PRIVATE)){
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations("SAMPLE"));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 0);
  }
}
 
Example 55
Project: ditb   File: ModifyTableProcedure.java   Source Code and License 5 votes vote down vote up
/**
 * update replica column families if necessary.
 * @param env MasterProcedureEnv
 * @throws IOException
 */
private void updateReplicaColumnsIfNeeded(
  final MasterProcedureEnv env,
  final HTableDescriptor oldHTableDescriptor,
  final HTableDescriptor newHTableDescriptor) throws IOException {
  final int oldReplicaCount = oldHTableDescriptor.getRegionReplication();
  final int newReplicaCount = newHTableDescriptor.getRegionReplication();

  if (newReplicaCount < oldReplicaCount) {
    Set<byte[]> tableRows = new HashSet<byte[]>();
    Connection connection = env.getMasterServices().getConnection();
    Scan scan = MetaTableAccessor.getScanForTableName(getTableName());
    scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);

    try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME)) {
      ResultScanner resScanner = metaTable.getScanner(scan);
      for (Result result : resScanner) {
        tableRows.add(result.getRow());
      }
      MetaTableAccessor.removeRegionReplicasFromMeta(
        tableRows,
        newReplicaCount,
        oldReplicaCount - newReplicaCount,
        connection);
    }
  }

  // Setup replication for region replicas if needed
  if (newReplicaCount > 1 && oldReplicaCount <= 1) {
    ServerRegionReplicaUtil.setupRegionReplicaReplication(env.getMasterConfiguration());
  }
}
 
Example 56
Project: ditb   File: SyncTable.java   Source Code and License 5 votes vote down vote up
private void finishRemainingHashRanges(Context context) throws IOException,
    InterruptedException {
  TableSplit split = (TableSplit) context.getInputSplit();
  byte[] splitEndRow = split.getEndRow();
  boolean reachedEndOfTable = HashTable.isTableEndRow(splitEndRow);

  // if there are more hash batches that begin before the end of this split move to them
  while (nextSourceKey != null
      && (nextSourceKey.compareTo(splitEndRow) < 0 || reachedEndOfTable)) {
    moveToNextBatch(context);
  }
  
  if (targetHasher.isBatchStarted()) {
    // need to complete the final open hash batch

    if ((nextSourceKey != null && nextSourceKey.compareTo(splitEndRow) > 0)
          || (nextSourceKey == null && !Bytes.equals(splitEndRow, sourceTableHash.stopRow))) {
      // the open hash range continues past the end of this region
      // add a scan to complete the current hash range
      Scan scan = sourceTableHash.initScan();
      scan.setStartRow(splitEndRow);
      if (nextSourceKey == null) {
        scan.setStopRow(sourceTableHash.stopRow);
      } else {
        scan.setStopRow(nextSourceKey.copyBytes());
      }
      
      ResultScanner targetScanner = targetTable.getScanner(scan);
      for (Result row : targetScanner) {
        targetHasher.hashResult(row);          
      }
    } // else current batch ends exactly at split end row

    finishBatchAndCompareHashes(context);
  }
}
 
Example 57
Project: ditb   File: PerformanceEvaluation.java   Source Code and License 5 votes vote down vote up
@Override
void testRow(final int i) throws IOException {
  Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
  Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
  scan.setCaching(opts.caching);
  if (opts.filterAll) {
    scan.setFilter(new FilterAllFilter());
  }
  if (opts.addColumns) {
    scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
  } else {
    scan.addFamily(FAMILY_NAME);
  }
  Result r = null;
  int count = 0;
  ResultScanner s = this.table.getScanner(scan);
  for (; (r = s.next()) != null;) {
    updateValueSize(r);
    count++;
  }
  if (i % 100 == 0) {
    LOG.info(String.format("Scan for key range %s - %s returned %s rows",
        Bytes.toString(startAndStopRow.getFirst()),
        Bytes.toString(startAndStopRow.getSecond()), count));
  }

  s.close();
}
 
Example 58
Project: ditb   File: TestFuzzyRowFilterEndToEnd.java   Source Code and License 5 votes vote down vote up
private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
    throws IOException {
  String cf = "f";
  Scan scan = new Scan();
  scan.addFamily(cf.getBytes());
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
  scan.setFilter(filterList);

  ResultScanner scanner = hTable.getScanner(scan);
  List<Cell> results = new ArrayList<Cell>();
  Result result;
  long timeBeforeScan = System.currentTimeMillis();
  while ((result = scanner.next()) != null) {
    for (Cell kv : result.listCells()) {
      LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
          + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
      results.add(kv);
    }
  }
  long scanTime = System.currentTimeMillis() - timeBeforeScan;
  scanner.close();

  LOG.info("scan time = " + scanTime + "ms");
  LOG.info("found " + results.size() + " results");

  assertEquals(expectedSize, results.size());
}
 
Example 59
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testCompleteResultWhenRegionMove() throws IOException {
  Table table=createTestTable(TableName.valueOf("testCompleteResultWhenRegionMove"),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setCaching(1);
  ResultScanner scanner = table.getScanner(scan);

  Result result1 = scanner.next();
  assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[0], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result1.isPartial());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result2.isPartial());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(NUM_FAMILIES * NUM_QUALIFIERS, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[2], FAMILIES[0], QUALIFIERS[0]);
  assertFalse(result3.isPartial());

}
 
Example 60
Project: ditb   File: HBaseTestingUtility.java   Source Code and License 5 votes vote down vote up
/**
 * Starts the hbase cluster up again after shutting it down previously in a
 * test.  Use this if you want to keep dfs/zk up and just stop/start hbase.
 * @param servers number of region servers
 * @throws IOException
 */
public void restartHBaseCluster(int servers) throws IOException, InterruptedException {
  this.hbaseCluster = new MiniHBaseCluster(this.conf, servers);
  // Don't leave here till we've done a successful scan of the hbase:meta
  Table t = new HTable(new Configuration(this.conf), TableName.META_TABLE_NAME);
  ResultScanner s = t.getScanner(new Scan());
  while (s.next() != null) {
    // do nothing
  }
  LOG.info("HBase has been restarted");
  s.close();
  t.close();
}