Java Code Examples for org.apache.hadoop.hbase.client.Scan.setMaxVersions()

The following are Jave code examples for showing how to use setMaxVersions() of the org.apache.hadoop.hbase.client.Scan class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: SparkIsax   File: HBaseUtils.java   Source Code and License Vote up 10 votes
/**
 * 获取单张表的所有记录
 * 
 * @throws IOException
 */
public static Map<byte[], byte[]> getAll(String TableName, String ColumnFamily, String ColumnName)
		throws IOException {
	Map<byte[], byte[]> tableContent = new HashMap<byte[], byte[]>();
	Scan s = new Scan();
	s.addColumn(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName));
	s.setMaxVersions(1);
	s.setCacheBlocks(false);
	ResultScanner rs = hbase_table.getScanner(s);
	for (Result r : rs) {
		byte[] key = r.getRow();
		byte[] value = r.getValue(Bytes.toBytes(ColumnFamily), Bytes.toBytes(ColumnName));
		tableContent.put(key, value);
	}
	rs.close();
	return tableContent;
}
 
Example 2
Project: ditb   File: HashTable.java   Source Code and License Vote up 6 votes
Scan initScan() throws IOException {
  Scan scan = new Scan();
  scan.setCacheBlocks(false);
  if (startTime != 0 || endTime != 0) {
    scan.setTimeRange(startTime, endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
  }
  if (scanBatch > 0) {
    scan.setBatch(scanBatch);
  }
  if (versions >= 0) {
    scan.setMaxVersions(versions);
  }
  if (!isTableStartRow(startRow)) {
    scan.setStartRow(startRow);
  }
  if (!isTableEndRow(stopRow)) {
    scan.setStopRow(stopRow);
  }
  if(families != null) {
    for(String fam : families.split(",")) {
      scan.addFamily(Bytes.toBytes(fam));
    }
  }
  return scan;
}
 
Example 3
Project: ditb   File: TestKeepDeletes.java   Source Code and License Vote up 6 votes
private int countDeleteMarkers(Region region) throws IOException {
  Scan s = new Scan();
  s.setRaw(true);
  // use max versions from the store(s)
  s.setMaxVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions());
  InternalScanner scan = region.getScanner(s);
  List<Cell> kvs = new ArrayList<Cell>();
  int res = 0;
  boolean hasMore;
  do {
    hasMore = scan.next(kvs);
    for (Cell kv : kvs) {
      if(CellUtil.isDelete(kv)) res++;
    }
    kvs.clear();
  } while (hasMore);
  scan.close();
  return res;
}
 
Example 4
Project: ditb   File: TestMultiRowRangeFilter.java   Source Code and License Vote up 6 votes
@Test
public void testMultiRowRangeFilterWithEmptyStartRow() throws IOException {
  tableName = Bytes.toBytes("testMultiRowRangeFilterWithEmptyStartRow");
  HTable ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(numRows, ht, family, qf, value);
  Scan scan = new Scan();
  scan.setMaxVersions();

  List<RowRange> ranges = new ArrayList<RowRange>();
  ranges.add(new RowRange(Bytes.toBytes(""), true, Bytes.toBytes(10), false));
  ranges.add(new RowRange(Bytes.toBytes(30), true, Bytes.toBytes(40), false));

  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);
  int resultsSize = getResultsSize(ht, scan);
  List<Cell> results1 = getScanResult(Bytes.toBytes(""), Bytes.toBytes(10), ht);
  List<Cell> results2 = getScanResult(Bytes.toBytes(30), Bytes.toBytes(40), ht);
  assertEquals(results1.size() + results2.size(), resultsSize);

  ht.close();
}
 
Example 5
Project: ditb   File: TestMultiRowRangeFilter.java   Source Code and License Vote up 6 votes
@Test
public void testMultiRowRangeFilterWithExclusive() throws IOException {
  tableName = Bytes.toBytes("testMultiRowRangeFilterWithExclusive");
  HTable ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(numRows, ht, family, qf, value);

  Scan scan = new Scan();
  scan.setMaxVersions();

  List<RowRange> ranges = new ArrayList<RowRange>();
  ranges.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false));
  ranges.add(new RowRange(Bytes.toBytes(20), false, Bytes.toBytes(40), false));
  ranges.add(new RowRange(Bytes.toBytes(65), true, Bytes.toBytes(75), false));

  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);
  int resultsSize = getResultsSize(ht, scan);
  LOG.info("found " + resultsSize + " results");
  List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(40), ht);
  List<Cell> results2 = getScanResult(Bytes.toBytes(65), Bytes.toBytes(75), ht);

  assertEquals((results1.size() - 1) + results2.size(), resultsSize);

  ht.close();
}
 
Example 6
Project: ditb   File: TestPartialResultsFromClientSide.java   Source Code and License Vote up 6 votes
public void testExpectedValuesOfPartialResults(boolean reversed) throws Exception {
  Scan partialScan = new Scan();
  partialScan.setMaxVersions();
  // Max result size of 1 ensures that each RPC request will return a single cell. The scanner
  // will need to reconstruct the results into a complete result before returning to the caller
  partialScan.setMaxResultSize(1);
  partialScan.setReversed(reversed);
  ResultScanner partialScanner = TABLE.getScanner(partialScan);

  final int startRow = reversed ? ROWS.length - 1 : 0;
  final int endRow = reversed ? -1 : ROWS.length;
  final int loopDelta = reversed ? -1 : 1;
  String message;

  for (int row = startRow; row != endRow; row = row + loopDelta) {
    message = "Ensuring the expected keyValues are present for row " + row;
    List<Cell> expectedKeyValues = createKeyValuesForRow(ROWS[row], FAMILIES, QUALIFIERS, VALUE);
    Result result = partialScanner.next();
    assertFalse(result.isPartial());
    verifyResult(result, expectedKeyValues, message);
  }

  partialScanner.close();
}
 
Example 7
Project: ditb   File: IntegrationTestMTTR.java   Source Code and License Vote up 6 votes
@Override
protected boolean doAction() throws Exception {
  ResultScanner rs = null;
  try {
    Scan s = new Scan();
    s.setBatch(2);
    s.addFamily(FAMILY);
    s.setFilter(new KeyOnlyFilter());
    s.setMaxVersions(1);

    rs = table.getScanner(s);
    Result result = rs.next();
    return result != null && result.size() > 0;
  } finally {
    if (rs != null) {
      rs.close();
    }
  }
}
 
Example 8
Project: ditb   File: TestMultiRowRangeFilter.java   Source Code and License Vote up 6 votes
@Test
public void testMultiRowRangeFilterWithEmptyStopRow() throws IOException {
  tableName = Bytes.toBytes("testMultiRowRangeFilterWithEmptyStopRow");
  HTable ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(numRows, ht, family, qf, value);
  Scan scan = new Scan();
  scan.setMaxVersions();

  List<RowRange> ranges = new ArrayList<RowRange>();
  ranges.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(""), false));
  ranges.add(new RowRange(Bytes.toBytes(30), true, Bytes.toBytes(40), false));

  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);
  int resultsSize = getResultsSize(ht, scan);
  List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(""), ht);
  assertEquals(results1.size(), resultsSize);

  ht.close();
}
 
Example 9
Project: ditb   File: TestBlocksScanned.java   Source Code and License Vote up 5 votes
private void _testBlocksScanned(HTableDescriptor table) throws Exception {
  Region r = createNewHRegion(table, START_KEY, END_KEY, TEST_UTIL.getConfiguration());
  addContent(r, FAMILY, COL);
  r.flush(true);

  CacheStats stats = new CacheConfig(TEST_UTIL.getConfiguration()).getBlockCache().getStats();
  long before = stats.getHitCount() + stats.getMissCount();
  // Do simple test of getting one row only first.
  Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz"));
  scan.addColumn(FAMILY, COL);
  scan.setMaxVersions(1);

  InternalScanner s = r.getScanner(scan);
  List<Cell> results = new ArrayList<Cell>();
  while (s.next(results))
    ;
  s.close();

  int expectResultSize = 'z' - 'a';
  assertEquals(expectResultSize, results.size());

  int kvPerBlock = (int) Math.ceil(BLOCK_SIZE / 
      (double) KeyValueUtil.ensureKeyValue(results.get(0)).getLength());
  Assert.assertEquals(2, kvPerBlock);

  long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock);
  long expectIndexBlockRead = expectDataBlockRead;

  assertEquals(expectIndexBlockRead+expectDataBlockRead, stats.getHitCount() + stats.getMissCount() - before);
}
 
Example 10
Project: ditb   File: VerifyReplication.java   Source Code and License Vote up 5 votes
/**
 * Sets up the actual job.
 *
 * @param conf  The current configuration.
 * @param args  The command line parameters.
 * @return The newly created job.
 * @throws java.io.IOException When setting up the job fails.
 */
public static Job createSubmittableJob(Configuration conf, String[] args)
throws IOException {
  if (!doCommandLine(args)) {
    return null;
  }
  if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
      HConstants.REPLICATION_ENABLE_DEFAULT)) {
    throw new IOException("Replication needs to be enabled to verify it.");
  }
  conf.set(NAME+".peerId", peerId);
  conf.set(NAME+".tableName", tableName);
  conf.setLong(NAME+".startTime", startTime);
  conf.setLong(NAME+".endTime", endTime);
  if (families != null) {
    conf.set(NAME+".families", families);
  }

  Pair<ReplicationPeerConfig, Configuration> peerConfigPair = getPeerQuorumConfig(conf);
  ReplicationPeerConfig peerConfig = peerConfigPair.getFirst();
  String peerQuorumAddress = peerConfig.getClusterKey();
  LOG.info("Peer Quorum Address: " + peerQuorumAddress + ", Peer Configuration: " +
      peerConfig.getConfiguration());
  conf.set(NAME + ".peerQuorumAddress", peerQuorumAddress);
  HBaseConfiguration.setWithPrefix(conf, PEER_CONFIG_PREFIX,
      peerConfig.getConfiguration().entrySet());

  conf.setInt(NAME + ".versions", versions);
  LOG.info("Number of version: " + versions);

  Job job = new Job(conf, NAME + "_" + tableName);
  job.setJarByClass(VerifyReplication.class);

  Scan scan = new Scan();
  scan.setTimeRange(startTime, endTime);
  if (versions >= 0) {
    scan.setMaxVersions(versions);
    LOG.info("Number of versions set to " + versions);
  }
  if(families != null) {
    String[] fams = families.split(",");
    for(String fam : fams) {
      scan.addFamily(Bytes.toBytes(fam));
    }
  }
  TableMapReduceUtil.initTableMapperJob(tableName, scan,
      Verifier.class, null, null, job);

  Configuration peerClusterConf = peerConfigPair.getSecond();
  // Obtain the auth token from peer cluster
  TableMapReduceUtil.initCredentialsForCluster(job, peerClusterConf);

  job.setOutputFormatClass(NullOutputFormat.class);
  job.setNumReduceTasks(0);
  return job;
}
 
Example 11
Project: ditb   File: TestMultiRowRangeFilter.java   Source Code and License Vote up 5 votes
@Test
public void testMultiRowRangeFilterWithRangeOverlap() throws IOException {
  tableName = Bytes.toBytes("testMultiRowRangeFilterWithRangeOverlap");
  HTable ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(numRows, ht, family, qf, value);

  Scan scan = new Scan();
  scan.setMaxVersions();

  List<RowRange> ranges = new ArrayList<RowRange>();
  ranges.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false));
  ranges.add(new RowRange(Bytes.toBytes(15), true, Bytes.toBytes(40), false));
  ranges.add(new RowRange(Bytes.toBytes(65), true, Bytes.toBytes(75), false));
  ranges.add(new RowRange(Bytes.toBytes(60), true, null, false));
  ranges.add(new RowRange(Bytes.toBytes(60), true, Bytes.toBytes(80), false));

  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);
  int resultsSize = getResultsSize(ht, scan);
  LOG.info("found " + resultsSize + " results");
  List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(40), ht);
  List<Cell> results2 = getScanResult(Bytes.toBytes(60), Bytes.toBytes(""), ht);

  assertEquals(results1.size() + results2.size(), resultsSize);

  ht.close();
}
 
Example 12
Project: ditb   File: TestMultiRowRangeFilter.java   Source Code and License Vote up 5 votes
@Test
public void testMultiRowRangeFilterWithInclusive() throws IOException {
  tableName = Bytes.toBytes("testMultiRowRangeFilterWithInclusive");
  HTable ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE);
  generateRows(numRows, ht, family, qf, value);

  Scan scan = new Scan();
  scan.setMaxVersions();

  List<RowRange> ranges = new ArrayList<RowRange>();
  ranges.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false));
  ranges.add(new RowRange(Bytes.toBytes(20), true, Bytes.toBytes(40), false));
  ranges.add(new RowRange(Bytes.toBytes(65), true, Bytes.toBytes(75), false));
  ranges.add(new RowRange(Bytes.toBytes(60), true, null, false));
  ranges.add(new RowRange(Bytes.toBytes(60), true, Bytes.toBytes(80), false));

  MultiRowRangeFilter filter = new MultiRowRangeFilter(ranges);
  scan.setFilter(filter);
  int resultsSize = getResultsSize(ht, scan);
  LOG.info("found " + resultsSize + " results");
  List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(40), ht);
  List<Cell> results2 = getScanResult(Bytes.toBytes(60), Bytes.toBytes(""), ht);

  assertEquals(results1.size() + results2.size(), resultsSize);

  ht.close();
}
 
Example 13
Project: ditb   File: TestWideScanner.java   Source Code and License Vote up 4 votes
public void testWideScanBatching() throws IOException {
  final int batch = 256;
  try {
    this.r = createNewHRegion(TESTTABLEDESC, null, null);
    int inserted = addWideContent(this.r);
    List<Cell> results = new ArrayList<Cell>();
    Scan scan = new Scan();
    scan.addFamily(A);
    scan.addFamily(B);
    scan.addFamily(C);
    scan.setMaxVersions(100);
    scan.setBatch(batch);
    InternalScanner s = r.getScanner(scan);
    int total = 0;
    int i = 0;
    boolean more;
    do {
      more = s.next(results);
      i++;
      LOG.info("iteration #" + i + ", results.size=" + results.size());

      // assert that the result set is no larger
      assertTrue(results.size() <= batch);

      total += results.size();

      if (results.size() > 0) {
        // assert that all results are from the same row
        byte[] row = CellUtil.cloneRow(results.get(0));
        for (Cell kv: results) {
          assertTrue(Bytes.equals(row, CellUtil.cloneRow(kv)));
        }
      }

      results.clear();

      // trigger ChangedReadersObservers
      Iterator<KeyValueScanner> scanners =
        ((HRegion.RegionScannerImpl)s).storeHeap.getHeap().iterator();
      while (scanners.hasNext()) {
        StoreScanner ss = (StoreScanner)scanners.next();
        ss.updateReaders();
      }
    } while (more);

    // assert that the scanner returned all values
    LOG.info("inserted " + inserted + ", scanned " + total);
    assertEquals(total, inserted);

    s.close();
  } finally {
    HRegion.closeHRegion(this.r);
  }
}
 
Example 14
Project: ditb   File: TestVisibilityLabelsWithDeletes.java   Source Code and License Vote up 4 votes
@Test
public void testDeleteFamilySpecificTimeStampWithMulipleVersions() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getHBaseAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
              + SECRET + "&" + TOPSECRET + ")"));
          d.addFamily(fam, 126l);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getHBaseAdmin().flush(tableName);
    Scan s = new Scan();
    s.setMaxVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(6);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 127l);
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 125l);
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 123l);
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));
  }
}
 
Example 15
Project: ditb   File: TestVisibilityLabelsWithDeletes.java   Source Code and License Vote up 4 votes
@Test
public void testScanAfterCompaction() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getHBaseAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" +
              SECRET + "&" + TOPSECRET+")"));
          d.addFamily(fam, 126l);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getHBaseAdmin().flush(tableName);
    Put put = new Put(Bytes.toBytes("row3"));
    put.add(fam, qual, 127l, value);
    put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE));
    table.put(put);
    TEST_UTIL.getHBaseAdmin().flush(tableName);
    TEST_UTIL.getHBaseAdmin().compact(tableName);
    Thread.sleep(5000);
    // Sleep to ensure compaction happens. Need to do it in a better way
    Scan s = new Scan();
    s.setMaxVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 3);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 127l);
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));
  }
}
 
Example 16
Project: ditb   File: TestScannerSelectionUsingTTL.java   Source Code and License Vote up 4 votes
@Test
public void testScannerSelection() throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setBoolean("hbase.store.delete.expired.storefile", false);
  HColumnDescriptor hcd =
    new HColumnDescriptor(FAMILY_BYTES)
        .setMaxVersions(Integer.MAX_VALUE)
        .setTimeToLive(TTL_SECONDS);
  HTableDescriptor htd = new HTableDescriptor(TABLE);
  htd.addFamily(hcd);
  HRegionInfo info = new HRegionInfo(TABLE);
  HRegion region =
      HRegion.createHRegion(info, TEST_UTIL.getDataTestDir(info.getEncodedName()),
          conf, htd);

  long ts = EnvironmentEdgeManager.currentTime();
  long version = 0; //make sure each new set of Put's have a new ts
  for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
    if (iFile == NUM_EXPIRED_FILES) {
      Threads.sleepWithoutInterrupt(TTL_MS);
      version += TTL_MS;
    }

    for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
      Put put = new Put(Bytes.toBytes("row" + iRow));
      for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
        put.add(FAMILY_BYTES, Bytes.toBytes("col" + iCol),
            ts + version, Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
      }
      region.put(put);
    }
    region.flush(true);
    version++;
  }

  Scan scan = new Scan();
  scan.setMaxVersions(Integer.MAX_VALUE);
  CacheConfig cacheConf = new CacheConfig(conf);
  LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
  cache.clearCache();
  InternalScanner scanner = region.getScanner(scan);
  List<Cell> results = new ArrayList<Cell>();
  final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
  int numReturnedRows = 0;
  LOG.info("Scanning the entire table");
  while (scanner.next(results) || results.size() > 0) {
    assertEquals(expectedKVsPerRow, results.size());
    ++numReturnedRows;
    results.clear();
  }
  assertEquals(NUM_ROWS, numReturnedRows);
  Set<String> accessedFiles = cache.getCachedFileNamesForTest();
  LOG.debug("Files accessed during scan: " + accessedFiles);

  // Exercise both compaction codepaths.
  if (explicitCompaction) {
    HStore store = (HStore)region.getStore(FAMILY_BYTES);
    store.compactRecentForTestingAssumingDefaultPolicy(totalNumFiles);
  } else {
    region.compact(false);
  }

  region.close();
}
 
Example 17
Project: ditb   File: TestSeekOptimizations.java   Source Code and License Vote up 4 votes
private void testScan(final int[] columnArr, final boolean lazySeekEnabled,
    final int startRow, final int endRow, int maxVersions)
    throws IOException {
  StoreScanner.enableLazySeekGlobally(lazySeekEnabled);
  final Scan scan = new Scan();
  final Set<String> qualSet = new HashSet<String>();
  for (int iColumn : columnArr) {
    String qualStr = getQualStr(iColumn);
    scan.addColumn(FAMILY_BYTES, Bytes.toBytes(qualStr));
    qualSet.add(qualStr);
  }
  scan.setMaxVersions(maxVersions);
  scan.setStartRow(rowBytes(startRow));

  // Adjust for the fact that for multi-row queries the end row is exclusive.
  {
    final byte[] scannerStopRow =
        rowBytes(endRow + (startRow != endRow ? 1 : 0));
    scan.setStopRow(scannerStopRow);
  }

  final long initialSeekCount = StoreFileScanner.getSeekCount();
  final InternalScanner scanner = region.getScanner(scan);
  final List<Cell> results = new ArrayList<Cell>();
  final List<Cell> actualKVs = new ArrayList<Cell>();

  // Such a clumsy do-while loop appears to be the official way to use an
  // internalScanner. scanner.next() return value refers to the _next_
  // result, not to the one already returned in results.
  boolean hasNext;
  do {
    hasNext = scanner.next(results);
    actualKVs.addAll(results);
    results.clear();
  } while (hasNext);

  List<Cell> filteredKVs = filterExpectedResults(qualSet,
      rowBytes(startRow), rowBytes(endRow), maxVersions);
  final String rowRestrictionStr =
      (startRow == -1 && endRow == -1) ? "all rows" : (
          startRow == endRow ? ("row=" + startRow) : ("startRow="
          + startRow + ", " + "endRow=" + endRow));
  final String columnRestrictionStr =
      columnArr.length == 0 ? "all columns"
          : ("columns=" + Arrays.toString(columnArr));
  final String testDesc =
      "Bloom=" + bloomType + ", compr=" + comprAlgo + ", "
          + (scan.isGetScan() ? "Get" : "Scan") + ": "
          + columnRestrictionStr + ", " + rowRestrictionStr
          + ", maxVersions=" + maxVersions + ", lazySeek=" + lazySeekEnabled;
  long seekCount = StoreFileScanner.getSeekCount() - initialSeekCount;
  if (VERBOSE) {
    System.err.println("Seek count: " + seekCount + ", KVs returned: "
      + actualKVs.size() + ". " + testDesc +
      (lazySeekEnabled ? "\n" : ""));
  }
  if (lazySeekEnabled) {
    totalSeekLazy += seekCount;
  } else {
    totalSeekDiligent += seekCount;
  }
  assertKVListsEqual(testDesc, filteredKVs, actualKVs);
}
 
Example 18
Project: ditb   File: TestMultipleColumnPrefixFilter.java   Source Code and License Vote up 4 votes
@Test
public void testMultipleColumnPrefixFilterWithColumnPrefixFilter() throws IOException {
  String family = "Family";
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("TestMultipleColumnPrefixFilter"));
  htd.addFamily(new HColumnDescriptor(family));
  HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
  HRegion region = HRegion.createHRegion(info, TEST_UTIL.
    getDataTestDir(), TEST_UTIL.getConfiguration(),htd);

  List<String> rows = generateRandomWords(100, "row");
  List<String> columns = generateRandomWords(10000, "column");
  long maxTimestamp = 2;

  String valueString = "ValueString";

  for (String row: rows) {
    Put p = new Put(Bytes.toBytes(row));
    p.setDurability(Durability.SKIP_WAL);
    for (String column: columns) {
      for (long timestamp = 1; timestamp <= maxTimestamp; timestamp++) {
        KeyValue kv = KeyValueTestUtil.create(row, family, column, timestamp,
            valueString);
        p.add(kv);
      }
    }
    region.put(p);
  }

  MultipleColumnPrefixFilter multiplePrefixFilter;
  Scan scan1 = new Scan();
  scan1.setMaxVersions();
  byte [][] filter_prefix = new byte [1][];
  filter_prefix[0] = new byte [] {'p'};
 
  multiplePrefixFilter = new MultipleColumnPrefixFilter(filter_prefix);
  scan1.setFilter(multiplePrefixFilter);
  List<Cell> results1 = new ArrayList<Cell>();  
  InternalScanner scanner1 = region.getScanner(scan1);
  while (scanner1.next(results1))
    ;
  
  ColumnPrefixFilter singlePrefixFilter;
  Scan scan2 = new Scan();
  scan2.setMaxVersions();
  singlePrefixFilter = new ColumnPrefixFilter(Bytes.toBytes("p"));
 
  scan2.setFilter(singlePrefixFilter);
  List<Cell> results2 = new ArrayList<Cell>();  
  InternalScanner scanner2 = region.getScanner(scan1);
  while (scanner2.next(results2))
    ;
  
  assertEquals(results1.size(), results2.size());

  HRegion.closeHRegion(region);
}
 
Example 19
Project: ditb   File: TestVisibilityLabelsWithDeletes.java   Source Code and License Vote up 4 votes
@Test
public void testVisibilityLabelsWithDeleteFamilyWithMultipleVersionsNoTimestamp()
    throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getHBaseAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          Delete d1 = new Delete(row1);
          d1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
          d1.addFamily(fam);
          table.delete(d1);

          Delete d2 = new Delete(row1);
          d2.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
          d2.addFamily(fam);
          table.delete(d2);

          Delete d3 = new Delete(row1);
          d3.setCellVisibility(new CellVisibility("(" + PRIVATE + "&" + CONFIDENTIAL + ")|("
              + SECRET + "&" + TOPSECRET + ")"));
          d3.addFamily(fam);
          table.delete(d3);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    Scan s = new Scan();
    s.setMaxVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertEquals(1, next.length);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));
  }
}
 
Example 20
Project: ditb   File: TestVisibilityLabelsWithDeletes.java   Source Code and License Vote up 4 votes
@Test
public void testDeleteFamilyAndDeleteColumnsWithAndWithoutVisibilityExp() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getHBaseAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        Delete d1 = new Delete(row1);
        d1.addFamily(fam);

        Delete d2 = new Delete(row1);
        d2.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
        d2.addColumns(fam, qual);
        try (Connection connection = ConnectionFactory.createConnection(conf);
             Table table = connection.getTable(tableName)) {
          table.delete(createList(d1, d2));
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    Scan s = new Scan();
    s.setMaxVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 127l);
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 126l);
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 124l);
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row1, 0, row1.length));
    assertEquals(current.getTimestamp(), 123l);
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
        current.getRowLength(), row2, 0, row2.length));
    scanner.close();
  }
}