Java Code Examples for org.apache.hadoop.hbase.util.Bytes#BYTES_COMPARATOR

The following examples show how to use org.apache.hadoop.hbase.util.Bytes#BYTES_COMPARATOR . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (Admin admin = connection.getAdmin()) {
    HTableDescriptor[] tableDescriptors = admin.listTables();
    LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
    if (tableDescriptors != null) {
      for (HTableDescriptor tableDescriptor : tableDescriptors) {
        if (isTransactionalTable(tableDescriptor)) {
          List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
          LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
          if (tableRegions != null) {
            for (HRegionInfo region : tableRegions) {
              regions.add(region.getRegionName());
            }
          }
        } else {
          LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
        }
      }
    }
  }
  return regions;
}
 
Example 2
protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (Admin admin = connection.getAdmin()) {
    HTableDescriptor[] tableDescriptors = admin.listTables();
    LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
    if (tableDescriptors != null) {
      for (HTableDescriptor tableDescriptor : tableDescriptors) {
        if (isTransactionalTable(tableDescriptor)) {
          List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
          LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
          if (tableRegions != null) {
            for (HRegionInfo region : tableRegions) {
              regions.add(region.getRegionName());
            }
          }
        } else {
          LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
        }
      }
    }
  }
  return regions;
}
 
Example 3
Source Project: kylin   File: HFileOutputFormat3.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Run inside the task to deserialize column family to given conf value map.
 *
 * @param conf to read the serialized values from
 * @param confName conf key to read from the configuration
 * @return a map of column family to the given configuration value
 */
private static Map<byte[], String> createFamilyConfValueMap(Configuration conf, String confName) {
    Map<byte[], String> confValMap = new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR);
    String confVal = conf.get(confName, "");
    for (String familyConf : confVal.split("&")) {
        String[] familySplit = familyConf.split("=");
        if (familySplit.length != 2) {
            continue;
        }
        try {
            confValMap.put(URLDecoder.decode(familySplit[0], "UTF-8").getBytes(StandardCharsets.UTF_8),
                    URLDecoder.decode(familySplit[1], "UTF-8"));
        } catch (UnsupportedEncodingException e) {
            // will not happen with UTF-8 encoding
            throw new AssertionError(e);
        }
    }
    return confValMap;
}
 
Example 4
Source Project: hbase   File: TestCellUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Was overflowing if 100k or so lists of cellscanners to return.
 */
@Test
public void testCreateCellScannerOverflow() throws IOException {
  consume(doCreateCellScanner(1, 1), 1);
  consume(doCreateCellScanner(3, 0), 0);
  consume(doCreateCellScanner(3, 3), 3 * 3);
  consume(doCreateCellScanner(0, 1), 0);
  // Do big number. See HBASE-11813 for why.
  final int hundredK = 100000;
  consume(doCreateCellScanner(hundredK, 0), 0);
  consume(doCreateCellArray(1), 1);
  consume(doCreateCellArray(0), 0);
  consume(doCreateCellArray(3), 3);
  List<CellScannable> cells = new ArrayList<>(hundredK);
  for (int i = 0; i < hundredK; i++) {
    cells.add(new TestCellScannable(1));
  }
  consume(CellUtil.createCellScanner(cells), hundredK);
  NavigableMap<byte [], List<Cell>> m = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  List<Cell> cellArray = new ArrayList<>(hundredK);
  for (int i = 0; i < hundredK; i++) {
    cellArray.add(new TestCell(i));
  }
  m.put(new byte [] {'f'}, cellArray);
  consume(CellUtil.createCellScanner(m), hundredK);
}
 
Example 5
Source Project: hbase   File: HFileOutputFormat2.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Run inside the task to deserialize column family to given conf value map.
 *
 * @param conf to read the serialized values from
 * @param confName conf key to read from the configuration
 * @return a map of column family to the given configuration value
 */
private static Map<byte[], String> createFamilyConfValueMap(
    Configuration conf, String confName) {
  Map<byte[], String> confValMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  String confVal = conf.get(confName, "");
  for (String familyConf : confVal.split("&")) {
    String[] familySplit = familyConf.split("=");
    if (familySplit.length != 2) {
      continue;
    }
    try {
      confValMap.put(Bytes.toBytes(URLDecoder.decode(familySplit[0], "UTF-8")),
          URLDecoder.decode(familySplit[1], "UTF-8"));
    } catch (UnsupportedEncodingException e) {
      // will not happen with UTF-8 encoding
      throw new AssertionError(e);
    }
  }
  return confValMap;
}
 
Example 6
Source Project: hbase   File: BackupSystemTable.java    License: Apache License 2.0 6 votes vote down vote up
Map<byte[], String> readBulkLoadedFiles(String backupId) throws IOException {
  Scan scan = BackupSystemTable.createScanForBulkLoadedFiles(backupId);
  try (Table table = connection.getTable(bulkLoadTableName);
      ResultScanner scanner = table.getScanner(scan)) {
    Result res = null;
    Map<byte[], String> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    while ((res = scanner.next()) != null) {
      res.advance();
      byte[] row = CellUtil.cloneRow(res.listCells().get(0));
      for (Cell cell : res.listCells()) {
        if (CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0,
          BackupSystemTable.PATH_COL.length) == 0) {
          map.put(row, Bytes.toString(CellUtil.cloneValue(cell)));
        }
      }
    }
    return map;
  }
}
 
Example 7
Source Project: phoenix-tephra   File: DataJanitorState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Return regions that were recorded as empty after the given time.
 *
 * @param time time in milliseconds
 * @param includeRegions If not null, the returned set will be an intersection of the includeRegions set
 *                       and the empty regions after the given time
 */
public SortedSet<byte[]> getEmptyRegionsAfterTime(long time, @Nullable SortedSet<byte[]> includeRegions)
  throws IOException {
  SortedSet<byte[]> emptyRegions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (HTableInterface stateTable = stateTableSupplier.get()) {
    Scan scan = new Scan(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY),
                         EMPTY_REGION_TIME_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] emptyRegion = getEmptyRegionFromKey(next.getRow());
        if (includeRegions == null || includeRegions.contains(emptyRegion)) {
          emptyRegions.add(emptyRegion);
        }
      }
    }
  }
  return Collections.unmodifiableSortedSet(emptyRegions);
}
 
Example 8
Source Project: phoenix-tephra   File: DataJanitorState.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get latest prune upper bounds for given regions. This is a batch operation of method
 * {@link #getPruneUpperBoundForRegion(byte[])}
 *
 * @param regions a set of regions
 * @return a map containing region id and its latest prune upper bound value
 * @throws IOException when not able to read the data from HBase
 */
public Map<byte[], Long> getPruneUpperBoundForRegions(SortedSet<byte[]> regions) throws IOException {
  Map<byte[], Long> resultMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  List<RegionPruneInfo> regionPruneInfos = getPruneInfoForRegions(regions);
  for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
    resultMap.put(regionPruneInfo.getRegionName(), regionPruneInfo.getPruneUpperBound());
  }
  return Collections.unmodifiableMap(resultMap);
}
 
Example 9
Source Project: phoenix-tephra   File: DataJanitorState.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
private TimeRegions getNextSetOfTimeRegions(Table stateTable, long time) throws IOException {
  byte[] timeBytes = Bytes.toBytes(getInvertedTime(time));
  Scan scan = new Scan(makeTimeRegionKey(timeBytes, EMPTY_BYTE_ARRAY), REGION_TIME_KEY_PREFIX_STOP);
  scan.addColumn(FAMILY, REGION_TIME_COL);


  long currentRegionTime = -1;
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  Result next;
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    while ((next = scanner.next()) != null) {
      Map.Entry<Long, byte[]> timeRegion = getTimeRegion(next.getRow());
      // Stop if reached next time value
      if (currentRegionTime == -1) {
        currentRegionTime = timeRegion.getKey();
      } else if (timeRegion.getKey() < currentRegionTime) {
        break;
      } else if (timeRegion.getKey() > currentRegionTime) {
        throw new IllegalStateException(
          String.format("Got out of order time %d when expecting time less than or equal to %d",
                        timeRegion.getKey(), currentRegionTime));
      }
      regions.add(timeRegion.getValue());
    }
  }
  return regions.isEmpty() ? null : new TimeRegions(currentRegionTime, Collections.unmodifiableSortedSet(regions));
}
 
Example 10
Source Project: hraven   File: TestByteUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * test get value as long
 */
@Test
public void testGetValueAsLong() {
  NavigableMap<byte[], byte[]> infoValues = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
  infoValues.put(JobHistoryKeys.KEYS_TO_BYTES.get(JobHistoryKeys.TOTAL_MAPS),
    Bytes.toBytes(JobDetailsValues.totalMaps));
  long expVal = JobDetailsValues.totalMaps;
  assertEquals(expVal, ByteUtil.getValueAsLong(JobHistoryKeys.KEYS_TO_BYTES
    .get(JobHistoryKeys.TOTAL_MAPS), infoValues));

  // test non existent value
  expVal = 0L;
  assertEquals(expVal, ByteUtil.getValueAsLong(JobHistoryKeys.KEYS_TO_BYTES
    .get(JobHistoryKeys.TOTAL_REDUCES), infoValues));
  
  infoValues.put(Constants.MEGABYTEMILLIS_BYTES, Bytes.toBytes(JobDetailsValues.megabytemillis));
  expVal = JobDetailsValues.megabytemillis;
  assertEquals(expVal, ByteUtil.getValueAsLong(Constants.MEGABYTEMILLIS_BYTES, infoValues));

  // test non existent value
  expVal = 0L;
  assertEquals(expVal, ByteUtil.getValueAsLong(Constants.HRAVEN_QUEUE_BYTES, infoValues));

  infoValues = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
  infoValues.put(Bytes.toBytes("checking_iae"),
    Bytes.toBytes("abc"));
  assertEquals(expVal, ByteUtil.getValueAsLong(Bytes.toBytes("checking_iae"), infoValues));

}
 
Example 11
@SuppressWarnings("deprecation")
@Override
public void readFields(DataInput in) throws IOException {
  delegate.getKeyValues().clear();
  if (delegate.getScopes() != null) {
    delegate.getScopes().clear();
  }
  // ----------------------------------------------------------------------------------------
  // no compression, so we do pretty much what the usual WALEdit does, plus a little magic to
  // capture the index updates
  // -----------------------------------------------------------------------------------------
  int versionOrLength = in.readInt();
  if (versionOrLength != VERSION_2) {
    throw new IOException("You must update your cluster to the lastest version of HBase and"
        + " clean out all logs (cleanly start and then shutdown) before enabling indexing!");
  }
  // this is new style HLog entry containing multiple KeyValues.
  List<KeyValue> kvs = KeyValueCodec.readKeyValues(in);
  delegate.getKeyValues().addAll(kvs);

  // then read in the rest of the WALEdit
  int numFamilies = in.readInt();
  NavigableMap<byte[], Integer> scopes = delegate.getScopes();
  if (numFamilies > 0) {
    if (scopes == null) {
      scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
    }
    for (int i = 0; i < numFamilies; i++) {
      byte[] fam = Bytes.readByteArray(in);
      int scope = in.readInt();
      scopes.put(fam, scope);
    }
    delegate.setScopes(scopes);
  }
}
 
Example 12
@SuppressWarnings("WeakerAccess")
public PruneUpperBoundWriter(TableName tableName, DataJanitorState dataJanitorState, long pruneFlushInterval) {
  this.tableName = tableName;
  this.dataJanitorState = dataJanitorState;
  this.pruneFlushInterval = pruneFlushInterval;
  this.pruneEntries = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
  this.emptyRegions = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
}
 
Example 13
@SuppressWarnings("WeakerAccess")
public PruneUpperBoundWriter(TableName tableName, DataJanitorState dataJanitorState, long pruneFlushInterval) {
  this.tableName = tableName;
  this.dataJanitorState = dataJanitorState;
  this.pruneFlushInterval = pruneFlushInterval;
  this.pruneEntries = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
  this.emptyRegions = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
}
 
Example 14
Source Project: hbase   File: TestWALFactory.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Just write multiple logs then split.  Before fix for HADOOP-2283, this
 * would fail.
 * @throws IOException
 */
@Test
public void testSplit() throws IOException {
  final TableName tableName = TableName.valueOf(currentTest.getMethodName());
  final byte [] rowName = tableName.getName();
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
  final int howmany = 3;
  RegionInfo[] infos = new RegionInfo[3];
  Path tableDataDir = CommonFSUtils.getTableDir(hbaseDir, tableName);
  fs.mkdirs(tableDataDir);
  Path tabledir = CommonFSUtils.getWALTableDir(conf, tableName);
  fs.mkdirs(tabledir);
  for (int i = 0; i < howmany; i++) {
    infos[i] = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("" + i))
        .setEndKey(Bytes.toBytes("" + (i + 1))).build();
    fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));
    fs.mkdirs(new Path(tableDataDir, infos[i].getEncodedName()));
    LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());
  }
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  scopes.put(Bytes.toBytes("column"), 0);


  // Add edits for three regions.
  for (int ii = 0; ii < howmany; ii++) {
    for (int i = 0; i < howmany; i++) {
      final WAL log =
          wals.getWAL(infos[i]);
      for (int j = 0; j < howmany; j++) {
        WALEdit edit = new WALEdit();
        byte [] family = Bytes.toBytes("column");
        byte [] qualifier = Bytes.toBytes(Integer.toString(j));
        byte [] column = Bytes.toBytes("column:" + Integer.toString(j));
        edit.add(new KeyValue(rowName, family, qualifier,
            System.currentTimeMillis(), column));
        LOG.info("Region " + i + ": " + edit);
        WALKeyImpl walKey =  new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName,
            System.currentTimeMillis(), mvcc, scopes);
        log.appendData(infos[i], walKey, edit);
        walKey.getWriteEntry();
      }
      log.sync();
      log.rollWriter(true);
    }
  }
  wals.shutdown();
  // The below calculation of logDir relies on insider information... WALSplitter should be connected better
  // with the WAL system.... not requiring explicit path. The oldLogDir is just made up not used.
  Path logDir =
      new Path(new Path(hbaseWALDir, HConstants.HREGION_LOGDIR_NAME),
          this.currentServername.toString());
  Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  List<Path> splits = WALSplitter.split(hbaseWALDir, logDir, oldLogDir, fs, conf, wals);
  verifySplits(splits, howmany);
}
 
Example 15
@Test(timeout = 30000L) // The timeout is used to verify the fix for TEPHRA-230, the test will timeout without the fix
public void testSaveRegionTime() throws Exception {
  int maxTime = 100;

  // Nothing should be present in the beginning
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(maxTime));

  // Save regions for time
  Map<Long, SortedSet<byte[]>> regionsTime = new TreeMap<>();
  for (long time = 0; time < maxTime; time += 10) {
    SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    for (long region = 0; region < 10; region += 2) {
      regions.add(Bytes.toBytes((time * 10) + region));
    }
    regionsTime.put(time, regions);
    dataJanitorState.saveRegionsForTime(time, regions);
  }

  // Verify saved regions
  Assert.assertEquals(new TimeRegions(0, regionsTime.get(0L)), dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(31));
  Assert.assertEquals(new TimeRegions(90, regionsTime.get(90L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(maxTime + 1000));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(-10));

  // Now change the count stored for regions saved at time 0, 30 and 90
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 30L), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 90L), 0);
  }

  // Now querying for time 0 should return null, and querying for time 30 should return regions from time 20
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(35));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Querying for anything higher than 90 should give 80 (reproduces TEPHRA-230)
  Assert.assertEquals(new TimeRegions(80, regionsTime.get(80L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(Long.MAX_VALUE));

  // Delete regions saved on or before time 30
  dataJanitorState.deleteAllRegionsOnOrBeforeTime(30);
  // Values on or before time 30 should be deleted
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Counts should be deleted for time on or before 30
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 30));
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 0));
  }
  // Values after time 30 should still exist
  Assert.assertEquals(new TimeRegions(40, regionsTime.get(40L)), dataJanitorState.getRegionsOnOrBeforeTime(40));
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(5, dataJanitorState.getRegionCountForTime(stateTable, 40));
  }
}
 
Example 16
/**
 * Increment the counter for the given family and column by the specified
 * amount
 * <p>
 * If the family and column already exist in the Increment the counter value
 * is incremented by the specified amount rather than overridden, as it is in
 * HBase's {@link Increment#addColumn(byte[], byte[], long)} method
 * 
 * @param inc
 *          The {@link Increment} to update
 * @param family
 *          The column family
 * @param qualifier
 *          The column qualifier
 * @param amount
 *          The amount to increment the counter by
 */
public static void addIncrement(Increment inc, final byte[] family, final byte[] qualifier, final Long amount) {
  
  NavigableMap<byte[], Long> set = inc.getFamilyMapOfLongs().get(family);
  if (set == null) {
    set = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
  }
  
  // If qualifier exists, increment amount
  Long counter = set.get(qualifier);
  if (counter == null) {
    counter = 0L;
  }
  set.put(qualifier, amount + counter);
  
  inc.getFamilyMapOfLongs().put(family, set);
}
 
Example 17
Source Project: hbase   File: WALEdit.java    License: Apache License 2.0 4 votes vote down vote up
private Set<byte[]> getOrCreateFamilies() {
  if (this.families == null) {
    this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  }
  return this.families;
}
 
Example 18
Source Project: phoenix   File: StatisticsUtil.java    License: Apache License 2.0 4 votes vote down vote up
public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, long clientTimeStamp)
        throws IOException {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    Scan s = MetaDataUtil.newTableRowsScan(tableNameBytes, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp);
    s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_BYTES);
    s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES);
    ResultScanner scanner = statsHTable.getScanner(s);
    Result result = null;
    long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
    TreeMap<byte[], GuidePostsInfo> guidePostsPerCf = new TreeMap<byte[], GuidePostsInfo>(
            Bytes.BYTES_COMPARATOR);
    while ((result = scanner.next()) != null) {
        CellScanner cellScanner = result.cellScanner();
        long rowCount = 0;
        ImmutableBytesPtr valuePtr = new ImmutableBytesPtr(HConstants.EMPTY_BYTE_ARRAY);
        byte[] cfName = null;
        int tableNameLength;
        int cfOffset;
        int cfLength;
        boolean valuesSet = false;
        // Only the two cells with quals GUIDE_POSTS_ROW_COUNT_BYTES and GUIDE_POSTS_BYTES would be retrieved
        while (cellScanner.advance()) {
            Cell current = cellScanner.current();
            if (!valuesSet) {
                tableNameLength = tableNameBytes.length + 1;
                cfOffset = current.getRowOffset() + tableNameLength;
                cfLength = getVarCharLength(current.getRowArray(), cfOffset, current.getRowLength()
                        - tableNameLength);
                ptr.set(current.getRowArray(), cfOffset, cfLength);
                valuesSet = true;
            }
            cfName = ByteUtil.copyKeyBytesIfNecessary(ptr);
            if (Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
                    current.getQualifierLength(), PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, 0,
                    PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES.length)) {
                rowCount = PLong.INSTANCE.getCodec().decodeLong(current.getValueArray(),
                        current.getValueOffset(), SortOrder.getDefault());
            } else {
                valuePtr.set(current.getValueArray(), current.getValueOffset(),
                    current.getValueLength());
            }
            if (current.getTimestamp() > timeStamp) {
                timeStamp = current.getTimestamp();
            }
        }
        if (cfName != null) {
            GuidePostsInfo newGPInfo = GuidePostsInfo.deserializeGuidePostsInfo(
                    valuePtr.get(), valuePtr.getOffset(), valuePtr.getLength(), rowCount);
            GuidePostsInfo oldInfo = guidePostsPerCf.put(cfName, newGPInfo);
            if (oldInfo != null) {
                newGPInfo.combine(oldInfo);
            }
        }
    }
    if (!guidePostsPerCf.isEmpty()) {
        return new PTableStatsImpl(guidePostsPerCf, timeStamp);
    }
    return PTableStats.EMPTY_STATS;
}
 
Example 19
@Test(timeout = 30000L) // The timeout is used to verify the fix for TEPHRA-230, the test will timeout without the fix
public void testSaveRegionTime() throws Exception {
  int maxTime = 100;

  // Nothing should be present in the beginning
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(maxTime));

  // Save regions for time
  Map<Long, SortedSet<byte[]>> regionsTime = new TreeMap<>();
  for (long time = 0; time < maxTime; time += 10) {
    SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    for (long region = 0; region < 10; region += 2) {
      regions.add(Bytes.toBytes((time * 10) + region));
    }
    regionsTime.put(time, regions);
    dataJanitorState.saveRegionsForTime(time, regions);
  }

  // Verify saved regions
  Assert.assertEquals(new TimeRegions(0, regionsTime.get(0L)), dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(31));
  Assert.assertEquals(new TimeRegions(90, regionsTime.get(90L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(maxTime + 1000));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(-10));

  // Now change the count stored for regions saved at time 0, 30 and 90
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 30L), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 90L), 0);
  }

  // Now querying for time 0 should return null, and querying for time 30 should return regions from time 20
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(35));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Querying for anything higher than 90 should give 80 (reproduces TEPHRA-230)
  Assert.assertEquals(new TimeRegions(80, regionsTime.get(80L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(Long.MAX_VALUE));

  // Delete regions saved on or before time 30
  dataJanitorState.deleteAllRegionsOnOrBeforeTime(30);
  // Values on or before time 30 should be deleted
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Counts should be deleted for time on or before 30
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 30));
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 0));
  }
  // Values after time 30 should still exist
  Assert.assertEquals(new TimeRegions(40, regionsTime.get(40L)), dataJanitorState.getRegionsOnOrBeforeTime(40));
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(5, dataJanitorState.getRegionCountForTime(stateTable, 40));
  }
}
 
Example 20
Source Project: hbase   File: TestSecureWAL.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSecureWAL() throws Exception {
  TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  scopes.put(tableName.getName(), 0);
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
  final int total = 10;
  final byte[] row = Bytes.toBytes("row");
  final byte[] family = Bytes.toBytes("family");
  final byte[] value = Bytes.toBytes("Test value");
  FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
  final WALFactory wals =
      new WALFactory(TEST_UTIL.getConfiguration(), tableName.getNameAsString());

  // Write the WAL
  final WAL wal = wals.getWAL(regionInfo);

  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();

  for (int i = 0; i < total; i++) {
    WALEdit kvs = new WALEdit();
    kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
    wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
      System.currentTimeMillis(), mvcc, scopes), kvs);
  }
  wal.sync();
  final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
  wals.shutdown();

  // Insure edits are not plaintext
  long length = fs.getFileStatus(walPath).getLen();
  FSDataInputStream in = fs.open(walPath);
  byte[] fileData = new byte[(int)length];
  IOUtils.readFully(in, fileData);
  in.close();
  assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));

  // Confirm the WAL can be read back
  WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
  int count = 0;
  WAL.Entry entry = new WAL.Entry();
  while (reader.next(entry) != null) {
    count++;
    List<Cell> cells = entry.getEdit().getCells();
    assertTrue("Should be one KV per WALEdit", cells.size() == 1);
    for (Cell cell: cells) {
      assertTrue("Incorrect row", Bytes.equals(cell.getRowArray(), cell.getRowOffset(),
        cell.getRowLength(), row, 0, row.length));
      assertTrue("Incorrect family", Bytes.equals(cell.getFamilyArray(), cell.getFamilyOffset(),
        cell.getFamilyLength(), family, 0, family.length));
      assertTrue("Incorrect value", Bytes.equals(cell.getValueArray(), cell.getValueOffset(),
        cell.getValueLength(), value, 0, value.length));
    }
  }
  assertEquals("Should have read back as many KVs as written", total, count);
  reader.close();
}