Java Code Examples for org.apache.hadoop.hbase.util.Bytes#BYTES_COMPARATOR

protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (Admin admin = connection.getAdmin()) {
    HTableDescriptor[] tableDescriptors = admin.listTables();
    LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
    if (tableDescriptors != null) {
      for (HTableDescriptor tableDescriptor : tableDescriptors) {
        if (isTransactionalTable(tableDescriptor)) {
          List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
          LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
          if (tableRegions != null) {
            for (HRegionInfo region : tableRegions) {
              regions.add(region.getRegionName());
            }
          }
        } else {
          LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
        }
      }
    }
  }
  return regions;
}
 

/**
 * Return regions that were recorded as empty after the given time.
 *
 * @param time time in milliseconds
 * @param includeRegions If not null, the returned set will be an intersection of the includeRegions set
 *                       and the empty regions after the given time
 */
public SortedSet<byte[]> getEmptyRegionsAfterTime(long time, @Nullable SortedSet<byte[]> includeRegions)
  throws IOException {
  SortedSet<byte[]> emptyRegions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (HTableInterface stateTable = stateTableSupplier.get()) {
    Scan scan = new Scan(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY),
                         EMPTY_REGION_TIME_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] emptyRegion = getEmptyRegionFromKey(next.getRow());
        if (includeRegions == null || includeRegions.contains(emptyRegion)) {
          emptyRegions.add(emptyRegion);
        }
      }
    }
  }
  return Collections.unmodifiableSortedSet(emptyRegions);
}
 

Map<byte[], String> readBulkLoadedFiles(String backupId) throws IOException {
  Scan scan = BackupSystemTable.createScanForBulkLoadedFiles(backupId);
  try (Table table = connection.getTable(bulkLoadTableName);
      ResultScanner scanner = table.getScanner(scan)) {
    Result res = null;
    Map<byte[], String> map = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    while ((res = scanner.next()) != null) {
      res.advance();
      byte[] row = CellUtil.cloneRow(res.listCells().get(0));
      for (Cell cell : res.listCells()) {
        if (CellUtil.compareQualifiers(cell, BackupSystemTable.PATH_COL, 0,
          BackupSystemTable.PATH_COL.length) == 0) {
          map.put(row, Bytes.toString(CellUtil.cloneValue(cell)));
        }
      }
    }
    return map;
  }
}
 

/**
 * Run inside the task to deserialize column family to given conf value map.
 *
 * @param conf to read the serialized values from
 * @param confName conf key to read from the configuration
 * @return a map of column family to the given configuration value
 */
private static Map<byte[], String> createFamilyConfValueMap(
    Configuration conf, String confName) {
  Map<byte[], String> confValMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  String confVal = conf.get(confName, "");
  for (String familyConf : confVal.split("&")) {
    String[] familySplit = familyConf.split("=");
    if (familySplit.length != 2) {
      continue;
    }
    try {
      confValMap.put(Bytes.toBytes(URLDecoder.decode(familySplit[0], "UTF-8")),
          URLDecoder.decode(familySplit[1], "UTF-8"));
    } catch (UnsupportedEncodingException e) {
      // will not happen with UTF-8 encoding
      throw new AssertionError(e);
    }
  }
  return confValMap;
}
 

/**
 * Was overflowing if 100k or so lists of cellscanners to return.
 */
@Test
public void testCreateCellScannerOverflow() throws IOException {
  consume(doCreateCellScanner(1, 1), 1);
  consume(doCreateCellScanner(3, 0), 0);
  consume(doCreateCellScanner(3, 3), 3 * 3);
  consume(doCreateCellScanner(0, 1), 0);
  // Do big number. See HBASE-11813 for why.
  final int hundredK = 100000;
  consume(doCreateCellScanner(hundredK, 0), 0);
  consume(doCreateCellArray(1), 1);
  consume(doCreateCellArray(0), 0);
  consume(doCreateCellArray(3), 3);
  List<CellScannable> cells = new ArrayList<>(hundredK);
  for (int i = 0; i < hundredK; i++) {
    cells.add(new TestCellScannable(1));
  }
  consume(CellUtil.createCellScanner(cells), hundredK);
  NavigableMap<byte [], List<Cell>> m = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  List<Cell> cellArray = new ArrayList<>(hundredK);
  for (int i = 0; i < hundredK; i++) {
    cellArray.add(new TestCell(i));
  }
  m.put(new byte [] {'f'}, cellArray);
  consume(CellUtil.createCellScanner(m), hundredK);
}
 

/**
 * Run inside the task to deserialize column family to given conf value map.
 *
 * @param conf to read the serialized values from
 * @param confName conf key to read from the configuration
 * @return a map of column family to the given configuration value
 */
private static Map<byte[], String> createFamilyConfValueMap(Configuration conf, String confName) {
    Map<byte[], String> confValMap = new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR);
    String confVal = conf.get(confName, "");
    for (String familyConf : confVal.split("&")) {
        String[] familySplit = familyConf.split("=");
        if (familySplit.length != 2) {
            continue;
        }
        try {
            confValMap.put(URLDecoder.decode(familySplit[0], "UTF-8").getBytes(StandardCharsets.UTF_8),
                    URLDecoder.decode(familySplit[1], "UTF-8"));
        } catch (UnsupportedEncodingException e) {
            // will not happen with UTF-8 encoding
            throw new AssertionError(e);
        }
    }
    return confValMap;
}
 

protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (Admin admin = connection.getAdmin()) {
    HTableDescriptor[] tableDescriptors = admin.listTables();
    LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
    if (tableDescriptors != null) {
      for (HTableDescriptor tableDescriptor : tableDescriptors) {
        if (isTransactionalTable(tableDescriptor)) {
          List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
          LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
          if (tableRegions != null) {
            for (HRegionInfo region : tableRegions) {
              regions.add(region.getRegionName());
            }
          }
        } else {
          LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
        }
      }
    }
  }
  return regions;
}
 

@Nullable
private TimeRegions getNextSetOfTimeRegions(Table stateTable, long time) throws IOException {
  byte[] timeBytes = Bytes.toBytes(getInvertedTime(time));
  Scan scan = new Scan(makeTimeRegionKey(timeBytes, EMPTY_BYTE_ARRAY), REGION_TIME_KEY_PREFIX_STOP);
  scan.addColumn(FAMILY, REGION_TIME_COL);


  long currentRegionTime = -1;
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  Result next;
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    while ((next = scanner.next()) != null) {
      Map.Entry<Long, byte[]> timeRegion = getTimeRegion(next.getRow());
      // Stop if reached next time value
      if (currentRegionTime == -1) {
        currentRegionTime = timeRegion.getKey();
      } else if (timeRegion.getKey() < currentRegionTime) {
        break;
      } else if (timeRegion.getKey() > currentRegionTime) {
        throw new IllegalStateException(
          String.format("Got out of order time %d when expecting time less than or equal to %d",
                        timeRegion.getKey(), currentRegionTime));
      }
      regions.add(timeRegion.getValue());
    }
  }
  return regions.isEmpty() ? null : new TimeRegions(currentRegionTime, Collections.unmodifiableSortedSet(regions));
}
 

/**
 * test get value as long
 */
@Test
public void testGetValueAsLong() {
  NavigableMap<byte[], byte[]> infoValues = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
  infoValues.put(JobHistoryKeys.KEYS_TO_BYTES.get(JobHistoryKeys.TOTAL_MAPS),
    Bytes.toBytes(JobDetailsValues.totalMaps));
  long expVal = JobDetailsValues.totalMaps;
  assertEquals(expVal, ByteUtil.getValueAsLong(JobHistoryKeys.KEYS_TO_BYTES
    .get(JobHistoryKeys.TOTAL_MAPS), infoValues));

  // test non existent value
  expVal = 0L;
  assertEquals(expVal, ByteUtil.getValueAsLong(JobHistoryKeys.KEYS_TO_BYTES
    .get(JobHistoryKeys.TOTAL_REDUCES), infoValues));
  
  infoValues.put(Constants.MEGABYTEMILLIS_BYTES, Bytes.toBytes(JobDetailsValues.megabytemillis));
  expVal = JobDetailsValues.megabytemillis;
  assertEquals(expVal, ByteUtil.getValueAsLong(Constants.MEGABYTEMILLIS_BYTES, infoValues));

  // test non existent value
  expVal = 0L;
  assertEquals(expVal, ByteUtil.getValueAsLong(Constants.HRAVEN_QUEUE_BYTES, infoValues));

  infoValues = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
  infoValues.put(Bytes.toBytes("checking_iae"),
    Bytes.toBytes("abc"));
  assertEquals(expVal, ByteUtil.getValueAsLong(Bytes.toBytes("checking_iae"), infoValues));

}
 

@SuppressWarnings("deprecation")
@Override
public void readFields(DataInput in) throws IOException {
  delegate.getKeyValues().clear();
  if (delegate.getScopes() != null) {
    delegate.getScopes().clear();
  }
  // ----------------------------------------------------------------------------------------
  // no compression, so we do pretty much what the usual WALEdit does, plus a little magic to
  // capture the index updates
  // -----------------------------------------------------------------------------------------
  int versionOrLength = in.readInt();
  if (versionOrLength != VERSION_2) {
    throw new IOException("You must update your cluster to the lastest version of HBase and"
        + " clean out all logs (cleanly start and then shutdown) before enabling indexing!");
  }
  // this is new style HLog entry containing multiple KeyValues.
  List<KeyValue> kvs = KeyValueCodec.readKeyValues(in);
  delegate.getKeyValues().addAll(kvs);

  // then read in the rest of the WALEdit
  int numFamilies = in.readInt();
  NavigableMap<byte[], Integer> scopes = delegate.getScopes();
  if (numFamilies > 0) {
    if (scopes == null) {
      scopes = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
    }
    for (int i = 0; i < numFamilies; i++) {
      byte[] fam = Bytes.readByteArray(in);
      int scope = in.readInt();
      scopes.put(fam, scope);
    }
    delegate.setScopes(scopes);
  }
}
 

@SuppressWarnings("WeakerAccess")
public PruneUpperBoundWriter(TableName tableName, DataJanitorState dataJanitorState, long pruneFlushInterval) {
  this.tableName = tableName;
  this.dataJanitorState = dataJanitorState;
  this.pruneFlushInterval = pruneFlushInterval;
  this.pruneEntries = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
  this.emptyRegions = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
}
 

@SuppressWarnings("WeakerAccess")
public PruneUpperBoundWriter(TableName tableName, DataJanitorState dataJanitorState, long pruneFlushInterval) {
  this.tableName = tableName;
  this.dataJanitorState = dataJanitorState;
  this.pruneFlushInterval = pruneFlushInterval;
  this.pruneEntries = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
  this.emptyRegions = new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR);
}
 

/**
 * Get latest prune upper bounds for given regions. This is a batch operation of method
 * {@link #getPruneUpperBoundForRegion(byte[])}
 *
 * @param regions a set of regions
 * @return a map containing region id and its latest prune upper bound value
 * @throws IOException when not able to read the data from HBase
 */
public Map<byte[], Long> getPruneUpperBoundForRegions(SortedSet<byte[]> regions) throws IOException {
  Map<byte[], Long> resultMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  List<RegionPruneInfo> regionPruneInfos = getPruneInfoForRegions(regions);
  for (RegionPruneInfo regionPruneInfo : regionPruneInfos) {
    resultMap.put(regionPruneInfo.getRegionName(), regionPruneInfo.getPruneUpperBound());
  }
  return Collections.unmodifiableMap(resultMap);
}
 

public static PTableStats readStatistics(HTableInterface statsHTable, byte[] tableNameBytes, long clientTimeStamp)
        throws IOException {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    Scan s = MetaDataUtil.newTableRowsScan(tableNameBytes, MetaDataProtocol.MIN_TABLE_TIMESTAMP, clientTimeStamp);
    s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_BYTES);
    s.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES);
    ResultScanner scanner = statsHTable.getScanner(s);
    Result result = null;
    long timeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
    TreeMap<byte[], GuidePostsInfo> guidePostsPerCf = new TreeMap<byte[], GuidePostsInfo>(
            Bytes.BYTES_COMPARATOR);
    while ((result = scanner.next()) != null) {
        CellScanner cellScanner = result.cellScanner();
        long rowCount = 0;
        ImmutableBytesPtr valuePtr = new ImmutableBytesPtr(HConstants.EMPTY_BYTE_ARRAY);
        byte[] cfName = null;
        int tableNameLength;
        int cfOffset;
        int cfLength;
        boolean valuesSet = false;
        // Only the two cells with quals GUIDE_POSTS_ROW_COUNT_BYTES and GUIDE_POSTS_BYTES would be retrieved
        while (cellScanner.advance()) {
            Cell current = cellScanner.current();
            if (!valuesSet) {
                tableNameLength = tableNameBytes.length + 1;
                cfOffset = current.getRowOffset() + tableNameLength;
                cfLength = getVarCharLength(current.getRowArray(), cfOffset, current.getRowLength()
                        - tableNameLength);
                ptr.set(current.getRowArray(), cfOffset, cfLength);
                valuesSet = true;
            }
            cfName = ByteUtil.copyKeyBytesIfNecessary(ptr);
            if (Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
                    current.getQualifierLength(), PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES, 0,
                    PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT_BYTES.length)) {
                rowCount = PLong.INSTANCE.getCodec().decodeLong(current.getValueArray(),
                        current.getValueOffset(), SortOrder.getDefault());
            } else {
                valuePtr.set(current.getValueArray(), current.getValueOffset(),
                    current.getValueLength());
            }
            if (current.getTimestamp() > timeStamp) {
                timeStamp = current.getTimestamp();
            }
        }
        if (cfName != null) {
            GuidePostsInfo newGPInfo = GuidePostsInfo.deserializeGuidePostsInfo(
                    valuePtr.get(), valuePtr.getOffset(), valuePtr.getLength(), rowCount);
            GuidePostsInfo oldInfo = guidePostsPerCf.put(cfName, newGPInfo);
            if (oldInfo != null) {
                newGPInfo.combine(oldInfo);
            }
        }
    }
    if (!guidePostsPerCf.isEmpty()) {
        return new PTableStatsImpl(guidePostsPerCf, timeStamp);
    }
    return PTableStats.EMPTY_STATS;
}
 

private Set<byte[]> getOrCreateFamilies() {
  if (this.families == null) {
    this.families = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  }
  return this.families;
}
 

@Test(timeout = 30000L) // The timeout is used to verify the fix for TEPHRA-230, the test will timeout without the fix
public void testSaveRegionTime() throws Exception {
  int maxTime = 100;

  // Nothing should be present in the beginning
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(maxTime));

  // Save regions for time
  Map<Long, SortedSet<byte[]>> regionsTime = new TreeMap<>();
  for (long time = 0; time < maxTime; time += 10) {
    SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    for (long region = 0; region < 10; region += 2) {
      regions.add(Bytes.toBytes((time * 10) + region));
    }
    regionsTime.put(time, regions);
    dataJanitorState.saveRegionsForTime(time, regions);
  }

  // Verify saved regions
  Assert.assertEquals(new TimeRegions(0, regionsTime.get(0L)), dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(31));
  Assert.assertEquals(new TimeRegions(90, regionsTime.get(90L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(maxTime + 1000));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(-10));

  // Now change the count stored for regions saved at time 0, 30 and 90
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 30L), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 90L), 0);
  }

  // Now querying for time 0 should return null, and querying for time 30 should return regions from time 20
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(35));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Querying for anything higher than 90 should give 80 (reproduces TEPHRA-230)
  Assert.assertEquals(new TimeRegions(80, regionsTime.get(80L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(Long.MAX_VALUE));

  // Delete regions saved on or before time 30
  dataJanitorState.deleteAllRegionsOnOrBeforeTime(30);
  // Values on or before time 30 should be deleted
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Counts should be deleted for time on or before 30
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 30));
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 0));
  }
  // Values after time 30 should still exist
  Assert.assertEquals(new TimeRegions(40, regionsTime.get(40L)), dataJanitorState.getRegionsOnOrBeforeTime(40));
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(5, dataJanitorState.getRegionCountForTime(stateTable, 40));
  }
}
 

/**
 * Increment the counter for the given family and column by the specified
 * amount
 * <p>
 * If the family and column already exist in the Increment the counter value
 * is incremented by the specified amount rather than overridden, as it is in
 * HBase's {@link Increment#addColumn(byte[], byte[], long)} method
 * 
 * @param inc
 *          The {@link Increment} to update
 * @param family
 *          The column family
 * @param qualifier
 *          The column qualifier
 * @param amount
 *          The amount to increment the counter by
 */
public static void addIncrement(Increment inc, final byte[] family, final byte[] qualifier, final Long amount) {
  
  NavigableMap<byte[], Long> set = inc.getFamilyMapOfLongs().get(family);
  if (set == null) {
    set = new TreeMap<byte[], Long>(Bytes.BYTES_COMPARATOR);
  }
  
  // If qualifier exists, increment amount
  Long counter = set.get(qualifier);
  if (counter == null) {
    counter = 0L;
  }
  set.put(qualifier, amount + counter);
  
  inc.getFamilyMapOfLongs().put(family, set);
}
 

@Test(timeout = 30000L) // The timeout is used to verify the fix for TEPHRA-230, the test will timeout without the fix
public void testSaveRegionTime() throws Exception {
  int maxTime = 100;

  // Nothing should be present in the beginning
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(maxTime));

  // Save regions for time
  Map<Long, SortedSet<byte[]>> regionsTime = new TreeMap<>();
  for (long time = 0; time < maxTime; time += 10) {
    SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
    for (long region = 0; region < 10; region += 2) {
      regions.add(Bytes.toBytes((time * 10) + region));
    }
    regionsTime.put(time, regions);
    dataJanitorState.saveRegionsForTime(time, regions);
  }

  // Verify saved regions
  Assert.assertEquals(new TimeRegions(0, regionsTime.get(0L)), dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  Assert.assertEquals(new TimeRegions(30, regionsTime.get(30L)), dataJanitorState.getRegionsOnOrBeforeTime(31));
  Assert.assertEquals(new TimeRegions(90, regionsTime.get(90L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(maxTime + 1000));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(-10));

  // Now change the count stored for regions saved at time 0, 30 and 90
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 30L), 3);
    dataJanitorState.saveRegionCountForTime(stateTable, Bytes.toBytes(Long.MAX_VALUE - 90L), 0);
  }

  // Now querying for time 0 should return null, and querying for time 30 should return regions from time 20
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(0));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(35));
  Assert.assertEquals(new TimeRegions(20, regionsTime.get(20L)), dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Querying for anything higher than 90 should give 80 (reproduces TEPHRA-230)
  Assert.assertEquals(new TimeRegions(80, regionsTime.get(80L)),
                      dataJanitorState.getRegionsOnOrBeforeTime(Long.MAX_VALUE));

  // Delete regions saved on or before time 30
  dataJanitorState.deleteAllRegionsOnOrBeforeTime(30);
  // Values on or before time 30 should be deleted
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(30));
  Assert.assertNull(dataJanitorState.getRegionsOnOrBeforeTime(25));
  // Counts should be deleted for time on or before 30
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 30));
    Assert.assertEquals(-1, dataJanitorState.getRegionCountForTime(stateTable, 0));
  }
  // Values after time 30 should still exist
  Assert.assertEquals(new TimeRegions(40, regionsTime.get(40L)), dataJanitorState.getRegionsOnOrBeforeTime(40));
  try (Table stateTable = testUtil.getConnection().getTable(pruneStateTable)) {
    Assert.assertEquals(5, dataJanitorState.getRegionCountForTime(stateTable, 40));
  }
}
 

@Test
public void testSecureWAL() throws Exception {
  TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  scopes.put(tableName.getName(), 0);
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
  final int total = 10;
  final byte[] row = Bytes.toBytes("row");
  final byte[] family = Bytes.toBytes("family");
  final byte[] value = Bytes.toBytes("Test value");
  FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
  final WALFactory wals =
      new WALFactory(TEST_UTIL.getConfiguration(), tableName.getNameAsString());

  // Write the WAL
  final WAL wal = wals.getWAL(regionInfo);

  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();

  for (int i = 0; i < total; i++) {
    WALEdit kvs = new WALEdit();
    kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
    wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
      System.currentTimeMillis(), mvcc, scopes), kvs);
  }
  wal.sync();
  final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
  wals.shutdown();

  // Insure edits are not plaintext
  long length = fs.getFileStatus(walPath).getLen();
  FSDataInputStream in = fs.open(walPath);
  byte[] fileData = new byte[(int)length];
  IOUtils.readFully(in, fileData);
  in.close();
  assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));

  // Confirm the WAL can be read back
  WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
  int count = 0;
  WAL.Entry entry = new WAL.Entry();
  while (reader.next(entry) != null) {
    count++;
    List<Cell> cells = entry.getEdit().getCells();
    assertTrue("Should be one KV per WALEdit", cells.size() == 1);
    for (Cell cell: cells) {
      assertTrue("Incorrect row", Bytes.equals(cell.getRowArray(), cell.getRowOffset(),
        cell.getRowLength(), row, 0, row.length));
      assertTrue("Incorrect family", Bytes.equals(cell.getFamilyArray(), cell.getFamilyOffset(),
        cell.getFamilyLength(), family, 0, family.length));
      assertTrue("Incorrect value", Bytes.equals(cell.getValueArray(), cell.getValueOffset(),
        cell.getValueLength(), value, 0, value.length));
    }
  }
  assertEquals("Should have read back as many KVs as written", total, count);
  reader.close();
}
 

/**
 * Just write multiple logs then split.  Before fix for HADOOP-2283, this
 * would fail.
 * @throws IOException
 */
@Test
public void testSplit() throws IOException {
  final TableName tableName = TableName.valueOf(currentTest.getMethodName());
  final byte [] rowName = tableName.getName();
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
  final int howmany = 3;
  RegionInfo[] infos = new RegionInfo[3];
  Path tableDataDir = CommonFSUtils.getTableDir(hbaseDir, tableName);
  fs.mkdirs(tableDataDir);
  Path tabledir = CommonFSUtils.getWALTableDir(conf, tableName);
  fs.mkdirs(tabledir);
  for (int i = 0; i < howmany; i++) {
    infos[i] = RegionInfoBuilder.newBuilder(tableName).setStartKey(Bytes.toBytes("" + i))
        .setEndKey(Bytes.toBytes("" + (i + 1))).build();
    fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));
    fs.mkdirs(new Path(tableDataDir, infos[i].getEncodedName()));
    LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());
  }
  NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  scopes.put(Bytes.toBytes("column"), 0);


  // Add edits for three regions.
  for (int ii = 0; ii < howmany; ii++) {
    for (int i = 0; i < howmany; i++) {
      final WAL log =
          wals.getWAL(infos[i]);
      for (int j = 0; j < howmany; j++) {
        WALEdit edit = new WALEdit();
        byte [] family = Bytes.toBytes("column");
        byte [] qualifier = Bytes.toBytes(Integer.toString(j));
        byte [] column = Bytes.toBytes("column:" + Integer.toString(j));
        edit.add(new KeyValue(rowName, family, qualifier,
            System.currentTimeMillis(), column));
        LOG.info("Region " + i + ": " + edit);
        WALKeyImpl walKey =  new WALKeyImpl(infos[i].getEncodedNameAsBytes(), tableName,
            System.currentTimeMillis(), mvcc, scopes);
        log.appendData(infos[i], walKey, edit);
        walKey.getWriteEntry();
      }
      log.sync();
      log.rollWriter(true);
    }
  }
  wals.shutdown();
  // The below calculation of logDir relies on insider information... WALSplitter should be connected better
  // with the WAL system.... not requiring explicit path. The oldLogDir is just made up not used.
  Path logDir =
      new Path(new Path(hbaseWALDir, HConstants.HREGION_LOGDIR_NAME),
          this.currentServername.toString());
  Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  List<Path> splits = WALSplitter.split(hbaseWALDir, logDir, oldLogDir, fs, conf, wals);
  verifySplits(splits, howmany);
}