org.apache.hadoop.hbase.Size Java Examples

The following examples show how to use org.apache.hadoop.hbase.Size. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HbaseSessions.java    From hugegraph with Apache License 2.0 6 votes vote down vote up
public long storeSize(String table) throws IOException {
    long total = 0;
    try (Admin admin = this.hbase.getAdmin()) {
        for (ServerName rs : admin.getRegionServers()) {
            // NOTE: we can use getLoad() before hbase 2.0
            //ServerLoad load = admin.getClusterStatus().getLoad(rs);
            //total += load.getStorefileSizeMB() * Bytes.MB;
            //total += load.getMemStoreSizeMB() * Bytes.MB;
            TableName tableName = TableName.valueOf(this.namespace, table);
            for (RegionMetrics m : admin.getRegionMetrics(rs, tableName)) {
                total += m.getStoreFileSize().get(Size.Unit.BYTE);
                total += m.getMemStoreSize().get(Size.Unit.BYTE);
            }
        }
    }
    return total;
}
 
Example #2
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void assertRecordInRegionServerMode(Record record, String longRegionServer,
  long requestCountPerSecond, long readRequestCountPerSecond,
  long filteredReadRequestCountPerSecond, long writeCountRequestPerSecond,
  Size storeFileSize, Size uncompressedStoreFileSize, int numStoreFiles,
  Size memStoreSize, int regionCount, Size usedHeapSize, Size maxHeapSize) {
  assertThat(record.size(), is(13));
  assertThat(record.get(Field.LONG_REGION_SERVER).asString(),
    is(longRegionServer));
  assertThat(record.get(Field.REQUEST_COUNT_PER_SECOND).asLong(),
    is(requestCountPerSecond));
  assertThat(record.get(Field.READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(readRequestCountPerSecond));
  assertThat(record.get(Field.FILTERED_READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(filteredReadRequestCountPerSecond));
  assertThat(record.get(Field.WRITE_REQUEST_COUNT_PER_SECOND).asLong(),
    is(writeCountRequestPerSecond));
  assertThat(record.get(Field.STORE_FILE_SIZE).asSize(), is(storeFileSize));
  assertThat(record.get(Field.UNCOMPRESSED_STORE_FILE_SIZE).asSize(),
    is(uncompressedStoreFileSize));
  assertThat(record.get(Field.NUM_STORE_FILES).asInt(), is(numStoreFiles));
  assertThat(record.get(Field.MEM_STORE_SIZE).asSize(), is(memStoreSize));
  assertThat(record.get(Field.REGION_COUNT).asInt(), is(regionCount));
  assertThat(record.get(Field.USED_HEAP_SIZE).asSize(), is(usedHeapSize));
  assertThat(record.get(Field.MAX_HEAP_SIZE).asSize(), is(maxHeapSize));
}
 
Example #3
Source File: TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java    From hbase with Apache License 2.0 6 votes vote down vote up
private ServerMetrics mockServerMetricsWithReadRequests(ServerName server,
    List<RegionInfo> regionsOnServer, long readRequestCount) {
  ServerMetrics serverMetrics = mock(ServerMetrics.class);
  Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(RegionInfo info : regionsOnServer){
    RegionMetrics rl = mock(RegionMetrics.class);
    when(rl.getReadRequestCount()).thenReturn(readRequestCount);
    when(rl.getCpRequestCount()).thenReturn(0L);
    when(rl.getWriteRequestCount()).thenReturn(0L);
    when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
    when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
    regionLoadMap.put(info.getRegionName(), rl);
  }
  when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
  return serverMetrics;
}
 
Example #4
Source File: TestStochasticLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
private ServerMetrics mockServerMetricsWithCpRequests(ServerName server,
                                                      List<RegionInfo> regionsOnServer,
                                                      long cpRequestCount) {
  ServerMetrics serverMetrics = mock(ServerMetrics.class);
  Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(RegionInfo info : regionsOnServer){
    RegionMetrics rl = mock(RegionMetrics.class);
    when(rl.getReadRequestCount()).thenReturn(0L);
    when(rl.getCpRequestCount()).thenReturn(cpRequestCount);
    when(rl.getWriteRequestCount()).thenReturn(0L);
    when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
    when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
    regionLoadMap.put(info.getRegionName(), rl);
  }
  when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
  return serverMetrics;
}
 
Example #5
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static void assertRecordsInRegionServerMode(List<Record> records) {
  assertThat(records.size(), is(2));

  for (Record record : records) {
    switch (record.get(Field.REGION_SERVER).asString()) {
      case "host1:1000":
        assertRecordInRegionServerMode(record, "host1.apache.com,1000,1", 0L, 0L, 0L, 0L,
          new Size(600, Size.Unit.MEGABYTE), new Size(1200, Size.Unit.MEGABYTE), 6,
          new Size(600, Size.Unit.MEGABYTE), 3, new Size(100, Size.Unit.MEGABYTE),
          new Size(200, Size.Unit.MEGABYTE));
        break;

      case "host2:1001":
        assertRecordInRegionServerMode(record, "host2.apache.com,1001,2", 0L, 0L, 0L, 0L,
          new Size(600, Size.Unit.MEGABYTE), new Size(1200, Size.Unit.MEGABYTE), 6,
          new Size(600, Size.Unit.MEGABYTE), 3, new Size(16, Size.Unit.GIGABYTE),
          new Size(32, Size.Unit.GIGABYTE));
        break;

      default:
        fail();
    }
  }
}
 
Example #6
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void assertRecordInTableMode(Record record, long requestCountPerSecond,
    long readRequestCountPerSecond, long filteredReadRequestCountPerSecond,
    long writeCountRequestPerSecond, Size storeFileSize, Size uncompressedStoreFileSize,
    int numStoreFiles, Size memStoreSize, int regionCount) {
  assertThat(record.size(), is(11));
  assertThat(record.get(Field.REQUEST_COUNT_PER_SECOND).asLong(),
    is(requestCountPerSecond));
  assertThat(record.get(Field.READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(readRequestCountPerSecond));
  assertThat(record.get(Field.FILTERED_READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(filteredReadRequestCountPerSecond));
  assertThat(record.get(Field.WRITE_REQUEST_COUNT_PER_SECOND).asLong(),
    is(writeCountRequestPerSecond));
  assertThat(record.get(Field.STORE_FILE_SIZE).asSize(), is(storeFileSize));
  assertThat(record.get(Field.UNCOMPRESSED_STORE_FILE_SIZE).asSize(),
    is(uncompressedStoreFileSize));
  assertThat(record.get(Field.NUM_STORE_FILES).asInt(), is(numStoreFiles));
  assertThat(record.get(Field.MEM_STORE_SIZE).asSize(), is(memStoreSize));
  assertThat(record.get(Field.REGION_COUNT).asInt(), is(regionCount));
}
 
Example #7
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void assertRecordInNamespaceMode(Record record, long requestCountPerSecond,
  long readRequestCountPerSecond, long filteredReadRequestCountPerSecond,
  long writeCountRequestPerSecond, Size storeFileSize, Size uncompressedStoreFileSize,
  int numStoreFiles, Size memStoreSize, int regionCount) {
  assertThat(record.size(), is(10));
  assertThat(record.get(Field.REQUEST_COUNT_PER_SECOND).asLong(),
    is(requestCountPerSecond));
  assertThat(record.get(Field.READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(readRequestCountPerSecond));
  assertThat(record.get(Field.FILTERED_READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(filteredReadRequestCountPerSecond));
  assertThat(record.get(Field.WRITE_REQUEST_COUNT_PER_SECOND).asLong(),
    is(writeCountRequestPerSecond));
  assertThat(record.get(Field.STORE_FILE_SIZE).asSize(), is(storeFileSize));
  assertThat(record.get(Field.UNCOMPRESSED_STORE_FILE_SIZE).asSize(),
    is(uncompressedStoreFileSize));
  assertThat(record.get(Field.NUM_STORE_FILES).asInt(), is(numStoreFiles));
  assertThat(record.get(Field.MEM_STORE_SIZE).asSize(), is(memStoreSize));
  assertThat(record.get(Field.REGION_COUNT).asInt(), is(regionCount));
}
 
Example #8
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static void assertRecordsInNamespaceMode(List<Record> records) {
  assertThat(records.size(), is(2));

  for (Record record : records) {
    switch (record.get(Field.NAMESPACE).asString()) {
      case "default":
        assertRecordInNamespaceMode(record, 0L, 0L, 0L, 0L, new Size(600, Size.Unit.MEGABYTE),
          new Size(1200, Size.Unit.MEGABYTE), 6, new Size(600, Size.Unit.MEGABYTE), 4);
        break;

      case "namespace":
        assertRecordInNamespaceMode(record, 0L, 0L, 0L, 0L, new Size(600, Size.Unit.MEGABYTE),
          new Size(1200, Size.Unit.MEGABYTE), 6, new Size(600, Size.Unit.MEGABYTE), 2);
        break;

      default:
        fail();
    }
  }
}
 
Example #9
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static RegionMetrics createRegionMetrics(String regionName, long readRequestCount,
  long filteredReadRequestCount, long writeRequestCount, Size storeFileSize,
  Size uncompressedStoreFileSize, int storeFileCount, Size memStoreSize, float locality,
  long compactedCellCount, long compactingCellCount, String lastMajorCompactionTime) {

  FastDateFormat df = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
  try {
    return RegionMetricsBuilder.newBuilder(Bytes.toBytes(regionName))
      .setReadRequestCount(readRequestCount)
      .setFilteredReadRequestCount(filteredReadRequestCount)
      .setWriteRequestCount(writeRequestCount).setStoreFileSize(storeFileSize)
      .setUncompressedStoreFileSize(uncompressedStoreFileSize).setStoreFileCount(storeFileCount)
      .setMemStoreSize(memStoreSize).setDataLocality(locality)
      .setCompactedCellCount(compactedCellCount).setCompactingCellCount(compactingCellCount)
      .setLastMajorCompactionTimestamp(df.parse(lastMajorCompactionTime).getTime()).build();
  } catch (ParseException e) {
    throw new IllegalArgumentException(e);
  }
}
 
Example #10
Source File: FieldValue.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Size optimizeSize(Size size) {
  if (size.get(Size.Unit.BYTE) < 1024d) {
    return size.getUnit() == Size.Unit.BYTE ?
      size : new Size(size.get(Size.Unit.BYTE), Size.Unit.BYTE);
  } else if (size.get(Size.Unit.KILOBYTE) < 1024d) {
    return size.getUnit() == Size.Unit.KILOBYTE ?
      size : new Size(size.get(Size.Unit.KILOBYTE), Size.Unit.KILOBYTE);
  } else if (size.get(Size.Unit.MEGABYTE) < 1024d) {
    return size.getUnit() == Size.Unit.MEGABYTE ?
      size : new Size(size.get(Size.Unit.MEGABYTE), Size.Unit.MEGABYTE);
  } else if (size.get(Size.Unit.GIGABYTE) < 1024d) {
    return size.getUnit() == Size.Unit.GIGABYTE ?
      size : new Size(size.get(Size.Unit.GIGABYTE), Size.Unit.GIGABYTE);
  } else if (size.get(Size.Unit.TERABYTE) < 1024d) {
    return size.getUnit() == Size.Unit.TERABYTE ?
      size : new Size(size.get(Size.Unit.TERABYTE), Size.Unit.TERABYTE);
  }
  return size.getUnit() == Size.Unit.PETABYTE ?
    size : new Size(size.get(Size.Unit.PETABYTE), Size.Unit.PETABYTE);
}
 
Example #11
Source File: TestUtils.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static void assertRecordsInTableMode(List<Record> records) {
  assertThat(records.size(), is(3));

  for (Record record : records) {
    String tableName = String.format("%s:%s", record.get(Field.NAMESPACE).asString(),
      record.get(Field.TABLE).asString());

    switch (tableName) {
      case "default:table1":
        assertRecordInTableMode(record, 0L, 0L, 0L, 0L, new Size(200, Size.Unit.MEGABYTE),
          new Size(400, Size.Unit.MEGABYTE), 2, new Size(200, Size.Unit.MEGABYTE), 2);
        break;

      case "default:table2":
        assertRecordInTableMode(record, 0L, 0L, 0L, 0L, new Size(400, Size.Unit.MEGABYTE),
          new Size(800, Size.Unit.MEGABYTE), 4, new Size(400, Size.Unit.MEGABYTE), 2);
        break;

      case "namespace:table3":
        assertRecordInTableMode(record, 0L, 0L, 0L, 0L, new Size(600, Size.Unit.MEGABYTE),
          new Size(1200, Size.Unit.MEGABYTE), 6, new Size(600, Size.Unit.MEGABYTE), 2);
        break;

      default:
        fail();
    }
  }
}
 
Example #12
Source File: FieldValue.java    From hbase with Apache License 2.0 5 votes vote down vote up
public FieldValue plus(FieldValue o) {
  if (type != o.type) {
    throw new IllegalArgumentException("invalid type");
  }

  switch (type) {
    case STRING:
      return new FieldValue(((String) value).concat((String) o.value), type);

    case INTEGER:
      return new FieldValue(((Integer) value) + ((Integer) o.value), type);

    case LONG:
      return new FieldValue(((Long) value) + ((Long) o.value), type);

    case FLOAT:
    case PERCENT:
      return new FieldValue(((Float) value) + ((Float) o.value), type);

    case SIZE:
      Size size = (Size) value;
      Size oSize = (Size) o.value;
      Size.Unit unit = size.getUnit();
      return new FieldValue(new Size(size.get(unit) + oSize.get(unit), unit), type);

    default:
      throw new AssertionError();
  }
}
 
Example #13
Source File: TestRecordFilter.java    From hbase with Apache License 2.0 5 votes vote down vote up
private List<Record> createTestRecords() {
  List<Record> ret = new ArrayList<>();
  ret.add(createTestRecord("region1", 100L, new Size(100, Size.Unit.MEGABYTE), 2, 1.0f, 80f));
  ret.add(createTestRecord("region2", 120L, new Size(100, Size.Unit.GIGABYTE), 10, 0.5f, 20f));
  ret.add(createTestRecord("region3", 50L, new Size(500, Size.Unit.KILOBYTE), 15, 0.8f, 50f));
  ret.add(createTestRecord("region4", 90L, new Size(10, Size.Unit.TERABYTE), 5, 0.9f, 30f));
  ret.add(createTestRecord("region5", 200L, new Size(1, Size.Unit.PETABYTE), 13, 0.1f, 40f));
  return ret;
}
 
Example #14
Source File: TestRecordFilter.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Record createTestRecord(String region, long requestCountPerSecond,
  Size storeFileSize, int numStoreFiles, float locality, float compactionProgress) {
  Record.Builder builder = Record.builder();
  builder.put(Field.REGION, region);
  builder.put(Field.REQUEST_COUNT_PER_SECOND, requestCountPerSecond);
  builder.put(Field.STORE_FILE_SIZE, storeFileSize);
  builder.put(Field.NUM_STORE_FILES, numStoreFiles);
  builder.put(Field.LOCALITY, locality);
  builder.put(Field.COMPACTION_PROGRESS, compactionProgress);
  return builder.build();
}
 
Example #15
Source File: TestUtils.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static ServerMetrics createServerMetrics(ServerName serverName, long reportTimestamp,
  Size usedHeapSize, Size maxHeapSize, long requestCountPerSecond,
  List<RegionMetrics> regionMetricsList, List<UserMetrics> userMetricsList) {

  return ServerMetricsBuilder.newBuilder(serverName)
    .setReportTimestamp(reportTimestamp)
    .setUsedHeapSize(usedHeapSize)
    .setMaxHeapSize(maxHeapSize)
    .setRequestCountPerSecond(requestCountPerSecond)
    .setRegionMetrics(regionMetricsList)
    .setUserMetrics(userMetricsList).build();
}
 
Example #16
Source File: TestUtils.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static void assertRecordInRegionMode(Record record, String namespace, String startCode,
  String replicaId, String table, String region, String regionServer, String longRegionServer,
  long requestCountPerSecond, long readRequestCountPerSecond,
  long filteredReadRequestCountPerSecond, long writeCountRequestPerSecond,
  Size storeFileSize, Size uncompressedStoreFileSize, int numStoreFiles,
  Size memStoreSize, float locality, String startKey, long compactingCellCount,
  long compactedCellCount, float compactionProgress, String lastMajorCompactionTime) {
  assertThat(record.size(), is(22));
  assertThat(record.get(Field.NAMESPACE).asString(), is(namespace));
  assertThat(record.get(Field.START_CODE).asString(), is(startCode));
  assertThat(record.get(Field.REPLICA_ID).asString(), is(replicaId));
  assertThat(record.get(Field.TABLE).asString(), is(table));
  assertThat(record.get(Field.REGION).asString(), is(region));
  assertThat(record.get(Field.REGION_SERVER).asString(), is(regionServer));
  assertThat(record.get(Field.LONG_REGION_SERVER).asString(), is(longRegionServer));
  assertThat(record.get(Field.REQUEST_COUNT_PER_SECOND).asLong(),
    is(requestCountPerSecond));
  assertThat(record.get(Field.READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(readRequestCountPerSecond));
  assertThat(record.get(Field.FILTERED_READ_REQUEST_COUNT_PER_SECOND).asLong(),
    is(filteredReadRequestCountPerSecond));
  assertThat(record.get(Field.WRITE_REQUEST_COUNT_PER_SECOND).asLong(),
    is(writeCountRequestPerSecond));
  assertThat(record.get(Field.STORE_FILE_SIZE).asSize(), is(storeFileSize));
  assertThat(record.get(Field.UNCOMPRESSED_STORE_FILE_SIZE).asSize(),
    is(uncompressedStoreFileSize));
  assertThat(record.get(Field.NUM_STORE_FILES).asInt(), is(numStoreFiles));
  assertThat(record.get(Field.MEM_STORE_SIZE).asSize(), is(memStoreSize));
  assertThat(record.get(Field.LOCALITY).asFloat(), is(locality));
  assertThat(record.get(Field.START_KEY).asString(), is(startKey));
  assertThat(record.get(Field.COMPACTING_CELL_COUNT).asLong(), is(compactingCellCount));
  assertThat(record.get(Field.COMPACTED_CELL_COUNT).asLong(), is(compactedCellCount));
  assertThat(record.get(Field.COMPACTION_PROGRESS).asFloat(), is(compactionProgress));
  assertThat(record.get(Field.LAST_MAJOR_COMPACTION_TIME).asString(),
    is(lastMajorCompactionTime));
}
 
Example #17
Source File: FieldValue.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(@NonNull FieldValue o) {
  if (type != o.type) {
    throw new IllegalArgumentException("invalid type");
  }

  switch (type) {
    case STRING:
      return ((String) value).compareTo((String) o.value);

    case INTEGER:
      return ((Integer) value).compareTo((Integer) o.value);

    case LONG:
      return ((Long) value).compareTo((Long) o.value);

    case FLOAT:
    case PERCENT:
      return ((Float) value).compareTo((Float) o.value);

    case SIZE:
      return ((Size) value).compareTo((Size) o.value);

    default:
      throw new AssertionError();
  }
}
 
Example #18
Source File: TestFieldValue.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testPlus() {
  // String
  FieldValue stringFieldValue = new FieldValue("a", FieldValueType.STRING);
  FieldValue stringFieldValue2 = new FieldValue("b", FieldValueType.STRING);
  assertThat(stringFieldValue.plus(stringFieldValue2).asString(), is("ab"));

  // Integer
  FieldValue integerFieldValue = new FieldValue(1, FieldValueType.INTEGER);
  FieldValue integerFieldValue2 = new FieldValue(2, FieldValueType.INTEGER);
  assertThat(integerFieldValue.plus(integerFieldValue2).asInt(), is(3));

  // Long
  FieldValue longFieldValue = new FieldValue(1L, FieldValueType.LONG);
  FieldValue longFieldValue2 = new FieldValue(2L, FieldValueType.LONG);
  assertThat(longFieldValue.plus(longFieldValue2).asLong(), is(3L));

  // Float
  FieldValue floatFieldValue = new FieldValue(1.2f, FieldValueType.FLOAT);
  FieldValue floatFieldValue2 = new FieldValue(2.2f, FieldValueType.FLOAT);
  assertThat(floatFieldValue.plus(floatFieldValue2).asFloat(), is(3.4f));

  // Size
  FieldValue sizeFieldValue =
    new FieldValue(new Size(100, Size.Unit.MEGABYTE), FieldValueType.SIZE);
  FieldValue sizeFieldValue2 =
    new FieldValue(new Size(200, Size.Unit.MEGABYTE), FieldValueType.SIZE);
  assertThat(sizeFieldValue.plus(sizeFieldValue2).asString(), is("300.0MB"));
  assertThat(sizeFieldValue.plus(sizeFieldValue2).asSize(),
    is(new Size(300, Size.Unit.MEGABYTE)));

  // Percent
  FieldValue percentFieldValue = new FieldValue(30f, FieldValueType.PERCENT);
  FieldValue percentFieldValue2 = new FieldValue(60f, FieldValueType.PERCENT);
  assertThat(percentFieldValue.plus(percentFieldValue2).asString(), is("90.00%"));
  assertThat(percentFieldValue.plus(percentFieldValue2).asFloat(), is(90f));
}
 
Example #19
Source File: HServerLoad.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public Set<PartitionLoad> getPartitionLoads(){
    Map<byte[], RegionMetrics> regionsLoad=load.getRegionMetrics();
    Set<PartitionLoad> loads = new HashSet<>(regionsLoad.size(),0.9f);
    for(Map.Entry<byte[], RegionMetrics> regionLoad:regionsLoad.entrySet()){
        String name = Bytes.toString(regionLoad.getKey());
        RegionMetrics rm = regionLoad.getValue();
        PartitionLoad pl = new HPartitionLoad(name, (long) rm.getStoreFileSize().get(Size.Unit.BYTE), (long) rm.getMemStoreSize().get(Size.Unit.BYTE));
        loads.add(pl);
    }
    return loads;
}
 
Example #20
Source File: RegionSizeCalculator.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void init(RegionLocator regionLocator, Admin admin)
    throws IOException {
  if (!enabled(admin.getConfiguration())) {
    LOG.info("Region size calculation disabled.");
    return;
  }

  if (regionLocator.getName().isSystemTable()) {
    LOG.info("Region size calculation disabled for system tables.");
    return;
  }

  LOG.info("Calculating region sizes for table \"" + regionLocator.getName() + "\".");

  // Get the servers which host regions of the table
  Set<ServerName> tableServers = getRegionServersOfTable(regionLocator);

  for (ServerName tableServerName : tableServers) {
    for (RegionMetrics regionLoad : admin.getRegionMetrics(
      tableServerName,regionLocator.getName())) {

      byte[] regionId = regionLoad.getRegionName();
      long regionSizeBytes
        = ((long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE)) * MEGABYTE;

      sizeMap.put(regionId, regionSizeBytes);

      if (LOG.isDebugEnabled()) {
        LOG.debug("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
      }
    }
  }
  LOG.debug("Region sizes calculated");
}
 
Example #21
Source File: HbaseMetrics.java    From hugegraph with Apache License 2.0 5 votes vote down vote up
private static Map<String, Object> formatMetrics(RegionMetrics region) {
    Map<String, Object> metrics = InsertionOrderUtil.newMap();
    metrics.put("mem_store_size",
                region.getMemStoreSize().get(Size.Unit.MEGABYTE));
    metrics.put("file_store_size",
                region.getStoreFileSize().get(Size.Unit.MEGABYTE));
    metrics.put("store_size_unit", "MB");
    return metrics;
}
 
Example #22
Source File: BalancerRegionLoad.java    From hbase with Apache License 2.0 5 votes vote down vote up
BalancerRegionLoad(RegionMetrics regionMetrics) {
  readRequestsCount = regionMetrics.getReadRequestCount();
  cpRequestsCount = regionMetrics.getCpRequestCount();
  writeRequestsCount = regionMetrics.getWriteRequestCount();
  memStoreSizeMB = (int) regionMetrics.getMemStoreSize().get(Size.Unit.MEGABYTE);
  storefileSizeMB = (int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
}
 
Example #23
Source File: SimpleRegionNormalizer.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @return size of region in MB and if region is not found than -1
 */
private long getRegionSizeMB(RegionInfo hri) {
  ServerName sn =
    masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(hri);
  RegionMetrics regionLoad =
    masterServices.getServerManager().getLoad(sn).getRegionMetrics().get(hri.getRegionName());
  if (regionLoad == null) {
    LOG.debug("{} was not found in RegionsLoad", hri.getRegionNameAsString());
    return -1;
  }
  return (long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE);
}
 
Example #24
Source File: TestSimpleRegionNormalizerOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static double getRegionSizeMB(final MasterServices masterServices,
  final RegionInfo regionInfo) {
  final ServerName sn = masterServices.getAssignmentManager()
    .getRegionStates()
    .getRegionServerOfRegion(regionInfo);
  final RegionMetrics regionLoad = masterServices.getServerManager()
    .getLoad(sn)
    .getRegionMetrics()
    .get(regionInfo.getRegionName());
  if (regionLoad == null) {
    LOG.debug("{} was not found in RegionsLoad", regionInfo.getRegionNameAsString());
    return -1;
  }
  return regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE);
}
 
Example #25
Source File: TestSimpleRegionNormalizer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("MockitoCast")
private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
  List<RegionInfo> regionInfoList) {
  masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);

  // for simplicity all regions are assumed to be on one server; doesn't matter to us
  ServerName sn = ServerName.valueOf("localhost", 0, 0L);
  when(masterServices.getAssignmentManager().getRegionStates()
    .getRegionsOfTable(any())).thenReturn(regionInfoList);
  when(masterServices.getAssignmentManager().getRegionStates()
    .getRegionServerOfRegion(any())).thenReturn(sn);
  when(masterServices.getAssignmentManager().getRegionStates()
    .getRegionState(any(RegionInfo.class))).thenReturn(
      RegionState.createForTesting(null, RegionState.State.OPEN));

  for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
    RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class);
    when(regionLoad.getRegionName()).thenReturn(region.getKey());
    when(regionLoad.getStoreFileSize())
      .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE));

    // this is possibly broken with jdk9, unclear if false positive or not
    // suppress it for now, fix it when we get to running tests on 9
    // see: http://errorprone.info/bugpattern/MockitoCast
    when((Object) masterServices.getServerManager().getLoad(sn)
      .getRegionMetrics().get(region.getKey())).thenReturn(regionLoad);
  }

  when(masterServices.isSplitOrMergeEnabled(any())).thenReturn(true);

  normalizer = new SimpleRegionNormalizer();
  normalizer.setConf(conf);
  normalizer.setMasterServices(masterServices);
}
 
Example #26
Source File: HbaseMetrics.java    From hugegraph with Apache License 2.0 5 votes vote down vote up
private static Map<String, Object> formatMetrics(
                                   ServerMetrics serverMetrics,
                                   List<RegionMetrics> regions) {
    Map<String, Object> metrics = InsertionOrderUtil.newMap();
    metrics.put("max_heap_size",
                serverMetrics.getMaxHeapSize().get(Size.Unit.MEGABYTE));
    metrics.put("used_heap_size",
                serverMetrics.getUsedHeapSize().get(Size.Unit.MEGABYTE));
    metrics.put("heap_size_unit", "MB");
    metrics.put("request_count", serverMetrics.getRequestCount());
    metrics.put("request_count_per_second",
                serverMetrics.getRequestCountPerSecond());
    metrics.put("regions", formatMetrics(regions));
    return metrics;
}
 
Example #27
Source File: TestRegionSizeCalculator.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Creates mock of region with given name and size.
 *
 * @param  fileSizeMb number of megabytes occupied by region in file store in megabytes
 * */
private RegionMetrics mockRegion(String regionName, int fileSizeMb) {
  RegionMetrics region = Mockito.mock(RegionMetrics.class);
  when(region.getRegionName()).thenReturn(Bytes.toBytes(regionName));
  when(region.getNameAsString()).thenReturn(regionName);
  when(region.getStoreFileSize()).thenReturn(new Size(fileSizeMb, Size.Unit.MEGABYTE));
  return region;
}
 
Example #28
Source File: FieldValue.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Size parseSizeString(String sizeString) {
  if (sizeString.length() < 3) {
    throw new IllegalArgumentException("invalid size");
  }

  String valueString = sizeString.substring(0, sizeString.length() - 2);
  String unitSimpleName = sizeString.substring(sizeString.length() - 2);
  return new Size(Double.parseDouble(valueString), convertToUnit(unitSimpleName));
}
 
Example #29
Source File: FieldValue.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Size.Unit convertToUnit(String unitSimpleName) {
  for (Size.Unit unit: Size.Unit.values()) {
    if (unitSimpleName.equals(unit.getSimpleName())) {
      return unit;
    }
  }
  throw new IllegalArgumentException("invalid size");
}
 
Example #30
Source File: StorageClusterStatusResource.java    From hbase with Apache License 2.0 4 votes vote down vote up
@GET
@Produces({MIMETYPE_TEXT, MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF,
  MIMETYPE_PROTOBUF_IETF})
public Response get(final @Context UriInfo uriInfo) {
  if (LOG.isTraceEnabled()) {
    LOG.trace("GET " + uriInfo.getAbsolutePath());
  }
  servlet.getMetrics().incrementRequests(1);
  try {
    ClusterMetrics status = servlet.getAdmin().getClusterMetrics(
      EnumSet.of(Option.LIVE_SERVERS, Option.DEAD_SERVERS));
    StorageClusterStatusModel model = new StorageClusterStatusModel();
    model.setRegions(status.getRegionCount());
    model.setRequests(status.getRequestCount());
    model.setAverageLoad(status.getAverageLoad());
    for (Map.Entry<ServerName, ServerMetrics> entry: status.getLiveServerMetrics().entrySet()) {
      ServerName sn = entry.getKey();
      ServerMetrics load = entry.getValue();
      StorageClusterStatusModel.Node node =
        model.addLiveNode(
          sn.getHostname() + ":" +
          Integer.toString(sn.getPort()),
          sn.getStartcode(), (int) load.getUsedHeapSize().get(Size.Unit.MEGABYTE),
          (int) load.getMaxHeapSize().get(Size.Unit.MEGABYTE));
      node.setRequests(load.getRequestCount());
      for (RegionMetrics region: load.getRegionMetrics().values()) {
        node.addRegion(region.getRegionName(), region.getStoreCount(),
          region.getStoreFileCount(),
          (int) region.getStoreFileSize().get(Size.Unit.MEGABYTE),
          (int) region.getMemStoreSize().get(Size.Unit.MEGABYTE),
          (long) region.getStoreFileIndexSize().get(Size.Unit.KILOBYTE),
          region.getReadRequestCount(),
          region.getCpRequestCount(),
          region.getWriteRequestCount(),
          (int) region.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE),
          (int) region.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE),
          (int) region.getBloomFilterSize().get(Size.Unit.KILOBYTE),
          region.getCompactingCellCount(),
          region.getCompactedCellCount());
      }
    }
    for (ServerName name: status.getDeadServerNames()) {
      model.addDeadNode(name.toString());
    }
    ResponseBuilder response = Response.ok(model);
    response.cacheControl(cacheControl);
    servlet.getMetrics().incrementSucessfulGetRequests(1);
    return response.build();
  } catch (IOException e) {
    servlet.getMetrics().incrementFailedGetRequests(1);
    return Response.status(Response.Status.SERVICE_UNAVAILABLE)
      .type(MIMETYPE_TEXT).entity("Unavailable" + CRLF)
      .build();
  }
}