org.apache.hadoop.hbase.RegionMetrics Java Examples
The following examples show how to use
org.apache.hadoop.hbase.RegionMetrics.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HbaseSessions.java From hugegraph with Apache License 2.0 | 6 votes |
public long storeSize(String table) throws IOException { long total = 0; try (Admin admin = this.hbase.getAdmin()) { for (ServerName rs : admin.getRegionServers()) { // NOTE: we can use getLoad() before hbase 2.0 //ServerLoad load = admin.getClusterStatus().getLoad(rs); //total += load.getStorefileSizeMB() * Bytes.MB; //total += load.getMemStoreSizeMB() * Bytes.MB; TableName tableName = TableName.valueOf(this.namespace, table); for (RegionMetrics m : admin.getRegionMetrics(rs, tableName)) { total += m.getStoreFileSize().get(Size.Unit.BYTE); total += m.getMemStoreSize().get(Size.Unit.BYTE); } } } return total; }
Example #2
Source File: TestUtils.java From hbase with Apache License 2.0 | 6 votes |
private static RegionMetrics createRegionMetrics(String regionName, long readRequestCount, long filteredReadRequestCount, long writeRequestCount, Size storeFileSize, Size uncompressedStoreFileSize, int storeFileCount, Size memStoreSize, float locality, long compactedCellCount, long compactingCellCount, String lastMajorCompactionTime) { FastDateFormat df = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss"); try { return RegionMetricsBuilder.newBuilder(Bytes.toBytes(regionName)) .setReadRequestCount(readRequestCount) .setFilteredReadRequestCount(filteredReadRequestCount) .setWriteRequestCount(writeRequestCount).setStoreFileSize(storeFileSize) .setUncompressedStoreFileSize(uncompressedStoreFileSize).setStoreFileCount(storeFileCount) .setMemStoreSize(memStoreSize).setDataLocality(locality) .setCompactedCellCount(compactedCellCount).setCompactingCellCount(compactingCellCount) .setLastMajorCompactionTimestamp(df.parse(lastMajorCompactionTime).getTime()).build(); } catch (ParseException e) { throw new IllegalArgumentException(e); } }
Example #3
Source File: TestRegionServerReadRequestMetrics.java From hbase with Apache License 2.0 | 6 votes |
private static long getReadRequest(ServerMetrics serverMetrics, RegionMetrics regionMetrics, Metric metric) { switch (metric) { case REGION_READ: return regionMetrics.getReadRequestCount(); case SERVER_READ: return serverMetrics.getRegionMetrics().get(regionMetrics.getRegionName()) .getReadRequestCount(); case FILTERED_REGION_READ: return regionMetrics.getFilteredReadRequestCount(); case FILTERED_SERVER_READ: return serverMetrics.getRegionMetrics().get(regionMetrics.getRegionName()) .getFilteredReadRequestCount(); default: throw new IllegalStateException(); } }
Example #4
Source File: TestStochasticLoadBalancer.java From hbase with Apache License 2.0 | 6 votes |
private ServerMetrics mockServerMetricsWithCpRequests(ServerName server, List<RegionInfo> regionsOnServer, long cpRequestCount) { ServerMetrics serverMetrics = mock(ServerMetrics.class); Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(RegionInfo info : regionsOnServer){ RegionMetrics rl = mock(RegionMetrics.class); when(rl.getReadRequestCount()).thenReturn(0L); when(rl.getCpRequestCount()).thenReturn(cpRequestCount); when(rl.getWriteRequestCount()).thenReturn(0L); when(rl.getMemStoreSize()).thenReturn(Size.ZERO); when(rl.getStoreFileSize()).thenReturn(Size.ZERO); regionLoadMap.put(info.getRegionName(), rl); } when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap); return serverMetrics; }
Example #5
Source File: TestClassLoading.java From hbase with Apache License 2.0 | 6 votes |
/** * return the subset of all regionservers * (actually returns set of ServerLoads) * which host some region in a given table. * used by assertAllRegionServers() below to * test reporting of loaded coprocessors. * @param tableName : given table. * @return subset of all servers. */ Map<ServerName, ServerMetrics> serversForTable(String tableName) { Map<ServerName, ServerMetrics> serverLoadHashMap = new HashMap<>(); for(Map.Entry<ServerName, ServerMetrics> server: TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager(). getOnlineServers().entrySet()) { for(Map.Entry<byte[], RegionMetrics> region: server.getValue().getRegionMetrics().entrySet()) { if (region.getValue().getNameAsString().equals(tableName)) { // this server hosts a region of tableName: add this server.. serverLoadHashMap.put(server.getKey(),server.getValue()); // .. and skip the rest of the regions that it hosts. break; } } } return serverLoadHashMap; }
Example #6
Source File: RegionsRecoveryChore.java From hbase with Apache License 2.0 | 6 votes |
private Map<TableName, List<byte[]>> getTableToRegionsByRefCount( final Map<ServerName, ServerMetrics> serverMetricsMap) { final Map<TableName, List<byte[]>> tableToReopenRegionsMap = new HashMap<>(); for (ServerMetrics serverMetrics : serverMetricsMap.values()) { Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); for (RegionMetrics regionMetrics : regionMetricsMap.values()) { // For each region, each compacted store file can have different ref counts // We need to find maximum of all such ref counts and if that max count of compacted // store files is beyond a threshold value, we should reopen the region. // Here, we take max ref count of all compacted store files and not the cumulative // count of all compacted store files final int maxCompactedStoreFileRefCount = regionMetrics .getMaxCompactedStoreFileRefCount(); if (maxCompactedStoreFileRefCount > storeFileRefCountThreshold) { final byte[] regionName = regionMetrics.getRegionName(); prepareTableToReopenRegionsMap(tableToReopenRegionsMap, regionName, maxCompactedStoreFileRefCount); } } } return tableToReopenRegionsMap; }
Example #7
Source File: StochasticLoadBalancer.java From hbase with Apache License 2.0 | 6 votes |
/** * Store the current region loads. */ private synchronized void updateRegionLoad() { // We create a new hashmap so that regions that are no longer there are removed. // However we temporarily need the old loads so we can use them to keep the rolling average. Map<String, Deque<BalancerRegionLoad>> oldLoads = loads; loads = new HashMap<>(); clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -> { sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -> { String regionNameAsString = RegionInfo.getRegionNameAsString(regionName); Deque<BalancerRegionLoad> rLoads = oldLoads.get(regionNameAsString); if (rLoads == null) { rLoads = new ArrayDeque<>(numRegionLoadsToRemember + 1); } else if (rLoads.size() >= numRegionLoadsToRemember) { rLoads.remove(); } rLoads.add(new BalancerRegionLoad(rm)); loads.put(regionNameAsString, rLoads); }); }); for(CostFromRegionLoadFunction cost : regionLoadFunctions) { cost.setLoads(loads); } }
Example #8
Source File: TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java From hbase with Apache License 2.0 | 6 votes |
private ServerMetrics mockServerMetricsWithReadRequests(ServerName server, List<RegionInfo> regionsOnServer, long readRequestCount) { ServerMetrics serverMetrics = mock(ServerMetrics.class); Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR); for(RegionInfo info : regionsOnServer){ RegionMetrics rl = mock(RegionMetrics.class); when(rl.getReadRequestCount()).thenReturn(readRequestCount); when(rl.getCpRequestCount()).thenReturn(0L); when(rl.getWriteRequestCount()).thenReturn(0L); when(rl.getMemStoreSize()).thenReturn(Size.ZERO); when(rl.getStoreFileSize()).thenReturn(Size.ZERO); regionLoadMap.put(info.getRegionName(), rl); } when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap); return serverMetrics; }
Example #9
Source File: TestAsyncClusterAdminApi.java From hbase with Apache License 2.0 | 6 votes |
private void compareRegionLoads(Collection<RegionMetrics> regionLoadCluster, Collection<RegionMetrics> regionLoads) { assertEquals("No of regionLoads from clusterStatus and regionloads from RS doesn't match", regionLoadCluster.size(), regionLoads.size()); for (RegionMetrics loadCluster : regionLoadCluster) { boolean matched = false; for (RegionMetrics load : regionLoads) { if (Bytes.equals(loadCluster.getRegionName(), load.getRegionName())) { matched = true; continue; } } assertTrue("The contents of region load from cluster and server should match", matched); } }
Example #10
Source File: RawAsyncHBaseAdmin.java From hbase with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<List<RegionMetrics>> getRegionMetrics(ServerName serverName, TableName tableName) { Preconditions.checkNotNull(tableName, "tableName is null. If you don't specify a tableName, use getRegionLoads() instead"); return getRegionMetrics(RequestConverter.buildGetRegionLoadRequest(tableName), serverName); }
Example #11
Source File: TestRegionsRecoveryChore.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testRegionReopensWithLessThreshold() throws Exception { regionNo = 0; ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4); final Map<ServerName, ServerMetrics> serverMetricsMap = clusterMetrics.getLiveServerMetrics(); LOG.debug("All Region Names with refCount...."); for (ServerMetrics serverMetrics : serverMetricsMap.values()) { Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); for (RegionMetrics regionMetrics : regionMetricsMap.values()) { LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " + regionMetrics.getStoreRefCount()); } } Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics); Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager); for (byte[] regionName : REGION_NAME_LIST) { Mockito.when(assignmentManager.getRegionInfo(regionName)) .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName)); } Stoppable stoppable = new StoppableImplementation(); Configuration configuration = getCustomConf(); configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400); regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster); regionsRecoveryChore.chore(); // Verify that we need to reopen regions of only 1 table Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(), Mockito.anyLong(), Mockito.anyLong()); Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics(); // Verify that we need to reopen only 1 region with refCount > 400 Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager(); Mockito.verify(assignmentManager, Mockito.times(1)) .getRegionInfo(Mockito.any()); }
Example #12
Source File: TestRegionsRecoveryChore.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testRegionReopensWithoutStoreRefConfig() throws Exception { regionNo = 0; ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10); final Map<ServerName, ServerMetrics> serverMetricsMap = clusterMetrics.getLiveServerMetrics(); LOG.debug("All Region Names with refCount...."); for (ServerMetrics serverMetrics : serverMetricsMap.values()) { Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); for (RegionMetrics regionMetrics : regionMetricsMap.values()) { LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " + regionMetrics.getStoreRefCount()); } } Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics); Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager); for (byte[] regionName : REGION_NAME_LIST) { Mockito.when(assignmentManager.getRegionInfo(regionName)) .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName)); } Stoppable stoppable = new StoppableImplementation(); Configuration configuration = getCustomConf(); configuration.unset("hbase.regions.recovery.store.file.ref.count"); regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster); regionsRecoveryChore.chore(); // Verify that by default the feature is turned off so no regions // should be reopened Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(), Mockito.anyLong(), Mockito.anyLong()); // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM) Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager(); Mockito.verify(assignmentManager, Mockito.times(0)) .getRegionInfo(Mockito.any()); }
Example #13
Source File: TestRegionServerReadRequestMetrics.java From hbase with Apache License 2.0 | 5 votes |
private void testReadRequests(byte[] regionName, int expectedReadRequests) throws Exception { for (ServerName serverName : serverNames) { ServerMetrics serverMetrics = admin.getClusterMetrics( EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().get(serverName); Map<byte[], RegionMetrics> regionMetrics = serverMetrics.getRegionMetrics(); RegionMetrics regionMetric = regionMetrics.get(regionName); if (regionMetric != null) { LOG.debug("server read request is " + serverMetrics.getRegionMetrics().get(regionName).getReadRequestCount() + ", region read request is " + regionMetric.getReadRequestCount()); assertEquals(3, serverMetrics.getRegionMetrics().get(regionName).getReadRequestCount()); assertEquals(3, regionMetric.getReadRequestCount()); } } }
Example #14
Source File: RegionSizeCalculator.java From hbase with Apache License 2.0 | 5 votes |
private void init(RegionLocator regionLocator, Admin admin) throws IOException { if (!enabled(admin.getConfiguration())) { LOG.info("Region size calculation disabled."); return; } if (regionLocator.getName().isSystemTable()) { LOG.info("Region size calculation disabled for system tables."); return; } LOG.info("Calculating region sizes for table \"" + regionLocator.getName() + "\"."); // Get the servers which host regions of the table Set<ServerName> tableServers = getRegionServersOfTable(regionLocator); for (ServerName tableServerName : tableServers) { for (RegionMetrics regionLoad : admin.getRegionMetrics( tableServerName,regionLocator.getName())) { byte[] regionId = regionLoad.getRegionName(); long regionSizeBytes = ((long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE)) * MEGABYTE; sizeMap.put(regionId, regionSizeBytes); if (LOG.isDebugEnabled()) { LOG.debug("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes); } } } LOG.debug("Region sizes calculated"); }
Example #15
Source File: TestRegionSizeCalculator.java From hbase with Apache License 2.0 | 5 votes |
/** * Creates mock of region with given name and size. * * @param fileSizeMb number of megabytes occupied by region in file store in megabytes * */ private RegionMetrics mockRegion(String regionName, int fileSizeMb) { RegionMetrics region = Mockito.mock(RegionMetrics.class); when(region.getRegionName()).thenReturn(Bytes.toBytes(regionName)); when(region.getNameAsString()).thenReturn(regionName); when(region.getStoreFileSize()).thenReturn(new Size(fileSizeMb, Size.Unit.MEGABYTE)); return region; }
Example #16
Source File: HbaseMetrics.java From hugegraph with Apache License 2.0 | 5 votes |
private static Map<String, Object> formatMetrics( List<RegionMetrics> regions) { Map<String, Object> metrics = InsertionOrderUtil.newMap(); for (RegionMetrics region : regions) { metrics.put(region.getNameAsString(), formatMetrics(region)); } return metrics; }
Example #17
Source File: RawAsyncHBaseAdmin.java From hbase with Apache License 2.0 | 5 votes |
private CompletableFuture<List<RegionMetrics>> getRegionMetrics(GetRegionLoadRequest request, ServerName serverName) { return this.<List<RegionMetrics>> newAdminCaller() .action((controller, stub) -> this .<GetRegionLoadRequest, GetRegionLoadResponse, List<RegionMetrics>> adminCall(controller, stub, request, (s, c, req, done) -> s.getRegionLoad(controller, req, done), RegionMetricsBuilder::toRegionMetrics)) .serverName(serverName).call(); }
Example #18
Source File: RegionModeStrategy.java From hbase with Apache License 2.0 | 5 votes |
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics, List<RecordFilter> pushDownFilters) { List<Record> ret = new ArrayList<>(); for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) { long lastReportTimestamp = sm.getLastReportTimestamp(); for (RegionMetrics rm : sm.getRegionMetrics().values()) { ret.add(createRecord(sm, rm, lastReportTimestamp)); } } return ret; }
Example #19
Source File: TestUtils.java From hbase with Apache License 2.0 | 5 votes |
private static ServerMetrics createServerMetrics(ServerName serverName, long reportTimestamp, Size usedHeapSize, Size maxHeapSize, long requestCountPerSecond, List<RegionMetrics> regionMetricsList, List<UserMetrics> userMetricsList) { return ServerMetricsBuilder.newBuilder(serverName) .setReportTimestamp(reportTimestamp) .setUsedHeapSize(usedHeapSize) .setMaxHeapSize(maxHeapSize) .setRequestCountPerSecond(requestCountPerSecond) .setRegionMetrics(regionMetricsList) .setUserMetrics(userMetricsList).build(); }
Example #20
Source File: HServerLoad.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@Override public Set<PartitionLoad> getPartitionLoads(){ Map<byte[], RegionMetrics> regionsLoad=load.getRegionMetrics(); Set<PartitionLoad> loads = new HashSet<>(regionsLoad.size(),0.9f); for(Map.Entry<byte[], RegionMetrics> regionLoad:regionsLoad.entrySet()){ String name = Bytes.toString(regionLoad.getKey()); RegionMetrics rm = regionLoad.getValue(); PartitionLoad pl = new HPartitionLoad(name, (long) rm.getStoreFileSize().get(Size.Unit.BYTE), (long) rm.getMemStoreSize().get(Size.Unit.BYTE)); loads.add(pl); } return loads; }
Example #21
Source File: TestSimpleRegionNormalizer.java From hbase with Apache License 2.0 | 5 votes |
@SuppressWarnings("MockitoCast") private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes, List<RegionInfo> regionInfoList) { masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS); // for simplicity all regions are assumed to be on one server; doesn't matter to us ServerName sn = ServerName.valueOf("localhost", 0, 0L); when(masterServices.getAssignmentManager().getRegionStates() .getRegionsOfTable(any())).thenReturn(regionInfoList); when(masterServices.getAssignmentManager().getRegionStates() .getRegionServerOfRegion(any())).thenReturn(sn); when(masterServices.getAssignmentManager().getRegionStates() .getRegionState(any(RegionInfo.class))).thenReturn( RegionState.createForTesting(null, RegionState.State.OPEN)); for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) { RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class); when(regionLoad.getRegionName()).thenReturn(region.getKey()); when(regionLoad.getStoreFileSize()) .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE)); // this is possibly broken with jdk9, unclear if false positive or not // suppress it for now, fix it when we get to running tests on 9 // see: http://errorprone.info/bugpattern/MockitoCast when((Object) masterServices.getServerManager().getLoad(sn) .getRegionMetrics().get(region.getKey())).thenReturn(regionLoad); } when(masterServices.isSplitOrMergeEnabled(any())).thenReturn(true); normalizer = new SimpleRegionNormalizer(); normalizer.setConf(conf); normalizer.setMasterServices(masterServices); }
Example #22
Source File: HbaseMetrics.java From hugegraph with Apache License 2.0 | 5 votes |
private static Map<String, Object> formatMetrics( ServerMetrics serverMetrics, List<RegionMetrics> regions) { Map<String, Object> metrics = InsertionOrderUtil.newMap(); metrics.put("max_heap_size", serverMetrics.getMaxHeapSize().get(Size.Unit.MEGABYTE)); metrics.put("used_heap_size", serverMetrics.getUsedHeapSize().get(Size.Unit.MEGABYTE)); metrics.put("heap_size_unit", "MB"); metrics.put("request_count", serverMetrics.getRequestCount()); metrics.put("request_count_per_second", serverMetrics.getRequestCountPerSecond()); metrics.put("regions", formatMetrics(regions)); return metrics; }
Example #23
Source File: HbaseMetrics.java From hugegraph with Apache License 2.0 | 5 votes |
private static Map<String, Object> formatMetrics(RegionMetrics region) { Map<String, Object> metrics = InsertionOrderUtil.newMap(); metrics.put("mem_store_size", region.getMemStoreSize().get(Size.Unit.MEGABYTE)); metrics.put("file_store_size", region.getStoreFileSize().get(Size.Unit.MEGABYTE)); metrics.put("store_size_unit", "MB"); return metrics; }
Example #24
Source File: BalancerRegionLoad.java From hbase with Apache License 2.0 | 5 votes |
BalancerRegionLoad(RegionMetrics regionMetrics) { readRequestsCount = regionMetrics.getReadRequestCount(); cpRequestsCount = regionMetrics.getCpRequestCount(); writeRequestsCount = regionMetrics.getWriteRequestCount(); memStoreSizeMB = (int) regionMetrics.getMemStoreSize().get(Size.Unit.MEGABYTE); storefileSizeMB = (int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE); }
Example #25
Source File: SimpleRegionNormalizer.java From hbase with Apache License 2.0 | 5 votes |
/** * @return size of region in MB and if region is not found than -1 */ private long getRegionSizeMB(RegionInfo hri) { ServerName sn = masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(hri); RegionMetrics regionLoad = masterServices.getServerManager().getLoad(sn).getRegionMetrics().get(hri.getRegionName()); if (regionLoad == null) { LOG.debug("{} was not found in RegionsLoad", hri.getRegionNameAsString()); return -1; } return (long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE); }
Example #26
Source File: ServerManager.java From hbase with Apache License 2.0 | 5 votes |
/** * Updates last flushed sequence Ids for the regions on server sn * @param sn * @param hsl */ private void updateLastFlushedSequenceIds(ServerName sn, ServerMetrics hsl) { for (Entry<byte[], RegionMetrics> entry : hsl.getRegionMetrics().entrySet()) { byte[] encodedRegionName = Bytes.toBytes(RegionInfo.encodeRegionName(entry.getKey())); Long existingValue = flushedSequenceIdByRegion.get(encodedRegionName); long l = entry.getValue().getCompletedSequenceId(); // Don't let smaller sequence ids override greater sequence ids. if (LOG.isTraceEnabled()) { LOG.trace(Bytes.toString(encodedRegionName) + ", existingValue=" + existingValue + ", completeSequenceId=" + l); } if (existingValue == null || (l != HConstants.NO_SEQNUM && l > existingValue)) { flushedSequenceIdByRegion.put(encodedRegionName, l); } else if (l != HConstants.NO_SEQNUM && l < existingValue) { LOG.warn("RegionServer " + sn + " indicates a last flushed sequence id (" + l + ") that is less than the previous last flushed sequence id (" + existingValue + ") for region " + Bytes.toString(entry.getKey()) + " Ignoring."); } ConcurrentNavigableMap<byte[], Long> storeFlushedSequenceId = computeIfAbsent(storeFlushedSequenceIdsByRegion, encodedRegionName, () -> new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR)); for (Entry<byte[], Long> storeSeqId : entry.getValue().getStoreSequenceId().entrySet()) { byte[] family = storeSeqId.getKey(); existingValue = storeFlushedSequenceId.get(family); l = storeSeqId.getValue(); if (LOG.isTraceEnabled()) { LOG.trace(Bytes.toString(encodedRegionName) + ", family=" + Bytes.toString(family) + ", existingValue=" + existingValue + ", completeSequenceId=" + l); } // Don't let smaller sequence ids override greater sequence ids. if (existingValue == null || (l != HConstants.NO_SEQNUM && l > existingValue.longValue())) { storeFlushedSequenceId.put(family, l); } } } }
Example #27
Source File: TestAsyncClusterAdminApi.java From hbase with Apache License 2.0 | 5 votes |
private void checkRegionsAndRegionLoads(Collection<RegionInfo> regions, Collection<RegionMetrics> regionLoads) { assertEquals("No of regions and regionloads doesn't match", regions.size(), regionLoads.size()); Map<byte[], RegionMetrics> regionLoadMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (RegionMetrics regionLoad : regionLoads) { regionLoadMap.put(regionLoad.getRegionName(), regionLoad); } for (RegionInfo info : regions) { assertTrue("Region not in regionLoadMap region:" + info.getRegionNameAsString() + " regionMap: " + regionLoadMap, regionLoadMap.containsKey(info.getRegionName())); } }
Example #28
Source File: TestSimpleRegionNormalizerOnCluster.java From hbase with Apache License 2.0 | 5 votes |
private static double getRegionSizeMB(final MasterServices masterServices, final RegionInfo regionInfo) { final ServerName sn = masterServices.getAssignmentManager() .getRegionStates() .getRegionServerOfRegion(regionInfo); final RegionMetrics regionLoad = masterServices.getServerManager() .getLoad(sn) .getRegionMetrics() .get(regionInfo.getRegionName()); if (regionLoad == null) { LOG.debug("{} was not found in RegionsLoad", regionInfo.getRegionNameAsString()); return -1; } return regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE); }
Example #29
Source File: TestRegionsRecoveryChore.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testRegionReopensWithStoreRefConfig() throws Exception { regionNo = 0; ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4); final Map<ServerName, ServerMetrics> serverMetricsMap = clusterMetrics.getLiveServerMetrics(); LOG.debug("All Region Names with refCount...."); for (ServerMetrics serverMetrics : serverMetricsMap.values()) { Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics(); for (RegionMetrics regionMetrics : regionMetricsMap.values()) { LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " + regionMetrics.getStoreRefCount()); } } Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics); Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager); for (byte[] regionName : REGION_NAME_LIST) { Mockito.when(assignmentManager.getRegionInfo(regionName)) .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName)); } Stoppable stoppable = new StoppableImplementation(); Configuration configuration = getCustomConf(); configuration.setInt("hbase.regions.recovery.store.file.ref.count", 300); regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster); regionsRecoveryChore.chore(); // Verify that we need to reopen regions of 2 tables Mockito.verify(hMaster, Mockito.times(2)).reopenRegions(Mockito.any(), Mockito.anyList(), Mockito.anyLong(), Mockito.anyLong()); Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics(); // Verify that we need to reopen total 3 regions that have refCount > 300 Mockito.verify(hMaster, Mockito.times(3)).getAssignmentManager(); Mockito.verify(assignmentManager, Mockito.times(3)) .getRegionInfo(Mockito.any()); }
Example #30
Source File: TestFromClientSide3.java From hbase with Apache License 2.0 | 5 votes |
private int getStoreFileCount(Admin admin, ServerName serverName, RegionInfo region) throws IOException { for (RegionMetrics metrics : admin.getRegionMetrics(serverName, region.getTable())) { if (Bytes.equals(region.getRegionName(), metrics.getRegionName())) { return metrics.getStoreFileCount(); } } return 0; }