Java Code Examples for org.apache.hadoop.hbase.HBaseIOException

The following examples show how to use org.apache.hadoop.hbase.HBaseIOException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase-operator-tools   Source File: TableReporter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @return First <code>fraction</code> of Table's regions.
 */
private static List<RegionInfo> getRegions(Connection connection, TableName tableName,
    double fraction, String encodedRegionName) throws IOException {
  try (Admin admin = connection.getAdmin()) {
    // Use deprecated API because running against old hbase.
    List<RegionInfo> regions = admin.getRegions(tableName);
    if (regions.size() <= 0) {
      throw new HBaseIOException("No regions found in " + tableName);
    }
    if (encodedRegionName != null) {
      return regions.stream().filter(ri -> ri.getEncodedName().equals(encodedRegionName)).
          collect(Collectors.toCollection(ArrayList::new));
    }
    return regions.subList(0, (int)(regions.size() * fraction)); // Rounds down.
  }
}
 
Example 2
Source Project: phoenix   Source File: IndexLoadBalancer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<ServerName, List<HRegionInfo>> roundRobinAssignment(List<HRegionInfo> regions,
        List<ServerName> servers) throws HBaseIOException {
    List<HRegionInfo> userRegions = new ArrayList<HRegionInfo>();
    List<HRegionInfo> indexRegions = new ArrayList<HRegionInfo>();
    for (HRegionInfo hri : regions) {
        seperateUserAndIndexRegion(hri, userRegions, indexRegions);
    }
    Map<ServerName, List<HRegionInfo>> bulkPlan = null;
    if (!userRegions.isEmpty()) {
        bulkPlan = this.delegator.roundRobinAssignment(userRegions, servers);
        // This should not happen.
        if (null == bulkPlan) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("No region plans selected for user regions in roundRobinAssignment.");
            }
            return null;
        }
        savePlan(bulkPlan);
    }
    bulkPlan = prepareIndexRegionsPlan(indexRegions, bulkPlan, servers);
    return bulkPlan;
}
 
Example 3
Source Project: phoenix   Source File: IndexLoadBalancer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<ServerName, List<HRegionInfo>> retainAssignment(
        Map<HRegionInfo, ServerName> regions, List<ServerName> servers) throws HBaseIOException {
    Map<HRegionInfo, ServerName> userRegionsMap =
            new ConcurrentHashMap<HRegionInfo, ServerName>();
    List<HRegionInfo> indexRegions = new ArrayList<HRegionInfo>();
    for (Entry<HRegionInfo, ServerName> e : regions.entrySet()) {
        seperateUserAndIndexRegion(e, userRegionsMap, indexRegions, servers);
    }
    Map<ServerName, List<HRegionInfo>> bulkPlan = null;
    if (!userRegionsMap.isEmpty()) {
        bulkPlan = this.delegator.retainAssignment(userRegionsMap, servers);
        if (bulkPlan == null) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Empty region plan for user regions.");
            }
            return null;
        }
        savePlan(bulkPlan);
    }
    bulkPlan = prepareIndexRegionsPlan(indexRegions, bulkPlan, servers);
    return bulkPlan;
}
 
Example 4
Source Project: phoenix   Source File: IndexLoadBalancer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ServerName randomAssignment(HRegionInfo regionInfo, List<ServerName> servers)
        throws HBaseIOException {
    if (!isTableColocated(regionInfo.getTable())) {
        return this.delegator.randomAssignment(regionInfo, servers);
    }
    ServerName sn = getServerNameFromMap(regionInfo, servers);
    if (sn == null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("No server found for region " + regionInfo.getRegionNameAsString() + '.');
        }
        sn = getRandomServer(regionInfo, servers);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Destination server for region " + regionInfo.getRegionNameAsString()
                + " is " + ((sn == null) ? "null" : sn.toString()) + '.');
    }
    return sn;
}
 
Example 5
Source Project: hbase   Source File: FavoredNodeAssignmentHelper.java    License: Apache License 2.0 6 votes vote down vote up
public List<ServerName> generateFavoredNodes(RegionInfo hri) throws IOException {

    List<ServerName> favoredNodesForRegion = new ArrayList<>(FAVORED_NODES_NUM);
    ServerName primary = servers.get(random.nextInt(servers.size()));
    favoredNodesForRegion.add(ServerName.valueOf(primary.getAddress(), ServerName.NON_STARTCODE));

    Map<RegionInfo, ServerName> primaryRSMap = new HashMap<>(1);
    primaryRSMap.put(hri, primary);
    Map<RegionInfo, ServerName[]> secondaryAndTertiaryRSMap =
        placeSecondaryAndTertiaryRS(primaryRSMap);
    ServerName[] secondaryAndTertiaryNodes = secondaryAndTertiaryRSMap.get(hri);
    if (secondaryAndTertiaryNodes != null && secondaryAndTertiaryNodes.length == 2) {
      for (ServerName sn : secondaryAndTertiaryNodes) {
        favoredNodesForRegion.add(ServerName.valueOf(sn.getAddress(), ServerName.NON_STARTCODE));
      }
      return favoredNodesForRegion;
    } else {
      throw new HBaseIOException("Unable to generate secondary and tertiary favored nodes.");
    }
  }
 
Example 6
Source Project: hbase   Source File: FavoredNodeAssignmentHelper.java    License: Apache License 2.0 6 votes vote down vote up
public Map<RegionInfo, List<ServerName>> generateFavoredNodesRoundRobin(
    Map<ServerName, List<RegionInfo>> assignmentMap, List<RegionInfo> regions)
    throws IOException {

  if (regions.size() > 0) {
    if (canPlaceFavoredNodes()) {
      Map<RegionInfo, ServerName> primaryRSMap = new HashMap<>();
      // Lets try to have an equal distribution for primary favored node
      placePrimaryRSAsRoundRobin(assignmentMap, primaryRSMap, regions);
      return generateFavoredNodes(primaryRSMap);

    } else {
      throw new HBaseIOException("Not enough nodes to generate favored nodes");
    }
  }
  return null;
}
 
Example 7
Source Project: hbase   Source File: RegionReplicaReplicationEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Long> replicate(TableDescriptor tableDesc, byte[] encodedRegionName,
    List<Entry> entries) {
  if (disabledTableCache.getIfPresent(tableDesc.getTableName()) != null) {
    logSkipped(tableDesc.getTableName(), entries, "cached as a disabled table");
    return CompletableFuture.completedFuture(Long.valueOf(entries.size()));
  }
  byte[] row = CellUtil.cloneRow(entries.get(0).getEdit().getCells().get(0));
  CompletableFuture<RegionLocations> locateFuture = new CompletableFuture<>();
  getRegionLocations(locateFuture, tableDesc, encodedRegionName, row, false);
  CompletableFuture<Long> future = new CompletableFuture<>();
  FutureUtils.addListener(locateFuture, (locs, error) -> {
    if (error != null) {
      future.completeExceptionally(error);
    } else if (locs.getDefaultRegionLocation() == null) {
      future.completeExceptionally(
        new HBaseIOException("No location found for default replica of table=" +
          tableDesc.getTableName() + " row='" + Bytes.toStringBinary(row) + "'"));
    } else {
      replicate(future, locs, tableDesc, encodedRegionName, row, entries);
    }
  });
  return future;
}
 
Example 8
Source Project: hbase   Source File: AssignmentManager.java    License: Apache License 2.0 6 votes vote down vote up
public TransitRegionStateProcedure createMoveRegionProcedure(RegionInfo regionInfo,
    ServerName targetServer) throws HBaseIOException {
  RegionStateNode regionNode = this.regionStates.getRegionStateNode(regionInfo);
  if (regionNode == null) {
    throw new UnknownRegionException("No RegionStateNode found for " +
        regionInfo.getEncodedName() + "(Closed/Deleted?)");
  }
  TransitRegionStateProcedure proc;
  regionNode.lock();
  try {
    preTransitCheck(regionNode, STATES_EXPECTED_ON_UNASSIGN_OR_MOVE);
    regionNode.checkOnline();
    proc = TransitRegionStateProcedure.move(getProcedureEnvironment(), regionInfo, targetServer);
    regionNode.setProcedure(proc);
  } finally {
    regionNode.unlock();
  }
  return proc;
}
 
Example 9
Source Project: hbase   Source File: MasterRegion.java    License: Apache License 2.0 6 votes vote down vote up
private static WAL createWAL(WALFactory walFactory, MasterRegionWALRoller walRoller,
  String serverName, FileSystem walFs, Path walRootDir, RegionInfo regionInfo)
  throws IOException {
  String logName = AbstractFSWALProvider.getWALDirectoryName(serverName);
  Path walDir = new Path(walRootDir, logName);
  LOG.debug("WALDir={}", walDir);
  if (walFs.exists(walDir)) {
    throw new HBaseIOException(
      "Already created wal directory at " + walDir + " for local region " + regionInfo);
  }
  if (!walFs.mkdirs(walDir)) {
    throw new IOException(
      "Can not create wal directory " + walDir + " for local region " + regionInfo);
  }
  WAL wal = walFactory.getWAL(regionInfo);
  walRoller.addWAL(wal);
  return wal;
}
 
Example 10
Source Project: hbase   Source File: RestoreSnapshotProcedure.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Constructor
 * @param env MasterProcedureEnv
 * @param tableDescriptor the table to operate on
 * @param snapshot snapshot to restore from
 * @throws IOException
 */
public RestoreSnapshotProcedure(
    final MasterProcedureEnv env,
    final TableDescriptor tableDescriptor,
    final SnapshotDescription snapshot,
    final boolean restoreAcl)
throws HBaseIOException {
  super(env);
  // This is the new schema we are going to write out as this modification.
  this.modifiedTableDescriptor = tableDescriptor;
  preflightChecks(env, null/*Table can be online when restore is called?*/);
  // Snapshot information
  this.snapshot = snapshot;
  this.restoreAcl = restoreAcl;

  // Monitor
  getMonitorStatus();
}
 
Example 11
Source Project: hbase   Source File: RSGroupBasedLoadBalancer.java    License: Apache License 2.0 6 votes vote down vote up
private void generateGroupMaps(List<RegionInfo> regions, List<ServerName> servers,
  ListMultimap<String, RegionInfo> regionMap, ListMultimap<String, ServerName> serverMap)
  throws HBaseIOException {
  try {
    RSGroupInfo defaultInfo = rsGroupInfoManager.getRSGroup(RSGroupInfo.DEFAULT_GROUP);
    for (RegionInfo region : regions) {
      String groupName =
          RSGroupUtil.getRSGroupInfo(masterServices, rsGroupInfoManager, region.getTable())
              .orElse(defaultInfo).getName();
      regionMap.put(groupName, region);
    }
    for (String groupKey : regionMap.keySet()) {
      RSGroupInfo info = rsGroupInfoManager.getRSGroup(groupKey);
      serverMap.putAll(groupKey, filterOfflineServers(info, servers));
      if (fallbackEnabled && serverMap.get(groupKey).isEmpty()) {
        serverMap.putAll(groupKey, getFallBackCandidates(servers));
      }
      if (serverMap.get(groupKey).isEmpty()) {
        serverMap.put(groupKey, LoadBalancer.BOGUS_SERVER_NAME);
      }
    }
  } catch(IOException e) {
    throw new HBaseIOException("Failed to generate group maps", e);
  }
}
 
Example 12
Source Project: hbase   Source File: TestAccessController.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTruncatePerms() throws Exception {
  try {
    List<UserPermission> existingPerms = AccessControlClient.getUserPermissions(
        systemUserConnection, TEST_TABLE.getNameAsString());
    assertTrue(existingPerms != null);
    assertTrue(existingPerms.size() > 1);
    TEST_UTIL.getAdmin().disableTable(TEST_TABLE);
    TEST_UTIL.truncateTable(TEST_TABLE);
    TEST_UTIL.waitTableAvailable(TEST_TABLE);
    List<UserPermission> perms = AccessControlClient.getUserPermissions(
        systemUserConnection, TEST_TABLE.getNameAsString());
    assertTrue(perms != null);
    assertEquals(existingPerms.size(), perms.size());
  } catch (Throwable e) {
    throw new HBaseIOException(e);
  }
}
 
Example 13
Source Project: hbase   Source File: TestEntityLocks.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test that abort is called when lockHeartbeat fails with IOException.
 */
@Test
public void testHeartbeatException() throws Exception {
  final long workerSleepTime = 100;  // in ms
  Abortable abortable = Mockito.mock(Abortable.class);
  EntityLock lock = admin.namespaceLock("namespace", "description", abortable);
  lock.setTestingSleepTime(workerSleepTime);

  when(master.requestLock(any(), any()))
      .thenReturn(LockResponse.newBuilder().setProcId(procId).build());
  when(master.lockHeartbeat(any(), any()))
      .thenReturn(LOCKED_RESPONSE)
      .thenThrow(new ServiceException("Failed heartbeat!"));

  lock.requestLock();
  lock.await();
  assertTrue(waitLockTimeOut(lock, 100 * workerSleepTime));
  while (lock.getWorker().isAlive()) {
    TimeUnit.MILLISECONDS.sleep(100);
  }
  verify(abortable, times(1)).abort(any(), isA(HBaseIOException.class));
  assertFalse(lock.getWorker().isAlive());
}
 
Example 14
Source Project: hbase   Source File: TestAssignmentManagerUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateUnassignProceduresForMergeFail() throws IOException {
  List<RegionInfo> regions = getPrimaryRegions();
  RegionInfo regionA = regions.get(0);
  RegionInfo regionB = regions.get(1);
  AM.getRegionStates().getRegionStateNode(regionB)
    .setProcedure(TransitRegionStateProcedure.unassign(ENV, regionB));
  try {
    AssignmentManagerUtil.createUnassignProceduresForSplitOrMerge(ENV,
      Stream.of(regionA, regionB), REGION_REPLICATION);
    fail("Should fail as the region is in transition");
  } catch (HBaseIOException e) {
    // expected
  }
  IntStream.range(0, REGION_REPLICATION)
    .mapToObj(i -> RegionReplicaUtil.getRegionInfoForReplica(regionA, i))
    .map(AM.getRegionStates()::getRegionStateNode).forEachOrdered(
      rn -> assertFalse("Should have unset the proc for " + rn, rn.isInTransition()));
}
 
Example 15
Source Project: hbase   Source File: TestFutureUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecordStackTrace() throws IOException {
  CompletableFuture<Void> future = new CompletableFuture<>();
  executor.execute(() -> future.completeExceptionally(new HBaseIOException("Inject error!")));
  try {
    FutureUtils.get(future);
    fail("The future should have been completed exceptionally");
  } catch (HBaseIOException e) {
    assertEquals("Inject error!", e.getMessage());
    StackTraceElement[] elements = e.getStackTrace();
    assertThat(elements[0].toString(), containsString("java.lang.Thread.getStackTrace"));
    assertThat(elements[1].toString(),
      startsWith("org.apache.hadoop.hbase.util.FutureUtils.setStackTrace"));
    assertThat(elements[2].toString(),
      startsWith("org.apache.hadoop.hbase.util.FutureUtils.rethrow"));
    assertThat(elements[3].toString(),
      startsWith("org.apache.hadoop.hbase.util.FutureUtils.get"));
    assertThat(elements[4].toString(),
      startsWith("org.apache.hadoop.hbase.util.TestFutureUtils.testRecordStackTrace"));
    assertTrue(Stream.of(elements)
      .anyMatch(element -> element.toString().contains("--------Future.get--------")));
  } catch (Throwable t) {
    throw new AssertionError("Caught unexpected Throwable", t);
  }
}
 
Example 16
Source Project: phoenix   Source File: ConcurrentMutationsExtendedIT.java    License: Apache License 2.0 6 votes vote down vote up
@Override public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
        MiniBatchOperationInProgress<Mutation> miniBatchOp) throws HBaseIOException {
    try {
        String tableName = c.getEnvironment().getRegionInfo().getTable().getNameAsString();
        if (tableName.startsWith(LOCK_TEST_TABLE_PREFIX)) {
            if (lockedTableRow) {
                throw new DoNotRetryIOException(
                        "Expected lock in preBatchMutate to be exclusive, but it wasn't for row "
                                + Bytes
                                .toStringBinary(miniBatchOp.getOperation(0).getRow()));
            }
            lockedTableRow = true;
            Thread.sleep(ROW_LOCK_WAIT_TIME + 2000);
        }
        Thread.sleep(Math.abs(RAND.nextInt()) % 10);
    } catch (InterruptedException e) {
    } finally {
        lockedTableRow = false;
    }

}
 
Example 17
Source Project: phoenix   Source File: IndexLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize() throws HBaseIOException {
    Class<? extends LoadBalancer> delegatorKlass =
            conf.getClass(INDEX_BALANCER_DELEGATOR, StochasticLoadBalancer.class,
                LoadBalancer.class);
    this.delegator = ReflectionUtils.newInstance(delegatorKlass, conf);
    this.delegator.setClusterStatus(clusterStatus);
    this.delegator.setMasterServices(this.master);
    this.delegator.initialize();
    try {
        populateTablesToColocate(this.master.getTableDescriptors().getAll());
    } catch (IOException e) {
        throw new HBaseIOException(e);
    }
}
 
Example 18
Source Project: phoenix   Source File: IndexLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
private Map<ServerName, List<HRegionInfo>> prepareIndexRegionsPlan(
        List<HRegionInfo> indexRegions, Map<ServerName, List<HRegionInfo>> bulkPlan,
        List<ServerName> servers) throws HBaseIOException {
    if (null != indexRegions && !indexRegions.isEmpty()) {
        if (null == bulkPlan) {
            bulkPlan = new ConcurrentHashMap<ServerName, List<HRegionInfo>>();
        }
        for (HRegionInfo hri : indexRegions) {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Preparing region plan for index region "
                        + hri.getRegionNameAsString() + '.');
            }
            ServerName destServer = getDestServerForIdxRegion(hri);
            List<HRegionInfo> destServerRegions = null;
            if (destServer == null) destServer = this.randomAssignment(hri, servers);
            if (destServer != null) {
                destServerRegions = bulkPlan.get(destServer);
                if (null == destServerRegions) {
                    destServerRegions = new ArrayList<HRegionInfo>();
                    bulkPlan.put(destServer, destServerRegions);
                }
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Server " + destServer + " selected for region "
                            + hri.getRegionNameAsString() + '.');
                }
                destServerRegions.add(hri);
                regionOnline(hri, destServer);
            }
        }
    }
    return bulkPlan;
}
 
Example 19
Source Project: phoenix   Source File: IndexLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
private ServerName getRandomServer(HRegionInfo regionInfo, List<ServerName> servers)
        throws HBaseIOException {
    ServerName sn = null;
    sn = this.delegator.randomAssignment(regionInfo, servers);
    if (sn == null) return null;
    regionOnline(regionInfo, sn);
    return sn;
}
 
Example 20
Source Project: hbase   Source File: FavoredNodeLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void initialize() throws HBaseIOException {
  super.initialize();
  super.setConf(conf);
  this.fnm = services.getFavoredNodesManager();
  this.rackManager = new RackManager(conf);
  super.setConf(conf);
}
 
Example 21
Source Project: hbase   Source File: FavoredNodeLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ServerName randomAssignment(RegionInfo regionInfo, List<ServerName> servers)
    throws HBaseIOException {
  try {
    FavoredNodeAssignmentHelper assignmentHelper =
        new FavoredNodeAssignmentHelper(servers, rackManager);
    assignmentHelper.initialize();
    ServerName primary = super.randomAssignment(regionInfo, servers);
    if (!FavoredNodesManager.isFavoredNodeApplicable(regionInfo)
        || !assignmentHelper.canPlaceFavoredNodes()) {
      return primary;
    }
    List<ServerName> favoredNodes = fnm.getFavoredNodes(regionInfo);
    // check if we have a favored nodes mapping for this region and if so, return
    // a server from the favored nodes list if the passed 'servers' contains this
    // server as well (available servers, that is)
    if (favoredNodes != null) {
      for (ServerName s : favoredNodes) {
        ServerName serverWithLegitStartCode = availableServersContains(servers, s);
        if (serverWithLegitStartCode != null) {
          return serverWithLegitStartCode;
        }
      }
    }
    List<RegionInfo> regions = new ArrayList<>(1);
    regions.add(regionInfo);
    Map<RegionInfo, ServerName> primaryRSMap = new HashMap<>(1);
    primaryRSMap.put(regionInfo, primary);
    assignSecondaryAndTertiaryNodesForRegion(assignmentHelper, regions, primaryRSMap);
    return primary;
  } catch (Exception ex) {
    LOG.warn("Encountered exception while doing favored-nodes (random)assignment " + ex +
        " Falling back to regular assignment");
    return super.randomAssignment(regionInfo, servers);
  }
}
 
Example 22
Source Project: hbase   Source File: RegionProcedureStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * In HBASE-20811, we have introduced a new TRSP to assign/unassign/move regions, and it is
 * incompatible with the old AssignProcedure/UnassignProcedure/MoveRegionProcedure. So we need to
 * make sure that there are none these procedures when upgrading. If there are, the master will
 * quit, you need to go back to the old version to finish these procedures first before upgrading.
 */
private void checkUnsupportedProcedure(Map<Class<?>, List<Procedure<?>>> procsByType)
  throws HBaseIOException {
  // Confirm that we do not have unfinished assign/unassign related procedures. It is not easy to
  // support both the old assign/unassign procedures and the new TransitRegionStateProcedure as
  // there will be conflict in the code for AM. We should finish all these procedures before
  // upgrading.
  for (Class<?> clazz : UNSUPPORTED_PROCEDURES) {
    List<Procedure<?>> procs = procsByType.get(clazz);
    if (procs != null) {
      LOG.error("Unsupported procedure type {} found, please rollback your master to the old" +
        " version to finish them, and then try to upgrade again." +
        " See https://hbase.apache.org/book.html#upgrade2.2 for more details." +
        " The full procedure list: {}", clazz, procs);
      throw new HBaseIOException("Unsupported procedure type " + clazz + " found");
    }
  }
  // A special check for SCP, as we do not support RecoverMetaProcedure any more so we need to
  // make sure that no one will try to schedule it but SCP does have a state which will schedule
  // it.
  if (procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
    .map(p -> (ServerCrashProcedure) p).anyMatch(ServerCrashProcedure::isInRecoverMetaState)) {
    LOG.error("At least one ServerCrashProcedure is going to schedule a RecoverMetaProcedure," +
      " which is not supported any more. Please rollback your master to the old version to" +
      " finish them, and then try to upgrade again." +
      " See https://hbase.apache.org/book.html#upgrade2.2 for more details.");
    throw new HBaseIOException("Unsupported procedure state found for ServerCrashProcedure");
  }
}
 
Example 23
Source Project: hbase   Source File: FavoredStochasticBalancer.java    License: Apache License 2.0 5 votes vote down vote up
private Pair<Map<ServerName, List<RegionInfo>>, List<RegionInfo>>
segregateRegionsAndAssignRegionsWithFavoredNodes(Collection<RegionInfo> regions,
    List<ServerName> onlineServers) throws HBaseIOException {

  // Since we expect FN to be present most of the time, lets create map with same size
  Map<ServerName, List<RegionInfo>> assignmentMapForFavoredNodes =
      new HashMap<>(onlineServers.size());
  List<RegionInfo> regionsWithNoFavoredNodes = new ArrayList<>();

  for (RegionInfo region : regions) {
    List<ServerName> favoredNodes = fnm.getFavoredNodes(region);
    ServerName primaryHost = null;
    ServerName secondaryHost = null;
    ServerName tertiaryHost = null;

    if (favoredNodes != null && !favoredNodes.isEmpty()) {
      for (ServerName s : favoredNodes) {
        ServerName serverWithLegitStartCode = getServerFromFavoredNode(onlineServers, s);
        if (serverWithLegitStartCode != null) {
          FavoredNodesPlan.Position position =
              FavoredNodesPlan.getFavoredServerPosition(favoredNodes, s);
          if (Position.PRIMARY.equals(position)) {
            primaryHost = serverWithLegitStartCode;
          } else if (Position.SECONDARY.equals(position)) {
            secondaryHost = serverWithLegitStartCode;
          } else if (Position.TERTIARY.equals(position)) {
            tertiaryHost = serverWithLegitStartCode;
          }
        }
      }
      assignRegionToAvailableFavoredNode(assignmentMapForFavoredNodes, region, primaryHost,
          secondaryHost, tertiaryHost);
    } else {
      regionsWithNoFavoredNodes.add(region);
    }
  }
  return new Pair<>(assignmentMapForFavoredNodes, regionsWithNoFavoredNodes);
}
 
Example 24
Source Project: hbase   Source File: BaseLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
protected Cluster createCluster(List<ServerName> servers, Collection<RegionInfo> regions)
    throws HBaseIOException {
  boolean hasRegionReplica = false;
  try {
    if (services != null && services.getTableDescriptors() != null) {
      Map<String, TableDescriptor> tds = services.getTableDescriptors().getAll();
      for (RegionInfo regionInfo : regions) {
        TableDescriptor td = tds.get(regionInfo.getTable().getNameWithNamespaceInclAsString());
        if (td != null && td.getRegionReplication() > 1) {
          hasRegionReplica = true;
          break;
        }
      }
    }
  } catch (IOException ioe) {
    throw new HBaseIOException(ioe);
  }

  // Get the snapshot of the current assignments for the regions in question, and then create
  // a cluster out of it. Note that we might have replicas already assigned to some servers
  // earlier. So we want to get the snapshot to see those assignments, but this will only contain
  // replicas of the regions that are passed (for performance).
  Map<ServerName, List<RegionInfo>> clusterState = null;
  if (!hasRegionReplica) {
    clusterState = getRegionAssignmentsByServer(regions);
  } else {
    // for the case where we have region replica it is better we get the entire cluster's snapshot
    clusterState = getRegionAssignmentsByServer(null);
  }

  for (ServerName server : servers) {
    if (!clusterState.containsKey(server)) {
      clusterState.put(server, EMPTY_REGION_LIST);
    }
  }
  return new Cluster(regions, clusterState, null, this.regionFinder,
      rackManager);
}
 
Example 25
Source Project: hbase   Source File: BaseLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Used to assign a single region to a random server.
 */
@Override
public ServerName randomAssignment(RegionInfo regionInfo, List<ServerName> servers)
    throws HBaseIOException {
  metricsBalancer.incrMiscInvocations();
  if (servers != null && servers.contains(masterServerName)) {
    if (shouldBeOnMaster(regionInfo)) {
      return masterServerName;
    }
    if (!LoadBalancer.isTablesOnMaster(getConf())) {
      // Guarantee we do not put any regions on master
      servers = new ArrayList<>(servers);
      servers.remove(masterServerName);
    }
  }

  int numServers = servers == null ? 0 : servers.size();
  if (numServers == 0) {
    LOG.warn("Wanted to retain assignment but no servers to assign to");
    return null;
  }
  if (numServers == 1) { // Only one server, nothing fancy we can do here
    return servers.get(0);
  }
  List<ServerName> idleServers = findIdleServers(servers);
  if (idleServers.size() == 1) {
    return idleServers.get(0);
  }
  final List<ServerName> finalServers = idleServers.isEmpty() ?
          servers : idleServers;
  List<RegionInfo> regions = Lists.newArrayList(regionInfo);
  Cluster cluster = createCluster(finalServers, regions);
  return randomAssignment(cluster, regionInfo, finalServers);
}
 
Example 26
Source Project: hbase   Source File: AssignmentManager.java    License: Apache License 2.0 5 votes vote down vote up
private void preTransitCheck(RegionStateNode regionNode, RegionState.State[] expectedStates)
    throws HBaseIOException {
  if (regionNode.getProcedure() != null) {
    throw new HBaseIOException(regionNode + " is currently in transition");
  }
  if (!regionNode.isInState(expectedStates)) {
    throw new DoNotRetryRegionException("Unexpected state for " + regionNode);
  }
  if (isTableDisabled(regionNode.getTable())) {
    throw new DoNotRetryIOException(regionNode.getTable() + " is disabled for " + regionNode);
  }
}
 
Example 27
Source Project: hbase   Source File: AssignmentManager.java    License: Apache License 2.0 5 votes vote down vote up
private void acceptPlan(final HashMap<RegionInfo, RegionStateNode> regions,
    final Map<ServerName, List<RegionInfo>> plan) throws HBaseIOException {
  final ProcedureEvent<?>[] events = new ProcedureEvent[regions.size()];
  final long st = System.currentTimeMillis();

  if (plan == null) {
    throw new HBaseIOException("unable to compute plans for regions=" + regions.size());
  }

  if (plan.isEmpty()) return;

  int evcount = 0;
  for (Map.Entry<ServerName, List<RegionInfo>> entry: plan.entrySet()) {
    final ServerName server = entry.getKey();
    for (RegionInfo hri: entry.getValue()) {
      final RegionStateNode regionNode = regions.get(hri);
      regionNode.setRegionLocation(server);
      if (server.equals(LoadBalancer.BOGUS_SERVER_NAME) && regionNode.isSystemTable()) {
        assignQueueLock.lock();
        try {
          pendingAssignQueue.add(regionNode);
        } finally {
          assignQueueLock.unlock();
        }
      }else {
        events[evcount++] = regionNode.getProcedureEvent();
      }
    }
  }
  ProcedureEvent.wakeEvents(getProcedureScheduler(), events);

  final long et = System.currentTimeMillis();
  if (LOG.isTraceEnabled()) {
    LOG.trace("ASSIGN ACCEPT " + events.length + " -> " +
        StringUtils.humanTimeDiff(et - st));
  }
}
 
Example 28
Source Project: hbase   Source File: HMaster.java    License: Apache License 2.0 5 votes vote down vote up
private void createProcedureExecutor() throws IOException {
  MasterProcedureEnv procEnv = new MasterProcedureEnv(this);
  procedureStore =
    new RegionProcedureStore(this, masterRegion, new MasterProcedureEnv.FsUtilsLeaseRecovery(this));
  procedureStore.registerListener(new ProcedureStoreListener() {

    @Override
    public void abortProcess() {
      abort("The Procedure Store lost the lease", null);
    }
  });
  MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler();
  procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler);
  configurationManager.registerObserver(procEnv);

  int cpus = Runtime.getRuntime().availableProcessors();
  final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(
    (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS));
  final boolean abortOnCorruption =
    conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION,
      MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION);
  procedureStore.start(numThreads);
  // Just initialize it but do not start the workers, we will start the workers later by calling
  // startProcedureExecutor. See the javadoc for finishActiveMasterInitialization for more
  // details.
  procedureExecutor.init(numThreads, abortOnCorruption);
  if (!procEnv.getRemoteDispatcher().start()) {
    throw new HBaseIOException("Failed start of remote dispatcher");
  }
}
 
Example 29
Source Project: hbase   Source File: HMaster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Remove decommission marker (previously called 'draining') from a region server to allow regions
 * assignments. Load regions onto the server asynchronously if a list of regions is given
 * @param server Region server to remove decommission marker from.
 */
public void recommissionRegionServer(final ServerName server,
    final List<byte[]> encodedRegionNames) throws IOException {
  // Remove the server from decommissioned (draining) server list.
  String parentZnode = getZooKeeper().getZNodePaths().drainingZNode;
  String node = ZNodePaths.joinZNode(parentZnode, server.getServerName());
  try {
    ZKUtil.deleteNodeFailSilent(getZooKeeper(), node);
  } catch (KeeperException ke) {
    throw new HBaseIOException(
      this.zooKeeper.prefix("Unable to recommission '" + server.getServerName() + "'."), ke);
  }
  this.serverManager.removeServerFromDrainList(server);

  // Load the regions onto the server if we are given a list of regions.
  if (encodedRegionNames == null || encodedRegionNames.isEmpty()) {
    return;
  }
  if (!this.serverManager.isServerOnline(server)) {
    return;
  }
  for (byte[] encodedRegionName : encodedRegionNames) {
    RegionState regionState =
      assignmentManager.getRegionStates().getRegionState(Bytes.toString(encodedRegionName));
    if (regionState == null) {
      LOG.warn("Unknown region " + Bytes.toStringBinary(encodedRegionName));
      continue;
    }
    RegionInfo hri = regionState.getRegion();
    if (server.equals(regionState.getServerName())) {
      LOG.info("Skipping move of region " + hri.getRegionNameAsString() +
        " because region already assigned to the same server " + server + ".");
      continue;
    }
    RegionPlan rp = new RegionPlan(hri, regionState.getServerName(), server);
    this.assignmentManager.moveAsync(rp);
  }
}
 
Example 30
Source Project: hbase   Source File: AbstractStateMachineTableProcedure.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check that cluster is up and master is running. Check table is modifiable.
 * If <code>enabled</code>, check table is enabled else check it is disabled.
 * Call in Procedure constructor so can pass any exception to caller.
 * @param enabled If true, check table is enabled and throw exception if not. If false, do the
 *                inverse. If null, do no table checks.
 */
protected void preflightChecks(MasterProcedureEnv env, Boolean enabled) throws HBaseIOException {
  MasterServices master = env.getMasterServices();
  if (!master.isClusterUp()) {
    throw new HBaseIOException("Cluster not up!");
  }
  if (master.isStopping() || master.isStopped()) {
    throw new HBaseIOException("Master stopping=" + master.isStopping() +
        ", stopped=" + master.isStopped());
  }
  if (enabled == null) {
    // Don't do any table checks.
    return;
  }
  try {
    // Checks table exists and is modifiable.
    checkTableModifiable(env);
    TableName tn = getTableName();
    TableStateManager tsm = master.getTableStateManager();
    TableState ts = tsm.getTableState(tn);
    if (enabled) {
      if (!ts.isEnabledOrEnabling()) {
        throw new TableNotEnabledException(tn);
      }
    } else {
      if (!ts.isDisabledOrDisabling()) {
        throw new TableNotDisabledException(tn);
      }
    }
  } catch (IOException ioe) {
    if (ioe instanceof HBaseIOException) {
      throw (HBaseIOException)ioe;
    }
    throw new HBaseIOException(ioe);
  }
}