org.apache.hadoop.hbase.replication.ReplicationException Java Examples

The following examples show how to use org.apache.hadoop.hbase.replication.ReplicationException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransitPeerSyncReplicationStateProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
protected void setPeerNewSyncReplicationState(MasterProcedureEnv env)
    throws ReplicationException {
  if (toState.equals(SyncReplicationState.STANDBY) ||
    (fromState.equals(SyncReplicationState.STANDBY) && serial) && enabled) {
    // Disable the peer if we are going to transit to STANDBY state, as we need to remove
    // all the pending replication files. If we do not disable the peer and delete the wal
    // queues on zk directly, RS will get NoNode exception when updating the wal position
    // and crash.
    // Disable the peer if we are going to transit from STANDBY to DOWNGRADE_ACTIVE, and the
    // replication is serial, as we need to update the lastPushedSequence id after we reopen all
    // the regions, and for performance reason here we will update in batch, without using CAS, if
    // we are still replicating at RS side, we may accidentally update the last pushed sequence id
    // to a less value and cause the replication to be stuck.
    env.getReplicationPeerManager().disablePeer(peerId);
  }
  env.getReplicationPeerManager().setPeerNewSyncReplicationState(peerId, toState);
}
 
Example #2
Source File: ReplicationSourceManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Refresh replication source will terminate the old source first, then the source thread will be
 * interrupted. Need to handle it instead of abort the region server.
 */
private void interruptOrAbortWhenFail(ReplicationQueueOperation op) {
  try {
    op.exec();
  } catch (ReplicationException e) {
    if (e.getCause() != null && e.getCause() instanceof KeeperException.SystemErrorException
        && e.getCause().getCause() != null && e.getCause()
        .getCause() instanceof InterruptedException) {
      // ReplicationRuntimeException(a RuntimeException) is thrown out here. The reason is
      // that thread is interrupted deep down in the stack, it should pass the following
      // processing logic and propagate to the most top layer which can handle this exception
      // properly. In this specific case, the top layer is ReplicationSourceShipper#run().
      throw new ReplicationRuntimeException(
        "Thread is interrupted, the replication source may be terminated",
        e.getCause().getCause());
    }
    server.abort("Failed to operate on replication queue", e);
  }
}
 
Example #3
Source File: RawAsyncHBaseAdmin.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Void> appendReplicationPeerTableCFs(String id,
    Map<TableName, List<String>> tableCfs) {
  if (tableCfs == null) {
    return failedFuture(new ReplicationException("tableCfs is null"));
  }

  CompletableFuture<Void> future = new CompletableFuture<Void>();
  addListener(getReplicationPeerConfig(id), (peerConfig, error) -> {
    if (!completeExceptionally(future, error)) {
      ReplicationPeerConfig newPeerConfig =
        ReplicationPeerConfigUtil.appendTableCFsToReplicationPeerConfig(tableCfs, peerConfig);
      addListener(updateReplicationPeerConfig(id, newPeerConfig), (result, err) -> {
        if (!completeExceptionally(future, error)) {
          future.complete(result);
        }
      });
    }
  });
  return future;
}
 
Example #4
Source File: AbstractPeerProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final void setLastPushedSequenceId(MasterProcedureEnv env,
    ReplicationPeerConfig peerConfig) throws IOException, ReplicationException {
  Map<String, Long> lastSeqIds = new HashMap<String, Long>();
  for (TableDescriptor td : env.getMasterServices().getTableDescriptors().getAll().values()) {
    if (!td.hasGlobalReplicationScope()) {
      continue;
    }
    TableName tn = td.getTableName();
    if (!peerConfig.needToReplicate(tn)) {
      continue;
    }
    setLastPushedSequenceIdForTable(env, tn, lastSeqIds);
  }
  if (!lastSeqIds.isEmpty()) {
    env.getReplicationPeerManager().getQueueStorage().setLastSequenceIds(peerId, lastSeqIds);
  }
}
 
Example #5
Source File: ReplicationSourceManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void adoptAbandonedQueues() {
  List<ServerName> currentReplicators = null;
  try {
    currentReplicators = queueStorage.getListOfReplicators();
  } catch (ReplicationException e) {
    server.abort("Failed to get all replicators", e);
    return;
  }
  if (currentReplicators == null || currentReplicators.isEmpty()) {
    return;
  }
  List<ServerName> otherRegionServers = replicationTracker.getListOfRegionServers().stream()
      .map(ServerName::valueOf).collect(Collectors.toList());
  LOG.info(
    "Current list of replicators: " + currentReplicators + " other RSs: " + otherRegionServers);

  // Look if there's anything to process after a restart
  for (ServerName rs : currentReplicators) {
    if (!otherRegionServers.contains(rs)) {
      transferQueues(rs);
    }
  }
}
 
Example #6
Source File: ReplicationHFileCleaner.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public boolean isFileDeletable(FileStatus fStat) {
  Set<String> hfileRefsFromQueue;
  // all members of this class are null if replication is disabled,
  // so do not stop from deleting the file
  if (getConf() == null) {
    return true;
  }

  try {
    hfileRefsFromQueue = rqs.getAllHFileRefs();
  } catch (ReplicationException e) {
    LOG.warn("Failed to read hfile references from zookeeper, skipping checking deletable "
        + "file for " + fStat.getPath());
    return false;
  }
  return !hfileRefsFromQueue.contains(fStat.getPath().getName());
}
 
Example #7
Source File: ReplicationPeerManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void checkQueuesDeleted(String peerId)
    throws ReplicationException, DoNotRetryIOException {
  for (ServerName replicator : queueStorage.getListOfReplicators()) {
    List<String> queueIds = queueStorage.getAllQueues(replicator);
    for (String queueId : queueIds) {
      ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId);
      if (queueInfo.getPeerId().equals(peerId)) {
        throw new DoNotRetryIOException("undeleted queue for peerId: " + peerId +
          ", replicator: " + replicator + ", queueId: " + queueId);
      }
    }
  }
  if (queueStorage.getAllPeersFromHFileRefsQueue().contains(peerId)) {
    throw new DoNotRetryIOException("Undeleted queue for peer " + peerId + " in hfile-refs");
  }
}
 
Example #8
Source File: UpdatePeerConfigProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected void updatePeerStorage(MasterProcedureEnv env) throws ReplicationException {
  env.getReplicationPeerManager().updatePeerConfig(peerId, peerConfig);
  // if we need to jump to the special states for serial peers, then we need to disable the peer
  // first if it is not disabled yet.
  if (enabled && nextStateAfterRefresh() != super.nextStateAfterRefresh()) {
    env.getReplicationPeerManager().disablePeer(peerId);
  }
}
 
Example #9
Source File: ReplicationSourceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void throwIOExceptionWhenFail(ReplicationQueueOperation op) throws IOException {
  try {
    op.exec();
  } catch (ReplicationException e) {
    throw new IOException(e);
  }
}
 
Example #10
Source File: UpdatePeerConfigProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected void postPeerModification(MasterProcedureEnv env)
    throws IOException, ReplicationException {
  if (oldPeerConfig.isSerial() && !peerConfig.isSerial()) {
    env.getReplicationPeerManager().removeAllLastPushedSeqIds(peerId);
  }
  LOG.info("Successfully updated peer config of {} to {}", peerId, peerConfig);
  MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.postUpdateReplicationPeerConfig(peerId, peerConfig);
  }
}
 
Example #11
Source File: MasterRpcServices.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public ListReplicationPeersResponse listReplicationPeers(RpcController controller,
    ListReplicationPeersRequest request) throws ServiceException {
  ListReplicationPeersResponse.Builder response = ListReplicationPeersResponse.newBuilder();
  try {
    List<ReplicationPeerDescription> peers = master
        .listReplicationPeers(request.hasRegex() ? request.getRegex() : null);
    for (ReplicationPeerDescription peer : peers) {
      response.addPeerDesc(ReplicationPeerConfigUtil.toProtoReplicationPeerDescription(peer));
    }
  } catch (ReplicationException | IOException e) {
    throw new ServiceException(e);
  }
  return response.build();
}
 
Example #12
Source File: RawAsyncHBaseAdmin.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<Void> removeReplicationPeerTableCFs(String id,
    Map<TableName, List<String>> tableCfs) {
  if (tableCfs == null) {
    return failedFuture(new ReplicationException("tableCfs is null"));
  }

  CompletableFuture<Void> future = new CompletableFuture<Void>();
  addListener(getReplicationPeerConfig(id), (peerConfig, error) -> {
    if (!completeExceptionally(future, error)) {
      ReplicationPeerConfig newPeerConfig = null;
      try {
        newPeerConfig = ReplicationPeerConfigUtil
          .removeTableCFsFromReplicationPeerConfig(tableCfs, peerConfig, id);
      } catch (ReplicationException e) {
        future.completeExceptionally(e);
        return;
      }
      addListener(updateReplicationPeerConfig(id, newPeerConfig), (result, err) -> {
        if (!completeExceptionally(future, error)) {
          future.complete(result);
        }
      });
    }
  });
  return future;
}
 
Example #13
Source File: HMaster.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public long addReplicationPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled)
    throws ReplicationException, IOException {
  LOG.info(getClientIdAuditPrefix() + " creating replication peer, id=" + peerId + ", config=" +
    peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED"));
  return executePeerProcedure(new AddPeerProcedure(peerId, peerConfig, enabled));
}
 
Example #14
Source File: MasterRpcServices.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public GetReplicationPeerConfigResponse getReplicationPeerConfig(RpcController controller,
    GetReplicationPeerConfigRequest request) throws ServiceException {
  GetReplicationPeerConfigResponse.Builder response = GetReplicationPeerConfigResponse
      .newBuilder();
  try {
    String peerId = request.getPeerId();
    ReplicationPeerConfig peerConfig = master.getReplicationPeerConfig(peerId);
    response.setPeerId(peerId);
    response.setPeerConfig(ReplicationPeerConfigUtil.convert(peerConfig));
  } catch (ReplicationException | IOException e) {
    throw new ServiceException(e);
  }
  return response.build();
}
 
Example #15
Source File: ReplicationSourceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * <ol>
 * <li>Add peer to replicationPeers</li>
 * <li>Add the normal source and related replication queue</li>
 * <li>Add HFile Refs</li>
 * </ol>
 * @param peerId the id of replication peer
 */
public void addPeer(String peerId) throws IOException {
  boolean added = false;
  try {
    added = this.replicationPeers.addPeer(peerId);
  } catch (ReplicationException e) {
    throw new IOException(e);
  }
  if (added) {
    addSource(peerId);
    if (replicationForBulkLoadDataEnabled) {
      throwIOExceptionWhenFail(() -> this.queueStorage.addPeerToHFileRefs(peerId));
    }
  }
}
 
Example #16
Source File: HMaster.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public ReplicationPeerConfig getReplicationPeerConfig(String peerId)
    throws ReplicationException, IOException {
  if (cpHost != null) {
    cpHost.preGetReplicationPeerConfig(peerId);
  }
  LOG.info(getClientIdAuditPrefix() + " get replication peer config, id=" + peerId);
  ReplicationPeerConfig peerConfig = this.replicationPeerManager.getPeerConfig(peerId)
      .orElseThrow(() -> new ReplicationPeerNotFoundException(peerId));
  if (cpHost != null) {
    cpHost.postGetReplicationPeerConfig(peerId);
  }
  return peerConfig;
}
 
Example #17
Source File: ReplicationSourceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void abortWhenFail(ReplicationQueueOperation op) {
  try {
    op.exec();
  } catch (ReplicationException e) {
    server.abort("Failed to operate on replication queue", e);
  }
}
 
Example #18
Source File: ReplicationChecker.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void fixUnDeletedQueues() throws ReplicationException {
  for (Map.Entry<ServerName, List<String>> replicatorAndQueueIds : undeletedQueueIds.entrySet()) {
    ServerName replicator = replicatorAndQueueIds.getKey();
    for (String queueId : replicatorAndQueueIds.getValue()) {
      queueStorage.removeQueue(replicator, queueId);
    }
    queueStorage.removeReplicatorIfQueueIsEmpty(replicator);
  }
  for (String peerId : undeletedHFileRefsPeerIds) {
    queueStorage.removePeerFromHFileRefs(peerId);
  }
}
 
Example #19
Source File: UpdatePeerConfigProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void addToList(List<String> encodedRegionNames, String encodedRegionName,
    ReplicationQueueStorage queueStorage) throws ReplicationException {
  encodedRegionNames.add(encodedRegionName);
  if (encodedRegionNames.size() >= UPDATE_LAST_SEQ_ID_BATCH_SIZE) {
    queueStorage.removeLastSequenceIds(peerId, encodedRegionNames);
    encodedRegionNames.clear();
  }
}
 
Example #20
Source File: ReplicationPeerConfigUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static ReplicationPeerConfig appendExcludeTableCFsToReplicationPeerConfig(
    Map<TableName, List<String>> excludeTableCfs, ReplicationPeerConfig peerConfig)
    throws ReplicationException {
  if (excludeTableCfs == null) {
    throw new ReplicationException("exclude tableCfs is null");
  }
  ReplicationPeerConfigBuilder builder = ReplicationPeerConfig.newBuilder(peerConfig);
  Map<TableName, List<String>> preExcludeTableCfs = peerConfig.getExcludeTableCFsMap();
  if (preExcludeTableCfs == null) {
    builder.setExcludeTableCFsMap(excludeTableCfs);
  } else {
    builder.setExcludeTableCFsMap(mergeTableCFs(preExcludeTableCfs, excludeTableCfs));
  }
  return builder.build();
}
 
Example #21
Source File: TestReplicationBarrierCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteBarriers() throws IOException, ReplicationException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo region = RegionInfoBuilder.newBuilder(tableName).build();
  addBarrier(region, 10, 20, 30, 40, 50, 60);
  // two peers
  ReplicationQueueStorage queueStorage = create(-1L, 2L, 15L, 25L, 20L, 25L, 65L, 55L, 70L, 70L);
  List<String> peerIds = Lists.newArrayList("1", "2");

  @SuppressWarnings("unchecked")
  ReplicationPeerManager peerManager =
    create(queueStorage, peerIds, peerIds, peerIds, peerIds, peerIds);
  ReplicationBarrierCleaner cleaner = create(peerManager);

  // beyond the first barrier, no deletion
  cleaner.chore();
  assertArrayEquals(new long[] { 10, 20, 30, 40, 50, 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));

  // in the first range, still no deletion
  cleaner.chore();
  assertArrayEquals(new long[] { 10, 20, 30, 40, 50, 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));

  // in the second range, 10 is deleted
  cleaner.chore();
  assertArrayEquals(new long[] { 20, 30, 40, 50, 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));

  // between 50 and 60, so the barriers before 50 will be deleted
  cleaner.chore();
  assertArrayEquals(new long[] { 50, 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));

  // in the last open range, 50 is deleted
  cleaner.chore();
  assertArrayEquals(new long[] { 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));
}
 
Example #22
Source File: TestReplicationBarrierCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
private ReplicationQueueStorage create(Long lastPushedSeqId, Long... lastPushedSeqIds)
    throws ReplicationException {
  ReplicationQueueStorage queueStorage = mock(ReplicationQueueStorage.class);
  if (lastPushedSeqIds.length == 0) {
    when(queueStorage.getLastSequenceId(anyString(), anyString())).thenReturn(lastPushedSeqId);
  } else {
    when(queueStorage.getLastSequenceId(anyString(), anyString())).thenReturn(lastPushedSeqId,
      lastPushedSeqIds);
  }
  return queueStorage;
}
 
Example #23
Source File: MasterRpcServices.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public TransitReplicationPeerSyncReplicationStateResponse
    transitReplicationPeerSyncReplicationState(RpcController controller,
        TransitReplicationPeerSyncReplicationStateRequest request) throws ServiceException {
  try {
    long procId = master.transitReplicationPeerSyncReplicationState(request.getPeerId(),
      ReplicationPeerConfigUtil.toSyncReplicationState(request.getSyncReplicationState()));
    return TransitReplicationPeerSyncReplicationStateResponse.newBuilder().setProcId(procId)
        .build();
  } catch (ReplicationException | IOException e) {
    throw new ServiceException(e);
  }
}
 
Example #24
Source File: TestModifyPeerProcedureRetryBackoff.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void tryFail() throws ReplicationException {
  synchronized (TestModifyPeerProcedureRetryBackoff.class) {
    if (FAIL) {
      throw new ReplicationException("Inject error");
    }
    FAIL = true;
  }
}
 
Example #25
Source File: ReplicationPeerManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void removePeer(String peerId) throws ReplicationException {
  if (!peers.containsKey(peerId)) {
    // this should be a retry, just return
    return;
  }
  peerStorage.removePeer(peerId);
  peers.remove(peerId);
}
 
Example #26
Source File: MasterRpcServices.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public DisableReplicationPeerResponse disableReplicationPeer(RpcController controller,
    DisableReplicationPeerRequest request) throws ServiceException {
  try {
    long procId = master.disableReplicationPeer(request.getPeerId());
    return DisableReplicationPeerResponse.newBuilder().setProcId(procId).build();
  } catch (ReplicationException | IOException e) {
    throw new ServiceException(e);
  }
}
 
Example #27
Source File: RemovePeerProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override  protected void postPeerModification(MasterProcedureEnv env)
    throws IOException, ReplicationException {
  if (peerConfig.isSyncReplication()) {
    removeRemoteWALs(env);
  }
  env.getReplicationPeerManager().removeAllQueuesAndHFileRefs(peerId);
  if (peerConfig.isSerial()) {
    env.getReplicationPeerManager().removeAllLastPushedSeqIds(peerId);
  }
  LOG.info("Successfully removed peer {}", peerId);
  MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.postRemoveReplicationPeer(peerId);
  }
}
 
Example #28
Source File: TestReplicationBarrierCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteRowForDeletedRegion() throws IOException, ReplicationException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo region = RegionInfoBuilder.newBuilder(tableName).build();
  addBarrier(region, 40, 50, 60);
  fillCatalogFamily(region);

  String peerId = "1";
  ReplicationQueueStorage queueStorage = create(59L);
  @SuppressWarnings("unchecked")
  ReplicationPeerManager peerManager = create(queueStorage, Lists.newArrayList(peerId));
  ReplicationBarrierCleaner cleaner = create(peerManager);

  // we have something in catalog family, so only delete 40
  cleaner.chore();
  assertArrayEquals(new long[] { 50, 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));
  verify(queueStorage, never()).removeLastSequenceIds(anyString(), anyList());

  // No catalog family, then we should remove the whole row
  clearCatalogFamily(region);
  cleaner.chore();
  try (Table table = UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
    assertFalse(table
      .exists(new Get(region.getRegionName()).addFamily(HConstants.REPLICATION_BARRIER_FAMILY)));
  }
  verify(queueStorage, times(1)).removeLastSequenceIds(peerId,
    Arrays.asList(region.getEncodedName()));
}
 
Example #29
Source File: ReplicationPeerManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void transitPeerSyncReplicationState(String peerId, SyncReplicationState newState)
    throws ReplicationException {
  if (peerStorage.getPeerNewSyncReplicationState(peerId) != SyncReplicationState.NONE) {
    // Only transit if this is not a retry
    peerStorage.transitPeerSyncReplicationState(peerId);
  }
  ReplicationPeerDescription desc = peers.get(peerId);
  if (desc.getSyncReplicationState() != newState) {
    // Only recreate the desc if this is not a retry
    peers.put(peerId,
      new ReplicationPeerDescription(peerId, desc.isEnabled(), desc.getPeerConfig(), newState));
  }
}
 
Example #30
Source File: ReplicationPeerManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void removeAllQueues(String peerId) throws ReplicationException {
  // Here we need two passes to address the problem of claimQueue. Maybe a claimQueue is still
  // on-going when the refresh peer config procedure is done, if a RS which has already been
  // scanned claims the queue of a RS which has not been scanned yet, we will miss that queue in
  // the scan here, and if the RS who has claimed the queue crashed before creating recovered
  // source, then the queue will leave there until the another RS detects the crash and helps
  // removing the queue.
  // A two pass scan can solve the problem. Anyway, the queue will not disappear during the
  // claiming, it will either under the old RS or under the new RS, and a queue can only be
  // claimed once after the refresh peer procedure done(as the next claim queue will just delete
  // it), so we can make sure that a two pass scan will finally find the queue and remove it,
  // unless it has already been removed by others.
  ReplicationUtils.removeAllQueues(queueStorage, peerId);
  ReplicationUtils.removeAllQueues(queueStorage, peerId);
}