org.apache.hadoop.ha.HAServiceProtocol.HAServiceState Java Examples

The following examples show how to use org.apache.hadoop.ha.HAServiceProtocol.HAServiceState. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMHATestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected void startRMs(MockRM rm1, Configuration confForRM1, MockRM rm2,
    Configuration confForRM2) throws IOException {
  rm1.init(confForRM1);
  rm1.start();
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm2.init(confForRM2);
  rm2.start();
  Assert.assertTrue(rm2.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm1.adminService.transitionToActive(requestInfo);
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.ACTIVE);
}
 
Example #2
Source File: TestFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailbackToFaultyServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc1.proxy).transitionToActive(anyReqInfo());
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());

  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 
Example #3
Source File: TestFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailureToFenceOnFailbackFailsTheFailback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new IOException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());
  AlwaysFailFencer.fenceCalled = 0;

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We did not fence svc1 because it cooperated and we didn't force it, 
  // we failed to failover so we fenced svc2, we failed to fence svc2
  // so we did not failback to svc1, ie it's still standby.
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc2, AlwaysFailFencer.fencedSvc);
}
 
Example #4
Source File: RMHATestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected void startRMs(MockRM rm1, Configuration confForRM1, MockRM rm2,
    Configuration confForRM2) throws IOException {
  rm1.init(confForRM1);
  rm1.start();
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm2.init(confForRM2);
  rm2.start();
  Assert.assertTrue(rm2.getRMContext().getHAServiceState()
      == HAServiceState.STANDBY);

  rm1.adminService.transitionToActive(requestInfo);
  Assert.assertTrue(rm1.getRMContext().getHAServiceState()
      == HAServiceState.ACTIVE);
}
 
Example #5
Source File: ResourceManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
synchronized void transitionToStandby(boolean initialize)
    throws Exception {
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.STANDBY) {
    LOG.info("Already in standby state");
    return;
  }

  LOG.info("Transitioning to standby state");
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.ACTIVE) {
    stopActiveServices();
    reinitialize(initialize);
  }
  rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
  LOG.info("Transitioned to standby state");
}
 
Example #6
Source File: TestFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverWithoutPermission() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc1.proxy).getServiceStatus();
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc2.proxy).getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover when access is denied");
  } catch (FailoverFailedException ffe) {
    assertTrue(ffe.getCause().getMessage().contains("Access denied"));
  }
}
 
Example #7
Source File: TestBackupNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
BackupNode startBackupNode(Configuration conf,
                           StartupOption startupOpt,
                           int idx) throws IOException {
  Configuration c = new HdfsConfiguration(conf);
  String dirs = getBackupNodeDir(startupOpt, idx);
  c.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dirs);
  c.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      "${" + DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + "}");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
      "127.0.0.1:0");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
          "127.0.0.1:0");

  BackupNode bn = (BackupNode)NameNode.createNameNode(
      new String[]{startupOpt.getName()}, c);
  assertTrue(bn.getRole() + " must be in SafeMode.", bn.isInSafeMode());
  assertTrue(bn.getRole() + " must be in StandbyState",
             bn.getNamesystem().getHAState()
               .equalsIgnoreCase(HAServiceState.STANDBY.name()));
  return bn;
}
 
Example #8
Source File: TestBPOfferService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Set up a mock NN with the bare minimum for a DN to register to it.
 */
private DatanodeProtocolClientSideTranslatorPB setupNNMock(int nnIdx)
    throws Exception {
  DatanodeProtocolClientSideTranslatorPB mock =
      Mockito.mock(DatanodeProtocolClientSideTranslatorPB.class);
  Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0))
      .when(mock).versionRequest();
  
  Mockito.doReturn(DFSTestUtil.getLocalDatanodeRegistration())
    .when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class));
  
  Mockito.doAnswer(new HeartbeatAnswer(nnIdx))
    .when(mock).sendHeartbeat(
        Mockito.any(DatanodeRegistration.class),
        Mockito.any(StorageReport[].class),
        Mockito.anyLong(),
        Mockito.anyLong(),
        Mockito.anyInt(),
        Mockito.anyInt(),
        Mockito.anyInt(),
        Mockito.any(VolumeFailureSummary.class));
  mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0);
  return mock;
}
 
Example #9
Source File: TestFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFencingFailureDuringFailover() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysFailFencer.class.getName());

  AlwaysFailFencer.fenceCalled = 0;
  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over even though fencing requested and failed");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // If fencing was requested and it failed we don't try to make
  // svc2 active anyway, and we don't failback to svc1.
  assertEquals(1, AlwaysFailFencer.fenceCalled);
  assertSame(svc1, AlwaysFailFencer.fencedSvc);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 
Example #10
Source File: DummyHAService.java    From big-c with Apache License 2.0 6 votes vote down vote up
DummyHAService(HAServiceState state, InetSocketAddress address,
    boolean testWithProtoBufRPC) {
  this.state = state;
  this.testWithProtoBufRPC = testWithProtoBufRPC;
  if (testWithProtoBufRPC) {
    this.address = startAndGetRPCServerAddress(address);
  } else {
    this.address = address;
  }
  Configuration conf = new Configuration();
  this.proxy = makeMock(conf, HA_HM_RPC_TIMEOUT_DEFAULT);
  try {
    conf.set(DUMMY_FENCE_KEY, DummyFencer.class.getName());
    this.fencer = Mockito.spy(
        NodeFencer.create(conf, DUMMY_FENCE_KEY));
  } catch (BadFencingConfigurationException e) {
    throw new RuntimeException(e);
  }
  synchronized (instances) {
    instances.add(this);
    this.index = instances.size();
  }
}
 
Example #11
Source File: TestFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverToUnreadyService() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
      .getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to a service that's not ready");
  } catch (FailoverFailedException ffe) {
    // Expected
    if (!ffe.getMessage().contains("injected not ready")) {
      throw ffe;
    }
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);

  // Forcing it means we ignore readyToBecomeActive
  doFailover(svc1, svc2, false, true);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 
Example #12
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverFromActiveToActive() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.ACTIVE, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to an already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 
Example #13
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverToUnreadyService() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doReturn(STATE_NOT_READY).when(svc2.proxy)
      .getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover to a service that's not ready");
  } catch (FailoverFailedException ffe) {
    // Expected
    if (!ffe.getMessage().contains("injected not ready")) {
      throw ffe;
    }
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);

  // Forcing it means we ignore readyToBecomeActive
  doFailover(svc1, svc2, false, true);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 
Example #14
Source File: TestBackupNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
BackupNode startBackupNode(Configuration conf,
                           StartupOption startupOpt,
                           int idx) throws IOException {
  Configuration c = new HdfsConfiguration(conf);
  String dirs = getBackupNodeDir(startupOpt, idx);
  c.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, dirs);
  c.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY,
      "${" + DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY + "}");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_ADDRESS_KEY,
      "127.0.0.1:0");
  c.set(DFSConfigKeys.DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
          "127.0.0.1:0");

  BackupNode bn = (BackupNode)NameNode.createNameNode(
      new String[]{startupOpt.getName()}, c);
  assertTrue(bn.getRole() + " must be in SafeMode.", bn.isInSafeMode());
  assertTrue(bn.getRole() + " must be in StandbyState",
             bn.getNamesystem().getHAState()
               .equalsIgnoreCase(HAServiceState.STANDBY.name()));
  return bn;
}
 
Example #15
Source File: TestHealthMonitor.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setupHM() throws InterruptedException, IOException {
  Configuration conf = new Configuration();
  conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 1);
  conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
  
  svc = new DummyHAService(HAServiceState.ACTIVE,
      new InetSocketAddress("0.0.0.0", 0), true);
  hm = new HealthMonitor(conf, svc) {
    @Override
    protected HAServiceProtocol createProxy() throws IOException {
      createProxyCount.incrementAndGet();
      if (throwOOMEOnCreate) {
        throw new OutOfMemoryError("oome");
      }
      return super.createProxy();
    }
  };
  LOG.info("Starting health monitor");
  hm.start();
  
  LOG.info("Waiting for HEALTHY signal");    
  waitForState(hm, HealthMonitor.State.SERVICE_HEALTHY);
}
 
Example #16
Source File: MiniZKFCCluster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Set up two services and their failover controllers. svc1 is started
 * first, so that it enters ACTIVE state, and then svc2 is started,
 * which enters STANDBY
 */
public void start() throws Exception {
  // Format the base dir, should succeed
  thrs = new DummyZKFCThread[2];
  thrs[0] = new DummyZKFCThread(ctx, svcs[0]);
  assertEquals(0, thrs[0].zkfc.run(new String[]{"-formatZK"}));
  ctx.addThread(thrs[0]);
  thrs[0].start();
  
  LOG.info("Waiting for svc0 to enter active state");
  waitForHAState(0, HAServiceState.ACTIVE);
  
  LOG.info("Adding svc1");
  thrs[1] = new DummyZKFCThread(ctx, svcs[1]);
  thrs[1].start();
  waitForHAState(1, HAServiceState.STANDBY);
}
 
Example #17
Source File: MiniZKFCCluster.java    From big-c with Apache License 2.0 6 votes vote down vote up
public MiniZKFCCluster(Configuration conf, ZooKeeperServer zks) {
  this.conf = conf;
  // Fast check interval so tests run faster
  conf.setInt(CommonConfigurationKeys.HA_HM_CHECK_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
  conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
  svcs = new DummyHAService[2];
  svcs[0] = new DummyHAService(HAServiceState.INITIALIZING,
      new InetSocketAddress("svc1", 1234));
  svcs[0].setSharedResource(sharedResource);
  svcs[1] = new DummyHAService(HAServiceState.INITIALIZING,
      new InetSocketAddress("svc2", 1234));
  svcs[1].setSharedResource(sharedResource);
  
  this.ctx = new TestContext();
  this.zks = zks;
}
 
Example #18
Source File: TestFailoverController.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testWeDontFailbackIfActiveWasFenced() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover and did not failback because we fenced
  // svc1 (we forced it), therefore svc1 and svc2 should be standby.
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 
Example #19
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testWeDontFailbackIfActiveWasFenced() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, true, false);
    fail("Failed over to service that won't transition to active");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // We failed to failover and did not failback because we fenced
  // svc1 (we forced it), therefore svc1 and svc2 should be standby.
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 
Example #20
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverToFaultyServiceFailsbackOK() throws Exception {
  DummyHAService svc1 = spy(new DummyHAService(HAServiceState.ACTIVE, svc1Addr));
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new ServiceFailedException("Failed!"))
      .when(svc2.proxy).transitionToActive(anyReqInfo());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to already active service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  // svc1 went standby then back to active
  verify(svc1.proxy).transitionToStandby(anyReqInfo());
  verify(svc1.proxy).transitionToActive(anyReqInfo());
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 
Example #21
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverToNonExistantServiceFails() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = spy(new DummyHAService(null, svc2Addr));
  Mockito.doThrow(new IOException("Failed to connect"))
    .when(svc2).getProxy(Mockito.<Configuration>any(),
        Mockito.anyInt());
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failed over to a non-existant standby");
  } catch (FailoverFailedException ffe) {
    // Expected
  }

  assertEquals(HAServiceState.ACTIVE, svc1.state);
}
 
Example #22
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverToUnhealthyServiceFailsAndFailsback() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new HealthCheckFailedException("Failed!"))
      .when(svc2.proxy).monitorHealth();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Failover to unhealthy service");
  } catch (FailoverFailedException ffe) {
    // Expected
  }
  assertEquals(HAServiceState.ACTIVE, svc1.state);
  assertEquals(HAServiceState.STANDBY, svc2.state);
}
 
Example #23
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverWithoutPermission() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.ACTIVE, svc1Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc1.proxy).getServiceStatus();
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  Mockito.doThrow(new AccessControlException("Access denied"))
      .when(svc2.proxy).getServiceStatus();
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  try {
    doFailover(svc1, svc2, false, false);
    fail("Can't failover when access is denied");
  } catch (FailoverFailedException ffe) {
    assertTrue(ffe.getCause().getMessage().contains("Access denied"));
  }
}
 
Example #24
Source File: TestBPOfferService.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * This test case test the {@link BPOfferService#trySendErrorReport} method
 * such that if call to standby namenode times out then that should not 
 * affect the active namenode heartbeat processing since this function 
 * are in writeLock.
 * @throws Exception
 */
@Test
public void testTrySendErrorReportWhenStandbyNNTimesOut() throws Exception {
  BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
  bpos.start();
  try {
    waitForInitialization(bpos);
    // Should start with neither NN as active.
    assertNull(bpos.getActiveNN());
    // Have NN1 claim active at txid 1
    mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
    bpos.triggerHeartbeatForTests();
    // Now mockNN1 is acting like active namenode and mockNN2 as Standby
    assertSame(mockNN1, bpos.getActiveNN());
    Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(0))
        .when(mockNN1).errorReport(Mockito.any(DatanodeRegistration.class),
        Mockito.anyInt(), Mockito.anyString());
    Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(1))
        .when(mockNN2).errorReport(Mockito.any(DatanodeRegistration.class),
        Mockito.anyInt(), Mockito.anyString());
    String errorString = "Can't send invalid block " + FAKE_BLOCK;
    bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
    bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
    Thread.sleep(10000);
    long difference = secondCallTime - firstCallTime;
    assertTrue("Active namenode trySendErrorReport processing "
        + "should be independent of standby namenode trySendErrorReport"
        + " processing ", difference < 5000);
  } finally {
    bpos.stop();
  }
}
 
Example #25
Source File: TestFailoverController.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testFailoverFromStandbyToStandby() throws Exception {
  DummyHAService svc1 = new DummyHAService(HAServiceState.STANDBY, svc1Addr);
  DummyHAService svc2 = new DummyHAService(HAServiceState.STANDBY, svc2Addr);
  svc1.fencer = svc2.fencer = setupFencer(AlwaysSucceedFencer.class.getName());

  doFailover(svc1, svc2, false, false);
  assertEquals(HAServiceState.STANDBY, svc1.state);
  assertEquals(HAServiceState.ACTIVE, svc2.state);
}
 
Example #26
Source File: HAAdmin.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Checks whether other target node is active or not
 * @param targetNodeToActivate
 * @return true if other target node is active or some other exception 
 * occurred and forceActive was set otherwise false
 * @throws IOException
 */
private boolean isOtherTargetNodeActive(String targetNodeToActivate, boolean forceActive)
    throws IOException  {
  Collection<String> targetIds = getTargetIds(targetNodeToActivate);
  targetIds.remove(targetNodeToActivate);
  for(String targetId : targetIds) {
    HAServiceTarget target = resolveTarget(targetId);
    if (!checkManualStateManagementOK(target)) {
      return true;
    }
    try {
      HAServiceProtocol proto = target.getProxy(getConf(), 5000);
      if(proto.getServiceStatus().getState() == HAServiceState.ACTIVE) {
        errOut.println("transitionToActive: Node " +  targetId +" is already active");
        printUsage(errOut, "-transitionToActive");
        return true;
      }
    } catch (Exception e) {
      //If forceActive switch is false then return true
      if(!forceActive) {
        errOut.println("Unexpected error occurred  " + e.getMessage());
        printUsage(errOut, "-transitionToActive");
        return true; 
      }
    }
  }
  return false;
}
 
Example #27
Source File: TestZKFailoverController.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Test that, when the health monitor indicates bad health status,
 * failover is triggered. Also ensures that graceful active->standby
 * transition is used when possible, falling back to fencing when
 * the graceful approach fails.
 */
@Test(timeout=15000)
public void testAutoFailoverOnBadState() throws Exception {
  try {
    cluster.start();
    DummyHAService svc0 = cluster.getService(0);
    LOG.info("Faking svc0 to change the state, should failover to svc1");
    svc0.state = HAServiceState.STANDBY;
    
    // Should fail back to svc0 at this point
    cluster.waitForHAState(1, HAServiceState.ACTIVE);
  } finally {
    cluster.stop();
  }
}
 
Example #28
Source File: ZKFailoverController.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void doCedeActive(int millisToCede) 
    throws AccessControlException, ServiceFailedException, IOException {
  int timeout = FailoverController.getGracefulFenceTimeout(conf);

  // Lock elector to maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      if (millisToCede <= 0) {
        delayJoiningUntilNanotime = 0;
        recheckElectability();
        return;
      }

      LOG.info("Requested by " + UserGroupInformation.getCurrentUser() +
          " at " + Server.getRemoteAddress() + " to cede active role.");
      boolean needFence = false;
      try {
        localTarget.getProxy(conf, timeout).transitionToStandby(createReqInfo());
        LOG.info("Successfully ensured local node is in standby mode");
      } catch (IOException ioe) {
        LOG.warn("Unable to transition local node to standby: " +
            ioe.getLocalizedMessage());
        LOG.warn("Quitting election but indicating that fencing is " +
            "necessary");
        needFence = true;
      }
      delayJoiningUntilNanotime = System.nanoTime() +
          TimeUnit.MILLISECONDS.toNanos(millisToCede);
      elector.quitElection(needFence);
      serviceState = HAServiceState.INITIALIZING;
    }
  }
  recheckElectability();
}
 
Example #29
Source File: ResourceManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStop() throws Exception {
  if (webApp != null) {
    webApp.stop();
  }
  if (fetcher != null) {
    fetcher.stop();
  }
  if (configurationProvider != null) {
    configurationProvider.close();
  }
  super.serviceStop();
  transitionToStandby(false);
  rmContext.setHAServiceState(HAServiceState.STOPPING);
}
 
Example #30
Source File: ZKFailoverController.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void doCedeActive(int millisToCede) 
    throws AccessControlException, ServiceFailedException, IOException {
  int timeout = FailoverController.getGracefulFenceTimeout(conf);

  // Lock elector to maintain lock ordering of elector -> ZKFC
  synchronized (elector) {
    synchronized (this) {
      if (millisToCede <= 0) {
        delayJoiningUntilNanotime = 0;
        recheckElectability();
        return;
      }

      LOG.info("Requested by " + UserGroupInformation.getCurrentUser() +
          " at " + Server.getRemoteAddress() + " to cede active role.");
      boolean needFence = false;
      try {
        localTarget.getProxy(conf, timeout).transitionToStandby(createReqInfo());
        LOG.info("Successfully ensured local node is in standby mode");
      } catch (IOException ioe) {
        LOG.warn("Unable to transition local node to standby: " +
            ioe.getLocalizedMessage());
        LOG.warn("Quitting election but indicating that fencing is " +
            "necessary");
        needFence = true;
      }
      delayJoiningUntilNanotime = System.nanoTime() +
          TimeUnit.MILLISECONDS.toNanos(millisToCede);
      elector.quitElection(needFence);
      serviceState = HAServiceState.INITIALIZING;
    }
  }
  recheckElectability();
}