Java Code Examples for org.apache.hadoop.ha.HAServiceProtocol

The following examples show how to use org.apache.hadoop.ha.HAServiceProtocol. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestMiniYARNClusterForHA.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException, InterruptedException {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");

  cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(),
      2, 1, 1, 1);
  cluster.init(conf);
  cluster.start();

  cluster.getResourceManager(0).getRMContext().getRMAdminService()
      .transitionToActive(new HAServiceProtocol.StateChangeRequestInfo(
          HAServiceProtocol.RequestSource.REQUEST_BY_USER));

  assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
}
 
Example 2
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
synchronized void transitionToActive() throws Exception {
  if (rmContext.getHAServiceState() == HAServiceProtocol.HAServiceState.ACTIVE) {
    LOG.info("Already in active state");
    return;
  }

  LOG.info("Transitioning to active state");

  this.rmLoginUGI.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      try {
        startActiveServices();
        return null;
      } catch (Exception e) {
        reinitialize(true);
        throw e;
      }
    }
  });

  rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.ACTIVE);
  LOG.info("Transitioned to active state");
}
 
Example 3
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
synchronized void transitionToStandby(boolean initialize)
    throws Exception {
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.STANDBY) {
    LOG.info("Already in standby state");
    return;
  }

  LOG.info("Transitioning to standby state");
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.ACTIVE) {
    stopActiveServices();
    reinitialize(initialize);
  }
  rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
  LOG.info("Transitioned to standby state");
}
 
Example 4
Source Project: hadoop   Source File: AdminService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void transitionToStandby(
    HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
  // call refreshAdminAcls before HA state transition
  // for the case that adminAcls have been updated in previous active RM
  try {
    refreshAdminAcls(false);
  } catch (YarnException ex) {
    throw new ServiceFailedException("Can not execute refreshAdminAcls", ex);
  }
  UserGroupInformation user = checkAccess("transitionToStandby");
  checkHaStateChange(reqInfo);
  try {
    rm.transitionToStandby(true);
    RMAuditLogger.logSuccess(user.getShortUserName(),
        "transitionToStandby", "RMHAProtocolService");
  } catch (Exception e) {
    RMAuditLogger.logFailure(user.getShortUserName(), "transitionToStandby",
        "", "RMHAProtocolService",
        "Exception transitioning to standby");
    throw new ServiceFailedException(
        "Error when transitioning to Standby mode", e);
  }
}
 
Example 5
Source Project: hadoop   Source File: TestRMAdminCLI.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 500)
public void testTransitionToActive() throws Exception {
  String[] args = {"-transitionToActive", "rm1"};

  // RM HA is disabled.
  // transitionToActive should not be executed
  assertEquals(-1, rmAdminCLI.run(args));
  verify(haadmin, never()).transitionToActive(
      any(HAServiceProtocol.StateChangeRequestInfo.class));

  // Now RM HA is enabled.
  // transitionToActive should be executed
  assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
  verify(haadmin).transitionToActive(
      any(HAServiceProtocol.StateChangeRequestInfo.class));
}
 
Example 6
Source Project: hadoop   Source File: TestRMAdminCLI.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 500)
public void testTransitionToStandby() throws Exception {
  String[] args = {"-transitionToStandby", "rm1"};

  // RM HA is disabled.
  // transitionToStandby should not be executed
  assertEquals(-1, rmAdminCLI.run(args));
  verify(haadmin, never()).transitionToStandby(
      any(HAServiceProtocol.StateChangeRequestInfo.class));

  // Now RM HA is enabled.
  // transitionToActive should be executed
  assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
  verify(haadmin).transitionToStandby(
      any(HAServiceProtocol.StateChangeRequestInfo.class));
}
 
Example 7
Source Project: hadoop   Source File: TestDFSHAAdmin.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
  mockZkfcProtocol = MockitoUtil.mockProtocol(ZKFCProtocol.class);
  tool = new DFSHAAdmin() {

    @Override
    protected HAServiceTarget resolveTarget(String nnId) {
      HAServiceTarget target = super.resolveTarget(nnId);
      HAServiceTarget spy = Mockito.spy(target);
      // OVerride the target to return our mock protocol
      try {
        Mockito.doReturn(mockProtocol).when(spy).getProxy(
            Mockito.<Configuration>any(), Mockito.anyInt());
        Mockito.doReturn(mockZkfcProtocol).when(spy).getZKFCProxy(
            Mockito.<Configuration>any(), Mockito.anyInt());
      } catch (IOException e) {
        throw new AssertionError(e); // mock setup doesn't really throw
      }
      return spy;
    }
  };
  tool.setConf(getHAConf());
  tool.setErrOut(new PrintStream(errOutBytes));
  tool.setOut(new PrintStream(outBytes));
}
 
Example 8
Source Project: big-c   Source File: TestMiniYARNClusterForHA.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException, InterruptedException {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0");

  cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(),
      2, 1, 1, 1);
  cluster.init(conf);
  cluster.start();

  cluster.getResourceManager(0).getRMContext().getRMAdminService()
      .transitionToActive(new HAServiceProtocol.StateChangeRequestInfo(
          HAServiceProtocol.RequestSource.REQUEST_BY_USER));

  assertFalse("RM never turned active", -1 == cluster.getActiveRMIndex());
}
 
Example 9
Source Project: big-c   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
synchronized void transitionToActive() throws Exception {
  if (rmContext.getHAServiceState() == HAServiceProtocol.HAServiceState.ACTIVE) {
    LOG.info("Already in active state");
    return;
  }

  LOG.info("Transitioning to active state");

  this.rmLoginUGI.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run() throws Exception {
      try {
        startActiveServices();
        return null;
      } catch (Exception e) {
        reinitialize(true);
        throw e;
      }
    }
  });

  rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.ACTIVE);
  LOG.info("Transitioned to active state");
}
 
Example 10
Source Project: big-c   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
synchronized void transitionToStandby(boolean initialize)
    throws Exception {
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.STANDBY) {
    LOG.info("Already in standby state");
    return;
  }

  LOG.info("Transitioning to standby state");
  if (rmContext.getHAServiceState() ==
      HAServiceProtocol.HAServiceState.ACTIVE) {
    stopActiveServices();
    reinitialize(initialize);
  }
  rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
  LOG.info("Transitioned to standby state");
}
 
Example 11
Source Project: big-c   Source File: AdminService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void transitionToStandby(
    HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
  // call refreshAdminAcls before HA state transition
  // for the case that adminAcls have been updated in previous active RM
  try {
    refreshAdminAcls(false);
  } catch (YarnException ex) {
    throw new ServiceFailedException("Can not execute refreshAdminAcls", ex);
  }
  UserGroupInformation user = checkAccess("transitionToStandby");
  checkHaStateChange(reqInfo);
  try {
    rm.transitionToStandby(true);
    RMAuditLogger.logSuccess(user.getShortUserName(),
        "transitionToStandby", "RMHAProtocolService");
  } catch (Exception e) {
    RMAuditLogger.logFailure(user.getShortUserName(), "transitionToStandby",
        "", "RMHAProtocolService",
        "Exception transitioning to standby");
    throw new ServiceFailedException(
        "Error when transitioning to Standby mode", e);
  }
}
 
Example 12
Source Project: big-c   Source File: TestRMAdminCLI.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 500)
public void testTransitionToActive() throws Exception {
  String[] args = {"-transitionToActive", "rm1"};

  // RM HA is disabled.
  // transitionToActive should not be executed
  assertEquals(-1, rmAdminCLI.run(args));
  verify(haadmin, never()).transitionToActive(
      any(HAServiceProtocol.StateChangeRequestInfo.class));

  // Now RM HA is enabled.
  // transitionToActive should be executed
  assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
  verify(haadmin).transitionToActive(
      any(HAServiceProtocol.StateChangeRequestInfo.class));
}
 
Example 13
Source Project: big-c   Source File: TestRMAdminCLI.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 500)
public void testTransitionToStandby() throws Exception {
  String[] args = {"-transitionToStandby", "rm1"};

  // RM HA is disabled.
  // transitionToStandby should not be executed
  assertEquals(-1, rmAdminCLI.run(args));
  verify(haadmin, never()).transitionToStandby(
      any(HAServiceProtocol.StateChangeRequestInfo.class));

  // Now RM HA is enabled.
  // transitionToActive should be executed
  assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
  verify(haadmin).transitionToStandby(
      any(HAServiceProtocol.StateChangeRequestInfo.class));
}
 
Example 14
Source Project: big-c   Source File: TestDFSHAAdmin.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  mockProtocol = MockitoUtil.mockProtocol(HAServiceProtocol.class);
  mockZkfcProtocol = MockitoUtil.mockProtocol(ZKFCProtocol.class);
  tool = new DFSHAAdmin() {

    @Override
    protected HAServiceTarget resolveTarget(String nnId) {
      HAServiceTarget target = super.resolveTarget(nnId);
      HAServiceTarget spy = Mockito.spy(target);
      // OVerride the target to return our mock protocol
      try {
        Mockito.doReturn(mockProtocol).when(spy).getProxy(
            Mockito.<Configuration>any(), Mockito.anyInt());
        Mockito.doReturn(mockZkfcProtocol).when(spy).getZKFCProxy(
            Mockito.<Configuration>any(), Mockito.anyInt());
      } catch (IOException e) {
        throw new AssertionError(e); // mock setup doesn't really throw
      }
      return spy;
    }
  };
  tool.setConf(getHAConf());
  tool.setErrOut(new PrintStream(errOutBytes));
  tool.setOut(new PrintStream(outBytes));
}
 
Example 15
Source Project: hadoop   Source File: RMHAUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static HAServiceState getHAState(YarnConfiguration yarnConf)
    throws Exception {
  HAServiceTarget haServiceTarget;
  int rpcTimeoutForChecks =
      yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY,
          CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT);

  yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
      yarnConf.get(YarnConfiguration.RM_PRINCIPAL, ""));
  haServiceTarget = new RMHAServiceTarget(yarnConf);
  HAServiceProtocol proto =
      haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks);
  HAServiceState haState = proto.getServiceStatus().getState();
  return haState;
}
 
Example 16
Source Project: hadoop   Source File: AdminService.java    License: Apache License 2.0 5 votes vote down vote up
protected void startServer() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server = (Server) rpc.getServer(
      ResourceManagerAdministrationProtocol.class, this, masterServiceBindAddress,
      conf, null,
      conf.getInt(YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
          YarnConfiguration.DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
      false)) {
    refreshServiceAcls(
        getConfiguration(conf,
            YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
        RMPolicyProvider.getInstance());
  }

  if (rmContext.isHAEnabled()) {
    RPC.setProtocolEngine(conf, HAServiceProtocolPB.class,
        ProtobufRpcEngine.class);

    HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator =
        new HAServiceProtocolServerSideTranslatorPB(this);
    BlockingService haPbService =
        HAServiceProtocolProtos.HAServiceProtocolService
            .newReflectiveBlockingService(haServiceProtocolXlator);
    server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
        HAServiceProtocol.class, haPbService);
  }

  this.server.start();
  conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
                         YarnConfiguration.RM_ADMIN_ADDRESS,
                         YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
                         server.getListenerAddress());
}
 
Example 17
Source Project: hadoop   Source File: AdminService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void transitionToActive(
    HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
  // call refreshAdminAcls before HA state transition
  // for the case that adminAcls have been updated in previous active RM
  try {
    refreshAdminAcls(false);
  } catch (YarnException ex) {
    throw new ServiceFailedException("Can not execute refreshAdminAcls", ex);
  }

  UserGroupInformation user = checkAccess("transitionToActive");
  checkHaStateChange(reqInfo);
  try {
    rm.transitionToActive();
    // call all refresh*s for active RM to get the updated configurations.
    refreshAll();
    RMAuditLogger.logSuccess(user.getShortUserName(),
        "transitionToActive", "RMHAProtocolService");
  } catch (Exception e) {
    RMAuditLogger.logFailure(user.getShortUserName(), "transitionToActive",
        "", "RMHAProtocolService",
        "Exception transitioning to active");
    throw new ServiceFailedException(
        "Error when transitioning to Active mode", e);
  }
}
 
Example 18
Source Project: hadoop   Source File: AdminService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized HAServiceStatus getServiceStatus() throws IOException {
  checkAccess("getServiceState");
  HAServiceState haState = rmContext.getHAServiceState();
  HAServiceStatus ret = new HAServiceStatus(haState);
  if (isRMActive() || haState == HAServiceProtocol.HAServiceState.STANDBY) {
    ret.setReadyToBecomeActive();
  } else {
    ret.setNotReadyToBecomeActive("State is " + haState);
  }
  return ret;
}
 
Example 19
Source Project: hadoop   Source File: TestNNHealthCheck.java    License: Apache License 2.0 5 votes vote down vote up
private void doNNHealthCheckTest() throws IOException {
  NameNodeResourceChecker mockResourceChecker = Mockito.mock(
      NameNodeResourceChecker.class);
  Mockito.doReturn(true).when(mockResourceChecker).hasAvailableDiskSpace();
  cluster.getNameNode(0).getNamesystem()
      .setNNResourceChecker(mockResourceChecker);

  NNHAServiceTarget haTarget = new NNHAServiceTarget(conf,
      DFSUtil.getNamenodeNameServiceId(conf), "nn1");
  HAServiceProtocol rpc = haTarget.getHealthMonitorProxy(conf, conf.getInt(
      HA_HM_RPC_TIMEOUT_KEY, HA_HM_RPC_TIMEOUT_DEFAULT));

  // Should not throw error, which indicates healthy.
  rpc.monitorHealth();

  Mockito.doReturn(false).when(mockResourceChecker).hasAvailableDiskSpace();

  try {
    // Should throw error - NN is unhealthy.
    rpc.monitorHealth();
    fail("Should not have succeeded in calling monitorHealth");
  } catch (HealthCheckFailedException hcfe) {
    GenericTestUtils.assertExceptionContains(
        "The NameNode has no resources available", hcfe);
  } catch (RemoteException re) {
    GenericTestUtils.assertExceptionContains(
        "The NameNode has no resources available",
        re.unwrapRemoteException(HealthCheckFailedException.class));
  }
}
 
Example 20
Source Project: big-c   Source File: RMHAUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static HAServiceState getHAState(YarnConfiguration yarnConf)
    throws Exception {
  HAServiceTarget haServiceTarget;
  int rpcTimeoutForChecks =
      yarnConf.getInt(CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_KEY,
          CommonConfigurationKeys.HA_FC_CLI_CHECK_TIMEOUT_DEFAULT);

  yarnConf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
      yarnConf.get(YarnConfiguration.RM_PRINCIPAL, ""));
  haServiceTarget = new RMHAServiceTarget(yarnConf);
  HAServiceProtocol proto =
      haServiceTarget.getProxy(yarnConf, rpcTimeoutForChecks);
  HAServiceState haState = proto.getServiceStatus().getState();
  return haState;
}
 
Example 21
Source Project: big-c   Source File: AdminService.java    License: Apache License 2.0 5 votes vote down vote up
protected void startServer() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server = (Server) rpc.getServer(
      ResourceManagerAdministrationProtocol.class, this, masterServiceBindAddress,
      conf, null,
      conf.getInt(YarnConfiguration.RM_ADMIN_CLIENT_THREAD_COUNT,
          YarnConfiguration.DEFAULT_RM_ADMIN_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
      false)) {
    refreshServiceAcls(
        getConfiguration(conf,
            YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE),
        RMPolicyProvider.getInstance());
  }

  if (rmContext.isHAEnabled()) {
    RPC.setProtocolEngine(conf, HAServiceProtocolPB.class,
        ProtobufRpcEngine.class);

    HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator =
        new HAServiceProtocolServerSideTranslatorPB(this);
    BlockingService haPbService =
        HAServiceProtocolProtos.HAServiceProtocolService
            .newReflectiveBlockingService(haServiceProtocolXlator);
    server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
        HAServiceProtocol.class, haPbService);
  }

  this.server.start();
  conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
                         YarnConfiguration.RM_ADMIN_ADDRESS,
                         YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
                         server.getListenerAddress());
}
 
Example 22
Source Project: big-c   Source File: AdminService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void transitionToActive(
    HAServiceProtocol.StateChangeRequestInfo reqInfo) throws IOException {
  // call refreshAdminAcls before HA state transition
  // for the case that adminAcls have been updated in previous active RM
  try {
    refreshAdminAcls(false);
  } catch (YarnException ex) {
    throw new ServiceFailedException("Can not execute refreshAdminAcls", ex);
  }

  UserGroupInformation user = checkAccess("transitionToActive");
  checkHaStateChange(reqInfo);
  try {
    rm.transitionToActive();
    // call all refresh*s for active RM to get the updated configurations.
    refreshAll();
    RMAuditLogger.logSuccess(user.getShortUserName(),
        "transitionToActive", "RMHAProtocolService");
  } catch (Exception e) {
    RMAuditLogger.logFailure(user.getShortUserName(), "transitionToActive",
        "", "RMHAProtocolService",
        "Exception transitioning to active");
    throw new ServiceFailedException(
        "Error when transitioning to Active mode", e);
  }
}
 
Example 23
Source Project: big-c   Source File: AdminService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized HAServiceStatus getServiceStatus() throws IOException {
  checkAccess("getServiceState");
  HAServiceState haState = rmContext.getHAServiceState();
  HAServiceStatus ret = new HAServiceStatus(haState);
  if (isRMActive() || haState == HAServiceProtocol.HAServiceState.STANDBY) {
    ret.setReadyToBecomeActive();
  } else {
    ret.setNotReadyToBecomeActive("State is " + haState);
  }
  return ret;
}
 
Example 24
Source Project: hadoop   Source File: TestRMAdminService.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRMHAWithFileSystemBasedConfiguration() throws IOException,
    YarnException {
  StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
      HAServiceProtocol.RequestSource.REQUEST_BY_USER);
  configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
      "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
  configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  configuration.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
  int base = 100;
  for (String confKey : YarnConfiguration
      .getServiceAddressConfKeys(configuration)) {
    configuration.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:"
        + (base + 20));
    configuration.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:"
        + (base + 40));
    base = base * 2;
  }
  Configuration conf1 = new Configuration(configuration);
  conf1.set(YarnConfiguration.RM_HA_ID, "rm1");
  Configuration conf2 = new Configuration(configuration);
  conf2.set(YarnConfiguration.RM_HA_ID, "rm2");

  // upload default configurations
  uploadDefaultConfiguration();

  MockRM rm1 = null;
  MockRM rm2 = null;
  try {
    rm1 = new MockRM(conf1);
    rm1.init(conf1);
    rm1.start();
    Assert.assertTrue(rm1.getRMContext().getHAServiceState()
        == HAServiceState.STANDBY);

    rm2 = new MockRM(conf2);
    rm2.init(conf1);
    rm2.start();
    Assert.assertTrue(rm2.getRMContext().getHAServiceState()
        == HAServiceState.STANDBY);

    rm1.adminService.transitionToActive(requestInfo);
    Assert.assertTrue(rm1.getRMContext().getHAServiceState()
        == HAServiceState.ACTIVE);

    CapacitySchedulerConfiguration csConf =
        new CapacitySchedulerConfiguration();
    csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
    uploadConfiguration(csConf, "capacity-scheduler.xml");

    rm1.adminService.refreshQueues(RefreshQueuesRequest.newInstance());

    int maxApps =
        ((CapacityScheduler) rm1.getRMContext().getScheduler())
            .getConfiguration().getMaximumSystemApplications();
    Assert.assertEquals(maxApps, 5000);

    // Before failover happens, the maxApps is
    // still the default value on the standby rm : rm2
    int maxAppsBeforeFailOver =
        ((CapacityScheduler) rm2.getRMContext().getScheduler())
            .getConfiguration().getMaximumSystemApplications();
    Assert.assertEquals(maxAppsBeforeFailOver, 10000);

    // Do the failover
    rm1.adminService.transitionToStandby(requestInfo);
    rm2.adminService.transitionToActive(requestInfo);
    Assert.assertTrue(rm1.getRMContext().getHAServiceState()
        == HAServiceState.STANDBY);
    Assert.assertTrue(rm2.getRMContext().getHAServiceState()
        == HAServiceState.ACTIVE);

    int maxAppsAfter =
        ((CapacityScheduler) rm2.getRMContext().getScheduler())
            .getConfiguration().getMaximumSystemApplications();

    Assert.assertEquals(maxAppsAfter, 5000);
  } finally {
    if (rm1 != null) {
      rm1.stop();
    }
    if (rm2 != null) {
      rm2.stop();
    }
  }
}
 
Example 25
Source Project: hadoop   Source File: TestRMHA.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test to verify the following RM HA transitions to the following states.
 * 1. Standby: Should be a no-op
 * 2. Active: Active services should start
 * 3. Active: Should be a no-op.
 *    While active, submit a couple of jobs
 * 4. Standby: Active services should stop
 * 5. Active: Active services should start
 * 6. Stop the RM: All services should stop and RM should not be ready to
 * become Active
 */
@Test (timeout = 30000)
public void testFailoverAndTransitions() throws Exception {
  configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  Configuration conf = new YarnConfiguration(configuration);

  rm = new MockRM(conf);
  rm.init(conf);
  StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
      HAServiceProtocol.RequestSource.REQUEST_BY_USER);

  assertEquals(STATE_ERR, HAServiceState.INITIALIZING,
      rm.adminService.getServiceStatus().getState());
  assertFalse("RM is ready to become active before being started",
      rm.adminService.getServiceStatus().isReadyToBecomeActive());
  checkMonitorHealth();

  rm.start();
  checkMonitorHealth();
  checkStandbyRMFunctionality();
  verifyClusterMetrics(0, 0, 0, 0, 0, 0);
  
  // 1. Transition to Standby - must be a no-op
  rm.adminService.transitionToStandby(requestInfo);
  checkMonitorHealth();
  checkStandbyRMFunctionality();
  verifyClusterMetrics(0, 0, 0, 0, 0, 0);
  
  // 2. Transition to active
  rm.adminService.transitionToActive(requestInfo);
  checkMonitorHealth();
  checkActiveRMFunctionality();
  verifyClusterMetrics(1, 1, 1, 1, 2048, 1);
  
  // 3. Transition to active - no-op
  rm.adminService.transitionToActive(requestInfo);
  checkMonitorHealth();
  checkActiveRMFunctionality();
  verifyClusterMetrics(1, 2, 2, 2, 2048, 2);
  
  // 4. Transition to standby
  rm.adminService.transitionToStandby(requestInfo);
  checkMonitorHealth();
  checkStandbyRMFunctionality();
  verifyClusterMetrics(0, 0, 0, 0, 0, 0);
 
  // 5. Transition to active to check Active->Standby->Active works
  rm.adminService.transitionToActive(requestInfo);
  checkMonitorHealth();
  checkActiveRMFunctionality();
  verifyClusterMetrics(1, 1, 1, 1, 2048, 1);
  
  // 6. Stop the RM. All services should stop and RM should not be ready to
  // become active
  rm.stop();
  assertEquals(STATE_ERR, HAServiceState.STOPPING,
      rm.adminService.getServiceStatus().getState());
  assertFalse("RM is ready to become active even after it is stopped",
      rm.adminService.getServiceStatus().isReadyToBecomeActive());
  assertFalse("Active RM services are started",
      rm.areActiveServicesRunning());
  checkMonitorHealth();
}
 
Example 26
Source Project: hadoop   Source File: TestRMHA.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRMDispatcherForHA() throws IOException {
  String errorMessageForEventHandler =
      "Expect to get the same number of handlers";
  String errorMessageForService = "Expect to get the same number of services";
  configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  Configuration conf = new YarnConfiguration(configuration);
  rm = new MockRM(conf) {
    @Override
    protected Dispatcher createDispatcher() {
      return new MyCountingDispatcher();
    }
  };
  rm.init(conf);
  int expectedEventHandlerCount =
      ((MyCountingDispatcher) rm.getRMContext().getDispatcher())
          .getEventHandlerCount();
  int expectedServiceCount = rm.getServices().size();
  assertTrue(expectedEventHandlerCount != 0);

  StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
      HAServiceProtocol.RequestSource.REQUEST_BY_USER);

  assertEquals(STATE_ERR, HAServiceState.INITIALIZING,
      rm.adminService.getServiceStatus().getState());
  assertFalse("RM is ready to become active before being started",
      rm.adminService.getServiceStatus().isReadyToBecomeActive());
  rm.start();

  //call transitions to standby and active a couple of times
  rm.adminService.transitionToStandby(requestInfo);
  rm.adminService.transitionToActive(requestInfo);
  rm.adminService.transitionToStandby(requestInfo);
  rm.adminService.transitionToActive(requestInfo);
  rm.adminService.transitionToStandby(requestInfo);
  
  MyCountingDispatcher dispatcher =
      (MyCountingDispatcher) rm.getRMContext().getDispatcher();
  assertTrue(!dispatcher.isStopped());

  rm.adminService.transitionToActive(requestInfo);
  assertEquals(errorMessageForEventHandler, expectedEventHandlerCount,
      ((MyCountingDispatcher) rm.getRMContext().getDispatcher())
      .getEventHandlerCount());
  assertEquals(errorMessageForService, expectedServiceCount,
      rm.getServices().size());

  
  // Keep the dispatcher reference before transitioning to standby
  dispatcher = (MyCountingDispatcher) rm.getRMContext().getDispatcher();
  
  
  rm.adminService.transitionToStandby(requestInfo);
  assertEquals(errorMessageForEventHandler, expectedEventHandlerCount,
      ((MyCountingDispatcher) rm.getRMContext().getDispatcher())
      .getEventHandlerCount());
  assertEquals(errorMessageForService, expectedServiceCount,
      rm.getServices().size());

  assertTrue(dispatcher.isStopped());
  
  rm.stop();
}
 
Example 27
Source Project: hadoop   Source File: TestRMHA.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout = 30000)
public void testFailoverWhenTransitionToActiveThrowException()
    throws Exception {
  configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  Configuration conf = new YarnConfiguration(configuration);

  MemoryRMStateStore memStore = new MemoryRMStateStore() {
    int count = 0;

    @Override
    public synchronized void startInternal() throws Exception {
      // first time throw exception
      if (count++ == 0) {
        throw new Exception("Session Expired");
      }
    }
  };
  // start RM
  memStore.init(conf);

  rm = new MockRM(conf, memStore);
  rm.init(conf);
  StateChangeRequestInfo requestInfo =
      new StateChangeRequestInfo(
          HAServiceProtocol.RequestSource.REQUEST_BY_USER);

  assertEquals(STATE_ERR, HAServiceState.INITIALIZING, rm.adminService
      .getServiceStatus().getState());
  assertFalse("RM is ready to become active before being started",
      rm.adminService.getServiceStatus().isReadyToBecomeActive());
  checkMonitorHealth();

  rm.start();
  checkMonitorHealth();
  checkStandbyRMFunctionality();

  // 2. Try Transition to active, throw exception
  try {
    rm.adminService.transitionToActive(requestInfo);
    Assert.fail("Transitioned to Active should throw exception.");
  } catch (Exception e) {
    assertTrue("Error when transitioning to Active mode".contains(e
        .getMessage()));
  }

  // 3. Transition to active, success
  rm.adminService.transitionToActive(requestInfo);
  checkMonitorHealth();
  checkActiveRMFunctionality();
}
 
Example 28
Source Project: hadoop   Source File: TestRMHA.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testFailoverClearsRMContext() throws Exception {
  configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  configuration.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  Configuration conf = new YarnConfiguration(configuration);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // 1. start RM
  rm = new MockRM(conf, memStore);
  rm.init(conf);
  rm.start();

  StateChangeRequestInfo requestInfo =
      new StateChangeRequestInfo(
          HAServiceProtocol.RequestSource.REQUEST_BY_USER);
  checkMonitorHealth();
  checkStandbyRMFunctionality();

  // 2. Transition to active
  rm.adminService.transitionToActive(requestInfo);
  checkMonitorHealth();
  checkActiveRMFunctionality();
  verifyClusterMetrics(1, 1, 1, 1, 2048, 1);
  assertEquals(1, rm.getRMContext().getRMNodes().size());
  assertEquals(1, rm.getRMContext().getRMApps().size());

  // 3. Create new RM
  rm = new MockRM(conf, memStore) {
    @Override
    protected ResourceTrackerService createResourceTrackerService() {
      return new ResourceTrackerService(this.rmContext,
          this.nodesListManager, this.nmLivelinessMonitor,
          this.rmContext.getContainerTokenSecretManager(),
          this.rmContext.getNMTokenSecretManager()) {
        @Override
        protected void serviceStart() throws Exception {
          throw new Exception("ResourceTracker service failed");
        }
      };
    }
  };
  rm.init(conf);
  rm.start();
  checkMonitorHealth();
  checkStandbyRMFunctionality();

  // 4. Try Transition to active, throw exception
  try {
    rm.adminService.transitionToActive(requestInfo);
    Assert.fail("Transitioned to Active should throw exception.");
  } catch (Exception e) {
    assertTrue("Error when transitioning to Active mode".contains(e
        .getMessage()));
  }
  // 5. Clears the metrics
  verifyClusterMetrics(0, 0, 0, 0, 0, 0);
  assertEquals(0, rm.getRMContext().getRMNodes().size());
  assertEquals(0, rm.getRMContext().getRMApps().size());
}
 
Example 29
Source Project: hadoop   Source File: TestZKRMStateStore.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testFencing() throws Exception {
  StateChangeRequestInfo req = new StateChangeRequestInfo(
      HAServiceProtocol.RequestSource.REQUEST_BY_USER);

  Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234);
  conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  ResourceManager rm1 = new ResourceManager();
  rm1.init(conf1);
  rm1.start();
  rm1.getRMContext().getRMAdminService().transitionToActive(req);
  assertEquals("RM with ZKStore didn't start",
      Service.STATE.STARTED, rm1.getServiceState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm1.getRMContext().getRMAdminService().getServiceStatus().getState());

  Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678);
  conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
  ResourceManager rm2 = new ResourceManager();
  rm2.init(conf2);
  rm2.start();
  rm2.getRMContext().getRMAdminService().transitionToActive(req);
  assertEquals("RM with ZKStore didn't start",
      Service.STATE.STARTED, rm2.getServiceState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm2.getRMContext().getRMAdminService().getServiceStatus().getState());

  for (int i = 0; i < ZK_TIMEOUT_MS / 50; i++) {
    if (HAServiceProtocol.HAServiceState.ACTIVE ==
        rm1.getRMContext().getRMAdminService().getServiceStatus().getState()) {
      Thread.sleep(100);
    }
  }
  assertEquals("RM should have been fenced",
      HAServiceProtocol.HAServiceState.STANDBY,
      rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
  assertEquals("RM should be Active",
      HAServiceProtocol.HAServiceState.ACTIVE,
      rm2.getRMContext().getRMAdminService().getServiceStatus().getState());
}
 
Example 30
Source Project: hadoop   Source File: TestRMAdminCLI.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("static-access")
@Before
public void configure() throws IOException, YarnException {
  remoteAdminServiceAccessed = false;
  admin = mock(ResourceManagerAdministrationProtocol.class);
  when(admin.addToClusterNodeLabels(any(AddToClusterNodeLabelsRequest.class)))
      .thenAnswer(new Answer<AddToClusterNodeLabelsResponse>() {

        @Override
        public AddToClusterNodeLabelsResponse answer(
            InvocationOnMock invocation) throws Throwable {
          remoteAdminServiceAccessed = true;
          return AddToClusterNodeLabelsResponse.newInstance();
        }
      });

  haadmin = mock(HAServiceProtocol.class);
  when(haadmin.getServiceStatus()).thenReturn(new HAServiceStatus(
      HAServiceProtocol.HAServiceState.INITIALIZING));

  final HAServiceTarget haServiceTarget = mock(HAServiceTarget.class);
  when(haServiceTarget.getProxy(any(Configuration.class), anyInt()))
      .thenReturn(haadmin);
  rmAdminCLI = new RMAdminCLI(new Configuration()) {
    @Override
    protected ResourceManagerAdministrationProtocol createAdminProtocol()
        throws IOException {
      return admin;
    }

    @Override
    protected HAServiceTarget resolveTarget(String rmId) {
      return haServiceTarget;
    }
  };
  initDummyNodeLabelsManager();
  rmAdminCLI.localNodeLabelsManager = dummyNodeLabelsManager;

  YarnConfiguration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  rmAdminCLIWithHAEnabled = new RMAdminCLI(conf) {

    @Override
    protected ResourceManagerAdministrationProtocol createAdminProtocol()
        throws IOException {
      return admin;
    }

    @Override
    protected HAServiceTarget resolveTarget(String rmId) {
      return haServiceTarget;
    }
  };
}