org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestZKRMStateStorePerf.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
Example #2
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
Example #3
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
Example #4
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
Example #5
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
Example #6
Source File: TestZKRMStateStorePerf.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
Example #7
Source File: RMActiveServiceContext.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
Example #8
Source File: RMStateStoreTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected Token<AMRMTokenIdentifier> generateAMRMToken(
    ApplicationAttemptId attemptId,
    AMRMTokenSecretManager appTokenMgr) {
  Token<AMRMTokenIdentifier> appToken =
      appTokenMgr.createAndGetAMRMToken(attemptId);
  appToken.setService(new Text("appToken service"));
  return appToken;
}
Example #9
Source File: RMActiveServiceContext.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
Example #10
Source File: RMStateStoreTestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected Token<AMRMTokenIdentifier> generateAMRMToken(
    ApplicationAttemptId attemptId,
    AMRMTokenSecretManager appTokenMgr) {
  Token<AMRMTokenIdentifier> appToken =
      appTokenMgr.createAndGetAMRMToken(attemptId);
  appToken.setService(new Text("appToken service"));
  return appToken;
}
Example #11
Source File: RMActiveServiceContext.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Private
@Unstable
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  this.amRMTokenSecretManager = amRMTokenSecretManager;
}
Example #12
Source File: MockRMContext.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up 
@Override
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return tokenSecretManager;
}
Example #13
Source File: MockRMContext.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up 
public void setTokenSecretManager(AMRMTokenSecretManager tokenSecretManager) {
  this.tokenSecretManager = tokenSecretManager;
}
Example #14
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test(timeout=60000)
public void testAMRMTokenUpdate() throws Exception {
  LOG.info("Running testAMRMTokenUpdate");

  final String rmAddr = "somermaddress:1234";
  final Configuration conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);

  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  AMRMTokenSecretManager secretMgr =
      rm.getRMContext().getAMRMTokenSecretManager();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  final ApplicationId appId = app.getApplicationId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  final Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));

  final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
      .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
  Assert.assertNotNull("app should have a token", oldToken);
  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
      new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
        @Override
        public Token<AMRMTokenIdentifier> run() throws Exception {
          MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
              appAttemptId, mockJob);

          // Keep heartbeating until RM thinks the token has been updated
          Token<AMRMTokenIdentifier> currentToken = oldToken;
          long startTime = Time.monotonicNow();
          while (currentToken == oldToken) {
            if (Time.monotonicNow() - startTime > 20000) {
              Assert.fail("Took to long to see AMRM token change");
            }
            Thread.sleep(100);
            allocator.schedule();
            currentToken = rm.getRMContext().getRMApps().get(appId)
                .getRMAppAttempt(appAttemptId).getAMRMToken();
          }

          return currentToken;
        }
      });

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(rmAddr), ugiToken.getService());
}
Example #15
Source File: RMSecretManagerService.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
protected AMRMTokenSecretManager createAMRMTokenSecretManager(
    Configuration conf, RMContext rmContext) {
  return new AMRMTokenSecretManager(conf, rmContext);
}
Example #16
Source File: RMStateStoreTestBase.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public void testAMRMTokenSecretManagerStateStore(
    RMStateStoreHelper stateStoreHelper) throws Exception {
  System.out.println("Start testing");
  RMStateStore store = stateStoreHelper.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getStateStore()).thenReturn(store);
  Configuration conf = new YarnConfiguration();
  AMRMTokenSecretManager appTokenMgr =
      new AMRMTokenSecretManager(conf, rmContext);

  //create and save the first masterkey
  MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();

  AMRMTokenSecretManagerState state1 =
      AMRMTokenSecretManagerState.newInstance(
        firstMasterKeyData.getMasterKey(), null);
  rmContext.getStateStore()
      .storeOrUpdateAMRMTokenSecretManager(state1,
    false);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state = store.loadState();
  Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertNull(state
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  //create and save the second masterkey
  MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
  AMRMTokenSecretManagerState state2 =
      AMRMTokenSecretManagerState
        .newInstance(firstMasterKeyData.getMasterKey(),
          secondMasterKeyData.getMasterKey());
  rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state2,
    true);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state_2 = store.loadState();
  Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  // re-create the masterKeyData based on the recovered masterkey
  // should have the same secretKey
  appTokenMgr.recover(state_2);
  Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(),
    firstMasterKeyData.getSecretKey());
  Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(),
    secondMasterKeyData.getSecretKey());

  store.close();
}
Example #17
Source File: RMActiveServiceContext.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Private
@Unstable
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return this.amRMTokenSecretManager;
}
Example #18
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
Example #19
Source File: RMContextImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  activeServiceContext.setAMRMTokenSecretManager(amRMTokenSecretManager);
}
Example #20
Source File: RMContextImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return activeServiceContext.getAMRMTokenSecretManager();
}
Example #21
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
Example #22
Source File: RMActiveServiceContext.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Private
@Unstable
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  this.amRMTokenSecretManager = amRMTokenSecretManager;
}
Example #23
Source File: RMActiveServiceContext.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Private
@Unstable
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return this.amRMTokenSecretManager;
}
Example #24
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return activeServiceContext.getAMRMTokenSecretManager();
}
Example #25
Source File: RMSecretManagerService.java    From big-c with Apache License 2.0 4 votes vote down vote up 
protected AMRMTokenSecretManager createAMRMTokenSecretManager(
    Configuration conf, RMContext rmContext) {
  return new AMRMTokenSecretManager(conf, rmContext);
}
Example #26
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test(timeout=60000)
public void testAMRMTokenUpdate() throws Exception {
  LOG.info("Running testAMRMTokenUpdate");

  final String rmAddr = "somermaddress:1234";
  final Configuration conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);

  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  AMRMTokenSecretManager secretMgr =
      rm.getRMContext().getAMRMTokenSecretManager();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  final ApplicationId appId = app.getApplicationId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  final Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));

  final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
      .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
  Assert.assertNotNull("app should have a token", oldToken);
  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
      new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
        @Override
        public Token<AMRMTokenIdentifier> run() throws Exception {
          MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
              appAttemptId, mockJob);

          // Keep heartbeating until RM thinks the token has been updated
          Token<AMRMTokenIdentifier> currentToken = oldToken;
          long startTime = Time.monotonicNow();
          while (currentToken == oldToken) {
            if (Time.monotonicNow() - startTime > 20000) {
              Assert.fail("Took to long to see AMRM token change");
            }
            Thread.sleep(100);
            allocator.schedule();
            currentToken = rm.getRMContext().getRMApps().get(appId)
                .getRMAppAttempt(appAttemptId).getAMRMToken();
          }

          return currentToken;
        }
      });

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(rmAddr), ugiToken.getService());
}
Example #27
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  activeServiceContext.setAMRMTokenSecretManager(amRMTokenSecretManager);
}
Example #28
Source File: RMStateStoreTestBase.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public void testAMRMTokenSecretManagerStateStore(
    RMStateStoreHelper stateStoreHelper) throws Exception {
  System.out.println("Start testing");
  RMStateStore store = stateStoreHelper.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getStateStore()).thenReturn(store);
  Configuration conf = new YarnConfiguration();
  AMRMTokenSecretManager appTokenMgr =
      new AMRMTokenSecretManager(conf, rmContext);

  //create and save the first masterkey
  MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();

  AMRMTokenSecretManagerState state1 =
      AMRMTokenSecretManagerState.newInstance(
        firstMasterKeyData.getMasterKey(), null);
  rmContext.getStateStore()
      .storeOrUpdateAMRMTokenSecretManager(state1,
    false);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state = store.loadState();
  Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertNull(state
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  //create and save the second masterkey
  MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
  AMRMTokenSecretManagerState state2 =
      AMRMTokenSecretManagerState
        .newInstance(firstMasterKeyData.getMasterKey(),
          secondMasterKeyData.getMasterKey());
  rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state2,
    true);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state_2 = store.loadState();
  Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  // re-create the masterKeyData based on the recovered masterkey
  // should have the same secretKey
  appTokenMgr.recover(state_2);
  Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(),
    firstMasterKeyData.getSecretKey());
  Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(),
    secondMasterKeyData.getSecretKey());

  store.close();
}
Example #29
Source File: RMContext.java    From hadoop with Apache License 2.0 votes vote down vote up 
AMRMTokenSecretManager getAMRMTokenSecretManager();
Example #30
Source File: RMContext.java    From big-c with Apache License 2.0 votes vote down vote up 
AMRMTokenSecretManager getAMRMTokenSecretManager();