Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 2
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
 
Example 3
Source Project: hadoop   Source File: TestZKRMStateStorePerf.java    License: Apache License 2.0 6 votes vote down vote up
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
 
Example 4
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 5
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
 
Example 6
Source Project: big-c   Source File: TestZKRMStateStorePerf.java    License: Apache License 2.0 6 votes vote down vote up
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
 
Example 7
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
 
Example 8
Source Project: hadoop   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected Token<AMRMTokenIdentifier> generateAMRMToken(
    ApplicationAttemptId attemptId,
    AMRMTokenSecretManager appTokenMgr) {
  Token<AMRMTokenIdentifier> appToken =
      appTokenMgr.createAndGetAMRMToken(attemptId);
  appToken.setService(new Text("appToken service"));
  return appToken;
}
 
Example 9
Source Project: big-c   Source File: RMActiveServiceContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
 
Example 10
Source Project: big-c   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected Token<AMRMTokenIdentifier> generateAMRMToken(
    ApplicationAttemptId attemptId,
    AMRMTokenSecretManager appTokenMgr) {
  Token<AMRMTokenIdentifier> appToken =
      appTokenMgr.createAndGetAMRMToken(attemptId);
  appToken.setService(new Text("appToken service"));
  return appToken;
}
 
Example 11
Source Project: hadoop   Source File: RMSecretManagerService.java    License: Apache License 2.0 4 votes vote down vote up
protected AMRMTokenSecretManager createAMRMTokenSecretManager(
    Configuration conf, RMContext rmContext) {
  return new AMRMTokenSecretManager(conf, rmContext);
}
 
Example 12
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 4 votes vote down vote up
@Private
@Unstable
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return this.amRMTokenSecretManager;
}
 
Example 13
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 4 votes vote down vote up
@Private
@Unstable
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  this.amRMTokenSecretManager = amRMTokenSecretManager;
}
 
Example 14
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return activeServiceContext.getAMRMTokenSecretManager();
}
 
Example 15
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 4 votes vote down vote up
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  activeServiceContext.setAMRMTokenSecretManager(amRMTokenSecretManager);
}
 
Example 16
Source Project: hadoop   Source File: TestRMAppTransitions.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
 
Example 17
Source Project: hadoop   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 4 votes vote down vote up
public void testAMRMTokenSecretManagerStateStore(
    RMStateStoreHelper stateStoreHelper) throws Exception {
  System.out.println("Start testing");
  RMStateStore store = stateStoreHelper.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getStateStore()).thenReturn(store);
  Configuration conf = new YarnConfiguration();
  AMRMTokenSecretManager appTokenMgr =
      new AMRMTokenSecretManager(conf, rmContext);

  //create and save the first masterkey
  MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();

  AMRMTokenSecretManagerState state1 =
      AMRMTokenSecretManagerState.newInstance(
        firstMasterKeyData.getMasterKey(), null);
  rmContext.getStateStore()
      .storeOrUpdateAMRMTokenSecretManager(state1,
    false);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state = store.loadState();
  Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertNull(state
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  //create and save the second masterkey
  MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
  AMRMTokenSecretManagerState state2 =
      AMRMTokenSecretManagerState
        .newInstance(firstMasterKeyData.getMasterKey(),
          secondMasterKeyData.getMasterKey());
  rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state2,
    true);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state_2 = store.loadState();
  Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  // re-create the masterKeyData based on the recovered masterkey
  // should have the same secretKey
  appTokenMgr.recover(state_2);
  Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(),
    firstMasterKeyData.getSecretKey());
  Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(),
    secondMasterKeyData.getSecretKey());

  store.close();
}
 
Example 18
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=60000)
public void testAMRMTokenUpdate() throws Exception {
  LOG.info("Running testAMRMTokenUpdate");

  final String rmAddr = "somermaddress:1234";
  final Configuration conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);

  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  AMRMTokenSecretManager secretMgr =
      rm.getRMContext().getAMRMTokenSecretManager();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  final ApplicationId appId = app.getApplicationId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  final Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));

  final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
      .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
  Assert.assertNotNull("app should have a token", oldToken);
  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
      new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
        @Override
        public Token<AMRMTokenIdentifier> run() throws Exception {
          MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
              appAttemptId, mockJob);

          // Keep heartbeating until RM thinks the token has been updated
          Token<AMRMTokenIdentifier> currentToken = oldToken;
          long startTime = Time.monotonicNow();
          while (currentToken == oldToken) {
            if (Time.monotonicNow() - startTime > 20000) {
              Assert.fail("Took to long to see AMRM token change");
            }
            Thread.sleep(100);
            allocator.schedule();
            currentToken = rm.getRMContext().getRMApps().get(appId)
                .getRMAppAttempt(appAttemptId).getAMRMToken();
          }

          return currentToken;
        }
      });

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(rmAddr), ugiToken.getService());
}
 
Example 19
Source Project: big-c   Source File: RMSecretManagerService.java    License: Apache License 2.0 4 votes vote down vote up
protected AMRMTokenSecretManager createAMRMTokenSecretManager(
    Configuration conf, RMContext rmContext) {
  return new AMRMTokenSecretManager(conf, rmContext);
}
 
Example 20
Source Project: big-c   Source File: RMActiveServiceContext.java    License: Apache License 2.0 4 votes vote down vote up
@Private
@Unstable
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return this.amRMTokenSecretManager;
}
 
Example 21
Source Project: big-c   Source File: RMActiveServiceContext.java    License: Apache License 2.0 4 votes vote down vote up
@Private
@Unstable
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  this.amRMTokenSecretManager = amRMTokenSecretManager;
}
 
Example 22
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return activeServiceContext.getAMRMTokenSecretManager();
}
 
Example 23
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 4 votes vote down vote up
void setAMRMTokenSecretManager(AMRMTokenSecretManager amRMTokenSecretManager) {
  activeServiceContext.setAMRMTokenSecretManager(amRMTokenSecretManager);
}
 
Example 24
Source Project: big-c   Source File: TestRMAppTransitions.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
 
Example 25
Source Project: big-c   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 4 votes vote down vote up
public void testAMRMTokenSecretManagerStateStore(
    RMStateStoreHelper stateStoreHelper) throws Exception {
  System.out.println("Start testing");
  RMStateStore store = stateStoreHelper.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getStateStore()).thenReturn(store);
  Configuration conf = new YarnConfiguration();
  AMRMTokenSecretManager appTokenMgr =
      new AMRMTokenSecretManager(conf, rmContext);

  //create and save the first masterkey
  MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();

  AMRMTokenSecretManagerState state1 =
      AMRMTokenSecretManagerState.newInstance(
        firstMasterKeyData.getMasterKey(), null);
  rmContext.getStateStore()
      .storeOrUpdateAMRMTokenSecretManager(state1,
    false);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state = store.loadState();
  Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertNull(state
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  //create and save the second masterkey
  MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
  AMRMTokenSecretManagerState state2 =
      AMRMTokenSecretManagerState
        .newInstance(firstMasterKeyData.getMasterKey(),
          secondMasterKeyData.getMasterKey());
  rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManager(state2,
    true);

  // load state
  store = stateStoreHelper.getRMStateStore();
  when(rmContext.getStateStore()).thenReturn(store);
  store.setRMDispatcher(dispatcher);
  RMState state_2 = store.loadState();
  Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
  Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getCurrentMasterKey());
  Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2
    .getAMRMTokenSecretManagerState().getNextMasterKey());

  // re-create the masterKeyData based on the recovered masterkey
  // should have the same secretKey
  appTokenMgr.recover(state_2);
  Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(),
    firstMasterKeyData.getSecretKey());
  Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(),
    secondMasterKeyData.getSecretKey());

  store.close();
}
 
Example 26
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=60000)
public void testAMRMTokenUpdate() throws Exception {
  LOG.info("Running testAMRMTokenUpdate");

  final String rmAddr = "somermaddress:1234";
  final Configuration conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);

  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  AMRMTokenSecretManager secretMgr =
      rm.getRMContext().getAMRMTokenSecretManager();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  final ApplicationId appId = app.getApplicationId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  final Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));

  final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
      .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
  Assert.assertNotNull("app should have a token", oldToken);
  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
      new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
        @Override
        public Token<AMRMTokenIdentifier> run() throws Exception {
          MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
              appAttemptId, mockJob);

          // Keep heartbeating until RM thinks the token has been updated
          Token<AMRMTokenIdentifier> currentToken = oldToken;
          long startTime = Time.monotonicNow();
          while (currentToken == oldToken) {
            if (Time.monotonicNow() - startTime > 20000) {
              Assert.fail("Took to long to see AMRM token change");
            }
            Thread.sleep(100);
            allocator.schedule();
            currentToken = rm.getRMContext().getRMApps().get(appId)
                .getRMAppAttempt(appAttemptId).getAMRMToken();
          }

          return currentToken;
        }
      });

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(rmAddr), ugiToken.getService());
}
 
Example 27
Source Project: incubator-myriad   Source File: MockRMContext.java    License: Apache License 2.0 4 votes vote down vote up
public void setTokenSecretManager(AMRMTokenSecretManager tokenSecretManager) {
  this.tokenSecretManager = tokenSecretManager;
}
 
Example 28
Source Project: incubator-myriad   Source File: MockRMContext.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public AMRMTokenSecretManager getAMRMTokenSecretManager() {
  return tokenSecretManager;
}
 
Example 29
Source Project: hadoop   Source File: RMContext.java    License: Apache License 2.0 votes vote down vote up
AMRMTokenSecretManager getAMRMTokenSecretManager(); 
Example 30
Source Project: big-c   Source File: RMContext.java    License: Apache License 2.0 votes vote down vote up
AMRMTokenSecretManager getAMRMTokenSecretManager();