org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Utils.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
public static void waitForNMToRegister(NodeManager nm) throws Exception{
    NMTokenSecretManagerInNM nmTokenSecretManagerNM =
            nm.getNMContext().getNMTokenSecretManager();
    NMContainerTokenSecretManager containerTokenSecretManager = nm.getNMContext().getContainerTokenSecretManager();
    int attempt = 60;
    while(attempt-- > 0) {
        try {
            if (nmTokenSecretManagerNM.getCurrentKey() != null && containerTokenSecretManager.getCurrentKey() != null) {
                break;
            }
        } catch (Exception e) {

        }
        Thread.sleep(2000);
    }
}
 
Example #2
Source File: TestResourceLocalizationService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore,null);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 
Example #3
Source File: TestLocalCacheDirectoryManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testMinimumPerDirectoryFileLimit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1");
  Exception e = null;
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
  ResourceLocalizationService service =
      new ResourceLocalizationService(null, null, null, null, nmContext);
  try {
    service.init(conf);
  } catch (Exception e1) {
    e = e1;
  }
  Assert.assertNotNull(e);
  Assert.assertEquals(YarnRuntimeException.class, e.getClass());
  Assert.assertEquals(e.getMessage(),
    YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
        + " parameter is configured with a value less than 37.");

}
 
Example #4
Source File: TestLocalCacheDirectoryManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testMinimumPerDirectoryFileLimit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "1");
  Exception e = null;
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), new NMNullStateStoreService());
  ResourceLocalizationService service =
      new ResourceLocalizationService(null, null, null, null, nmContext);
  try {
    service.init(conf);
  } catch (Exception e1) {
    e = e1;
  }
  Assert.assertNotNull(e);
  Assert.assertEquals(YarnRuntimeException.class, e.getClass());
  Assert.assertEquals(e.getMessage(),
    YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY
        + " parameter is configured with a value less than 37.");

}
 
Example #5
Source File: TestResourceLocalizationService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 
Example #6
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void waitForNMToReceiveNMTokenKey(
    NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm)
    throws InterruptedException {
  int attempt = 60;
  ContainerManagerImpl cm =
      ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
  while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM
      .getNodeId() == null) && attempt-- > 0) {
    Thread.sleep(2000);
  }
}
 
Example #7
Source File: TestResourceLocalizationService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws IOException {
  conf = new Configuration();
  spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  lfs = FileContext.getFileContext(spylfs, conf);

  String logDir = lfs.makeQualified(new Path(basedir, "logdir ")).toString();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
  nmContext = new NMContext(new NMContainerTokenSecretManager(
    conf), new NMTokenSecretManagerInNM(), null,
    new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
}
 
Example #8
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
    NMStateStoreService stateStore, CoresManager coresManager) {
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;
  this.dirsHandler = dirsHandler;
  this.aclsManager = aclsManager;
  this.nodeHealthStatus.setIsNodeHealthy(true);
  this.nodeHealthStatus.setHealthReport("Healthy");
  this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
  this.stateStore = stateStore;
  this.coresManager = coresManager;
}
 
Example #9
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void recoverTokens(NMTokenSecretManagerInNM nmTokenSecretManager,
    NMContainerTokenSecretManager containerTokenSecretManager)
        throws IOException {
  if (nmStore.canRecover()) {
    nmTokenSecretManager.recover();
    containerTokenSecretManager.recover();
  }
}
 
Example #10
Source File: NodeManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected NMContext createNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    NMStateStoreService stateStore,
    CoresManager coresManager) {
  return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
      dirsHandler, aclsManager, stateStore,coresManager);
}
 
Example #11
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void rollNMTokenMasterKey(
    NMTokenSecretManagerInRM nmTokenSecretManagerRM,
    NMTokenSecretManagerInNM nmTokenSecretManagerNM) throws Exception {
  int oldKeyId = nmTokenSecretManagerRM.getCurrentKey().getKeyId();
  nmTokenSecretManagerRM.rollMasterKey();
  int interval = 40;
  while (nmTokenSecretManagerNM.getCurrentKey().getKeyId() == oldKeyId
      && interval-- > 0) {
    Thread.sleep(1000);
  }
  nmTokenSecretManagerRM.activateNextMasterKey();
  Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
      == nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
}
 
Example #12
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void waitForNMToReceiveNMTokenKey(
    NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm)
    throws InterruptedException {
  int attempt = 60;
  ContainerManagerImpl cm =
      ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
  while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM
      .getNodeId() == null) && attempt-- > 0) {
    Thread.sleep(2000);
  }
}
 
Example #13
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void recoverTokens(NMTokenSecretManagerInNM nmTokenSecretManager,
    NMContainerTokenSecretManager containerTokenSecretManager)
        throws IOException {
  if (nmStore.canRecover()) {
    nmTokenSecretManager.recover();
    containerTokenSecretManager.recover();
  }
}
 
Example #14
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void rollNMTokenMasterKey(
    NMTokenSecretManagerInRM nmTokenSecretManagerRM,
    NMTokenSecretManagerInNM nmTokenSecretManagerNM) throws Exception {
  int oldKeyId = nmTokenSecretManagerRM.getCurrentKey().getKeyId();
  nmTokenSecretManagerRM.rollMasterKey();
  int interval = 40;
  while (nmTokenSecretManagerNM.getCurrentKey().getKeyId() == oldKeyId
      && interval-- > 0) {
    Thread.sleep(1000);
  }
  nmTokenSecretManagerRM.activateNextMasterKey();
  Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
      == nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
}
 
Example #15
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected NMContext createNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    NMStateStoreService stateStore) {
  return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
      dirsHandler, aclsManager, stateStore);
}
 
Example #16
Source File: TestResourceLocalizationService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws IOException {
  conf = new Configuration();
  spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  lfs = FileContext.getFileContext(spylfs, conf);

  String logDir = lfs.makeQualified(new Path(basedir, "logdir ")).toString();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
  nmContext = new NMContext(new NMContainerTokenSecretManager(
    conf), new NMTokenSecretManagerInNM(), null,
    new ApplicationACLsManager(conf), new NMNullStateStoreService());
}
 
Example #17
Source File: NodeManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager,
    LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
    NMStateStoreService stateStore) {
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;
  this.dirsHandler = dirsHandler;
  this.aclsManager = aclsManager;
  this.nodeHealthStatus.setIsNodeHealthy(true);
  this.nodeHealthStatus.setHealthReport("Healthy");
  this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
  this.stateStore = stateStore;
}
 
Example #18
Source File: NodeManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public NMTokenSecretManagerInNM getNMTokenSecretManager() {
  return this.nmTokenSecretManager;
}
 
Example #19
Source File: TestApplication.java    From hadoop with Apache License 2.0 4 votes vote down vote up
WrappedApplication(int id, long timestamp, String user, int numContainers) {
  Configuration conf = new Configuration();
  
  dispatcher = new DrainDispatcher();
  containerTokenIdentifierMap =
      new HashMap<ContainerId, ContainerTokenIdentifier>();
  dispatcher.init(conf);

  localizerBus = mock(EventHandler.class);
  launcherBus = mock(EventHandler.class);
  monitorBus = mock(EventHandler.class);
  auxBus = mock(EventHandler.class);
  containerBus = mock(EventHandler.class);
  logAggregationBus = mock(EventHandler.class);

  dispatcher.register(LocalizationEventType.class, localizerBus);
  dispatcher.register(ContainersLauncherEventType.class, launcherBus);
  dispatcher.register(ContainersMonitorEventType.class, monitorBus);
  dispatcher.register(AuxServicesEventType.class, auxBus);
  dispatcher.register(ContainerEventType.class, containerBus);
  dispatcher.register(LogHandlerEventType.class, logAggregationBus);

  nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class);

  context = mock(Context.class);
  
  when(context.getContainerTokenSecretManager()).thenReturn(
    new NMContainerTokenSecretManager(conf));
  when(context.getApplicationACLsManager()).thenReturn(
    new ApplicationACLsManager(conf));
  when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
  
  // Setting master key
  MasterKey masterKey = new MasterKeyPBImpl();
  masterKey.setKeyId(123);
  masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123)
    .byteValue()) }));
  context.getContainerTokenSecretManager().setMasterKey(masterKey);
  
  this.user = user;
  this.appId = BuilderUtils.newApplicationId(timestamp, id);

  app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
  containers = new ArrayList<Container>();
  for (int i = 0; i < numContainers; i++) {
    Container container = createMockedContainer(this.appId, i);
    containers.add(container);
    long currentTime = System.currentTimeMillis();
    ContainerTokenIdentifier identifier =
        new ContainerTokenIdentifier(container.getContainerId(), "", "",
          null, currentTime + 2000, masterKey.getKeyId(), currentTime,
          Priority.newInstance(0), 0);
    containerTokenIdentifierMap
      .put(identifier.getContainerID(), identifier);
    context.getContainerTokenSecretManager().startContainerSuccessful(
      identifier);
    Assert.assertFalse(context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(identifier));
  }

  dispatcher.start();
}
 
Example #20
Source File: TestContainerLauncher.java    From big-c with Apache License 2.0 4 votes vote down vote up
public MRAppWithSlowNM(NMTokenSecretManagerInNM tokenSecretManager) {
  super(1, 0, false, "TestContainerLauncher", true);
  this.tokenSecretManager = tokenSecretManager;
}
 
Example #21
Source File: TestContainerLauncher.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout = 15000)
public void testSlowNM() throws Exception {

  conf = new Configuration();
  int maxAttempts = 1;
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts);
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);
  conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  NMTokenSecretManagerInNM tokenSecretManager =
      new NMTokenSecretManagerInNM();
  MasterKey masterKey = Records.newRecord(MasterKey.class);
  masterKey.setBytes(ByteBuffer.wrap("key".getBytes()));
  tokenSecretManager.setMasterKey(masterKey);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "token");
  server =
      rpc.getServer(ContainerManagementProtocol.class,
        new DummyContainerManager(), addr, conf, tokenSecretManager, 1);
  server.start();

  MRApp app = new MRAppWithSlowNM(tokenSecretManager);

  try {
  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);

  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());

  Task task = tasks.values().iterator().next();
  app.waitForState(task, TaskState.SCHEDULED);

  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator()
      .next().getAttempts();
    Assert.assertEquals("Num attempts is not correct", maxAttempts,
        attempts.size());

  TaskAttempt attempt = attempts.values().iterator().next();
    app.waitForInternalState((TaskAttemptImpl) attempt,
        TaskAttemptStateInternal.ASSIGNED);

  app.waitForState(job, JobState.FAILED);

  String diagnostics = attempt.getDiagnostics().toString();
  LOG.info("attempt.getDiagnostics: " + diagnostics);

    Assert.assertTrue(diagnostics.contains("Container launch failed for "
        + "container_0_0000_01_000000 : "));
    Assert
        .assertTrue(diagnostics
            .contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"));

  } finally {
    server.stop();
  app.stop();
}
}
 
Example #22
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 4 votes vote down vote up
public MyNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager) {
  super(containerTokenSecretManager, nmTokenSecretManager, null, null,
      new NMNullStateStoreService(), null);
}
 
Example #23
Source File: TestNodeStatusUpdater.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Test completed containerStatus get back up when heart beat lost, and will
 * be sent via next heart beat.
 */
@Test(timeout = 200000)
public void testCompletedContainerStatusBackup() throws Exception {
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater2 myNodeStatusUpdater =
          new MyNodeStatusUpdater2(context, dispatcher, healthChecker,
              metrics);
      return myNodeStatusUpdater;
    }

    //@Override
    protected NMContext createNMContext(
        NMContainerTokenSecretManager containerTokenSecretManager,
        NMTokenSecretManagerInNM nmTokenSecretManager,
        NMStateStoreService store) {
      return new MyNMContext(containerTokenSecretManager,
        nmTokenSecretManager);
    }
  };

  YarnConfiguration conf = createNMConfig();
  nm.init(conf);
  nm.start();

  int waitCount = 0;
  while (heartBeatID <= 4 && waitCount++ != 20) {
    Thread.sleep(500);
  }
  if (heartBeatID <= 4) {
    Assert.fail("Failed to get all heartbeats in time, " +
        "heartbeatID:" + heartBeatID);
  }
  if(assertionFailedInThread.get()) {
    Assert.fail("ContainerStatus Backup failed");
  }
  Assert.assertNotNull(nm.getNMContext().getSystemCredentialsForApps()
    .get(ApplicationId.newInstance(1234, 1)).getToken(new Text("token1")));
  nm.stop();
}
 
Example #24
Source File: TestApplication.java    From big-c with Apache License 2.0 4 votes vote down vote up
WrappedApplication(int id, long timestamp, String user, int numContainers) {
  Configuration conf = new Configuration();
  
  dispatcher = new DrainDispatcher();
  containerTokenIdentifierMap =
      new HashMap<ContainerId, ContainerTokenIdentifier>();
  dispatcher.init(conf);

  localizerBus = mock(EventHandler.class);
  launcherBus = mock(EventHandler.class);
  monitorBus = mock(EventHandler.class);
  auxBus = mock(EventHandler.class);
  containerBus = mock(EventHandler.class);
  logAggregationBus = mock(EventHandler.class);

  dispatcher.register(LocalizationEventType.class, localizerBus);
  dispatcher.register(ContainersLauncherEventType.class, launcherBus);
  dispatcher.register(ContainersMonitorEventType.class, monitorBus);
  dispatcher.register(AuxServicesEventType.class, auxBus);
  dispatcher.register(ContainerEventType.class, containerBus);
  dispatcher.register(LogHandlerEventType.class, logAggregationBus);

  nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class);

  context = mock(Context.class);
  
  when(context.getContainerTokenSecretManager()).thenReturn(
    new NMContainerTokenSecretManager(conf));
  when(context.getApplicationACLsManager()).thenReturn(
    new ApplicationACLsManager(conf));
  when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
  
  // Setting master key
  MasterKey masterKey = new MasterKeyPBImpl();
  masterKey.setKeyId(123);
  masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123)
    .byteValue()) }));
  context.getContainerTokenSecretManager().setMasterKey(masterKey);
  
  this.user = user;
  this.appId = BuilderUtils.newApplicationId(timestamp, id);

  app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
  containers = new ArrayList<Container>();
  for (int i = 0; i < numContainers; i++) {
    Container container = createMockedContainer(this.appId, i);
    containers.add(container);
    long currentTime = System.currentTimeMillis();
    ContainerTokenIdentifier identifier =
        new ContainerTokenIdentifier(container.getContainerId(), "", "",
          null, currentTime + 2000, masterKey.getKeyId(), currentTime,
          Priority.newInstance(0), 0);
    containerTokenIdentifierMap
      .put(identifier.getContainerID(), identifier);
    context.getContainerTokenSecretManager().startContainerSuccessful(
      identifier);
    Assert.assertFalse(context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(identifier));
  }

  dispatcher.start();
}
 
Example #25
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * This tests a malice user getting a proper token but then messing with it by
 * tampering with containerID/Resource etc.. His/her containers should be
 * rejected.
 * 
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerToken(Configuration conf) throws IOException,
    InterruptedException, YarnException {

  LOG.info("Running test for malice user");
  /*
   * We need to check for containerToken (authorization).
   * Here we will be assuming that we have valid NMToken  
   * 1) ContainerToken used is expired.
   * 2) ContainerToken is tampered (resource is modified).
   */
  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
        .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";
  
  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();
  
  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
  
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  
  Resource r = Resource.newInstance(1230, 2, 2);
  
  Token containerToken = 
      containerTokenSecretManager.createContainerToken(
          cId, nodeId, user, r, Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier = 
      getContainerTokenIdentifierFromToken(containerToken);
  
  // Verify new compatible version ContainerTokenIdentifier can work successfully.
  ContainerTokenIdentifierForTest newVersionTokenIdentifier = 
      new ContainerTokenIdentifierForTest(containerTokenIdentifier, "message");
  byte[] password = 
      containerTokenSecretManager.createPassword(newVersionTokenIdentifier);
  
  Token newContainerToken = BuilderUtils.newContainerToken(
      nodeId, password, newVersionTokenIdentifier);
  
  Token nmToken =
          nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
  YarnRPC rpc = YarnRPC.create(conf);
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      newContainerToken, nmToken, false).isEmpty());
  
  // Creating a tampered Container Token
  RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  tamperedContainerTokenSecretManager.rollMasterKey();
  do {
    tamperedContainerTokenSecretManager.rollMasterKey();
    tamperedContainerTokenSecretManager.activateNextMasterKey();
  } while (containerTokenSecretManager.getCurrentKey().getKeyId()
      == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
  
  ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
  // Creating modified containerToken
  Token containerToken2 =
      tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId,
          user, r, Priority.newInstance(0), 0);
  
  StringBuilder sb = new StringBuilder("Given Container ");
  sb.append(cId2);
  sb.append(" seems to have an illegally generated token.");
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      containerToken2, nmToken, true).contains(sb.toString()));
}
 
Example #26
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * This tests whether a containerId is serialized/deserialized with epoch.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerTokenWithEpoch(Configuration conf)
    throws IOException, InterruptedException, YarnException {

  LOG.info("Running test for serializing/deserializing containerIds");

  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
          .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";

  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();

  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());

  // Creating a normal Container Token
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  Resource r = Resource.newInstance(1230, 2);
  Token containerToken =
      containerTokenSecretManager.createContainerToken(cId, nodeId, user, r,
          Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier();
  byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
  containerTokenIdentifier.readFields(dib);
  
  
  Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
  Assert.assertEquals(
      cId.toString(), containerTokenIdentifier.getContainerID().toString());

  Token nmToken =
      nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);

  YarnRPC rpc = YarnRPC.create(conf);
  testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken,
      false);

  List<ContainerId> containerIds = new LinkedList<ContainerId>();
  containerIds.add(cId);
  ContainerManagementProtocol proxy
      = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
  GetContainerStatusesResponse res = proxy.getContainerStatuses(
      GetContainerStatusesRequest.newInstance(containerIds));
  Assert.assertNotNull(res.getContainerStatuses().get(0));
  Assert.assertEquals(
      cId, res.getContainerStatuses().get(0).getContainerId());
  Assert.assertEquals(cId.toString(),
      res.getContainerStatuses().get(0).getContainerId().toString());
}
 
Example #27
Source File: TestContainerLauncher.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public MRAppWithSlowNM(NMTokenSecretManagerInNM tokenSecretManager) {
  super(1, 0, false, "TestContainerLauncher", true);
  this.tokenSecretManager = tokenSecretManager;
}
 
Example #28
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Test completed containerStatus get back up when heart beat lost, and will
 * be sent via next heart beat.
 */
@Test(timeout = 200000)
public void testCompletedContainerStatusBackup() throws Exception {
  nm = new NodeManager() {
    @Override
    protected NodeStatusUpdater createNodeStatusUpdater(Context context,
        Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
      MyNodeStatusUpdater2 myNodeStatusUpdater =
          new MyNodeStatusUpdater2(context, dispatcher, healthChecker,
              metrics);
      return myNodeStatusUpdater;
    }

    @Override
    protected NMContext createNMContext(
        NMContainerTokenSecretManager containerTokenSecretManager,
        NMTokenSecretManagerInNM nmTokenSecretManager,
        NMStateStoreService store) {
      return new MyNMContext(containerTokenSecretManager,
        nmTokenSecretManager);
    }
  };

  YarnConfiguration conf = createNMConfig();
  nm.init(conf);
  nm.start();

  int waitCount = 0;
  while (heartBeatID <= 4 && waitCount++ != 20) {
    Thread.sleep(500);
  }
  if (heartBeatID <= 4) {
    Assert.fail("Failed to get all heartbeats in time, " +
        "heartbeatID:" + heartBeatID);
  }
  if(assertionFailedInThread.get()) {
    Assert.fail("ContainerStatus Backup failed");
  }
  Assert.assertNotNull(nm.getNMContext().getSystemCredentialsForApps()
    .get(ApplicationId.newInstance(1234, 1)).getToken(new Text("token1")));
  nm.stop();
}
 
Example #29
Source File: NodeManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {

  conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);

  rmWorkPreservingRestartEnabled = conf.getBoolean(YarnConfiguration
          .RM_WORK_PRESERVING_RECOVERY_ENABLED,
      YarnConfiguration.DEFAULT_RM_WORK_PRESERVING_RECOVERY_ENABLED);

  initAndStartRecoveryStore(conf);

  NMContainerTokenSecretManager containerTokenSecretManager =
      new NMContainerTokenSecretManager(conf, nmStore);

  NMTokenSecretManagerInNM nmTokenSecretManager =
      new NMTokenSecretManagerInNM(nmStore);

  recoverTokens(nmTokenSecretManager, containerTokenSecretManager);
  
  this.aclsManager = new ApplicationACLsManager(conf);

  ContainerExecutor exec = ReflectionUtils.newInstance(
      conf.getClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
        DefaultContainerExecutor.class, ContainerExecutor.class), conf);
  try {
    exec.init();
  } catch (IOException e) {
    throw new YarnRuntimeException("Failed to initialize container executor", e);
  }    
  DeletionService del = createDeletionService(exec);
  addService(del);

  // NodeManager level dispatcher
  this.dispatcher = new AsyncDispatcher();
  
  this.coresManager = new CoresManagerImpl();
  this.coresManager.init(conf);
  
  nodeHealthChecker = new NodeHealthCheckerService();
  addService(nodeHealthChecker);
  dirsHandler = nodeHealthChecker.getDiskHandler();

  this.context = createNMContext(containerTokenSecretManager,
      nmTokenSecretManager, nmStore, coresManager);
  
  nodeStatusUpdater =
      createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);

  NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
  addService(nodeResourceMonitor);

  containerManager =
      createContainerManager(context, exec, del, nodeStatusUpdater,
      this.aclsManager, dirsHandler);
  addService(containerManager);
  ((NMContext) context).setContainerManager(containerManager);

  WebServer webServer = createWebServer(context, containerManager
      .getContainersMonitor(), this.aclsManager, dirsHandler);
  addService(webServer);
  ((NMContext) context).setWebServer(webServer);

  dispatcher.register(ContainerManagerEventType.class, containerManager);
  dispatcher.register(NodeManagerEventType.class, this);
  addService(dispatcher);
  
  DefaultMetricsSystem.initialize("NodeManager");

  // StatusUpdater should be added last so that it get started last 
  // so that we make sure everything is up before registering with RM. 
  addService(nodeStatusUpdater);
  
  super.serviceInit(conf);
  // TODO add local dirs to del
}
 
Example #30
Source File: TestNodeStatusUpdater.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public MyNMContext(
    NMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInNM nmTokenSecretManager) {
  super(containerTokenSecretManager, nmTokenSecretManager, null, null,
      new NMNullStateStoreService());
}