Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Token<ClientToAMTokenIdentifier> createClientToken(String client) {
  this.readLock.lock();

  try {
    Token<ClientToAMTokenIdentifier> token = null;
    ClientToAMTokenSecretManagerInRM secretMgr =
        this.rmContext.getClientToAMTokenSecretManager();
    if (client != null &&
        secretMgr.getMasterKey(this.applicationAttemptId) != null) {
      token = new Token<ClientToAMTokenIdentifier>(
          new ClientToAMTokenIdentifier(this.applicationAttemptId, client),
          secretMgr);
    }
    return token;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 2
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 3
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
 
Example 4
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example 5
Source Project: hadoop   Source File: TestRMWebApp.java    License: Apache License 2.0 6 votes vote down vote up
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 
Example 6
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
 
Example 7
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
 
Example 8
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 9
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 10
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 11
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 12
Source Project: hadoop   Source File: TestZKRMStateStorePerf.java    License: Apache License 2.0 6 votes vote down vote up
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
 
Example 13
Source Project: big-c   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Token<ClientToAMTokenIdentifier> createClientToken(String client) {
  this.readLock.lock();

  try {
    Token<ClientToAMTokenIdentifier> token = null;
    ClientToAMTokenSecretManagerInRM secretMgr =
        this.rmContext.getClientToAMTokenSecretManager();
    if (client != null &&
        secretMgr.getMasterKey(this.applicationAttemptId) != null) {
      token = new Token<ClientToAMTokenIdentifier>(
          new ClientToAMTokenIdentifier(this.applicationAttemptId, client),
          secretMgr);
    }
    return token;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 14
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 15
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
 
Example 16
Source Project: big-c   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example 17
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 6 votes vote down vote up
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 
Example 18
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
 
Example 19
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
 
Example 20
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 21
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 22
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 23
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 24
Source Project: big-c   Source File: TestZKRMStateStorePerf.java    License: Apache License 2.0 6 votes vote down vote up
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
 
Example 25
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
 
Example 26
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitMaxAppAttempts() throws Exception {
  int[] globalMaxAppAttempts = new int[] { 10, 1 };
  int[][] individualMaxAppAttempts = new int[][]{
      new int[]{ 9, 10, 11, 0 },
      new int[]{ 1, 10, 0, -1 }};
  int[][] expectedNums = new int[][]{
      new int[]{ 9, 10, 10, 10 },
      new int[]{ 1, 1, 1, 1 }};
  for (int i = 0; i < globalMaxAppAttempts.length; ++i) {
    for (int j = 0; j < individualMaxAppAttempts.length; ++j) {
      ResourceScheduler scheduler = mockResourceScheduler();
      Configuration conf = new Configuration();
      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, globalMaxAppAttempts[i]);
      ApplicationMasterService masterService =
          new ApplicationMasterService(rmContext, scheduler);
      TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
          new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
          new ApplicationACLsManager(conf), conf);

      ApplicationId appID = MockApps.newAppID(i * 4 + j + 1);
      asContext.setApplicationId(appID);
      if (individualMaxAppAttempts[i][j] != 0) {
        asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
      }
      appMonitor.submitApplication(asContext, "test");
      RMApp app = rmContext.getRMApps().get(appID);
      Assert.assertEquals("max application attempts doesn't match",
          expectedNums[i][j], app.getMaxAppAttempts());

      // wait for event to be processed
      int timeoutSecs = 0;
      while ((getAppEventType() == RMAppEventType.KILL) &&
          timeoutSecs++ < 20) {
        Thread.sleep(1000);
      }
      setAppEventType(RMAppEventType.KILL);
    }
  }
}
 
Example 27
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
 
Example 28
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(csConf);
  cs.start();
  cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
    null, null, new RMContainerTokenSecretManager(csConf),
    new NMTokenSecretManagerInRM(csConf),
    new ClientToAMTokenSecretManagerInRM(), null));

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));

  Assert.assertEquals(6 * GB, cs.getClusterResource().getMemory());

  // reconnect n1 with downgraded memory
  n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory());
  cs.stop();
}
 
Example 29
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 30
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}