org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #2
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
Example #3
Source File: TestAppManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
Example #4
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #5
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
Example #6
Source File: TestAppManager.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
Example #7
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
Example #8
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
Example #9
Source File: TestZKRMStateStorePerf.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
Example #10
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
Example #11
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
Example #12
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
Example #13
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
Example #14
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public Token<ClientToAMTokenIdentifier> createClientToken(String client) {
  this.readLock.lock();

  try {
    Token<ClientToAMTokenIdentifier> token = null;
    ClientToAMTokenSecretManagerInRM secretMgr =
        this.rmContext.getClientToAMTokenSecretManager();
    if (client != null &&
        secretMgr.getMasterKey(this.applicationAttemptId) != null) {
      token = new Token<ClientToAMTokenIdentifier>(
          new ClientToAMTokenIdentifier(this.applicationAttemptId, client),
          secretMgr);
    }
    return token;
  } finally {
    this.readLock.unlock();
  }
}
Example #15
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #16
Source File: TestZKRMStateStorePerf.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void initStore(String hostPort) {
  Optional<String> optHostPort = Optional.fromNullable(hostPort);
  RMContext rmContext = mock(RMContext.class);

  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, optHostPort.or(this.hostPort));
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);

  store = new ZKRMStateStore();
  store.init(conf);
  store.start();
  when(rmContext.getStateStore()).thenReturn(store);
  appTokenMgr = new AMRMTokenSecretManager(conf, rmContext);
  appTokenMgr.start();
  clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM();
}
Example #17
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #18
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public Token<ClientToAMTokenIdentifier> createClientToken(String client) {
  this.readLock.lock();

  try {
    Token<ClientToAMTokenIdentifier> token = null;
    ClientToAMTokenSecretManagerInRM secretMgr =
        this.rmContext.getClientToAMTokenSecretManager();
    if (client != null &&
        secretMgr.getMasterKey(this.applicationAttemptId) != null) {
      token = new Token<ClientToAMTokenIdentifier>(
          new ClientToAMTokenIdentifier(this.applicationAttemptId, client),
          secretMgr);
    }
    return token;
  } finally {
    this.readLock.unlock();
  }
}
Example #19
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabelsInherit() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));

  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabelsInheritConfig(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #20
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #21
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #22
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
Example #23
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
Example #24
Source File: TestQueueParsing.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #25
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);
  csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
      "red", 24);
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
      ImmutableSet.of(RMNodeLabelsManager.ANY));
  csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
      ImmutableSet.of(RMNodeLabelsManager.ANY));

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
Example #26
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
Example #27
Source File: TestQueueParsing.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IOException.class)
public void testQueueParsingWithMoveQueue()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  csConf.setQueues("root", new String[] { "a" });
  csConf.setQueues("root.a", new String[] { "x", "y" });
  csConf.setCapacity("root.a", 100);
  csConf.setCapacity("root.a.x", 50);
  csConf.setCapacity("root.a.y", 50);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  
  csConf.setQueues("root", new String[] { "a", "x" });
  csConf.setQueues("root.a", new String[] { "y" });
  csConf.setCapacity("root.x", 50);
  csConf.setCapacity("root.a", 50);
  csConf.setCapacity("root.a.y", 100);
  
  capacityScheduler.reinitialize(csConf, rmContext);
}
Example #28
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 30000)
public void testRMAppSubmitMaxAppAttempts() throws Exception {
  int[] globalMaxAppAttempts = new int[] { 10, 1 };
  int[][] individualMaxAppAttempts = new int[][]{
      new int[]{ 9, 10, 11, 0 },
      new int[]{ 1, 10, 0, -1 }};
  int[][] expectedNums = new int[][]{
      new int[]{ 9, 10, 10, 10 },
      new int[]{ 1, 1, 1, 1 }};
  for (int i = 0; i < globalMaxAppAttempts.length; ++i) {
    for (int j = 0; j < individualMaxAppAttempts.length; ++j) {
      ResourceScheduler scheduler = mockResourceScheduler();
      Configuration conf = new Configuration();
      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, globalMaxAppAttempts[i]);
      ApplicationMasterService masterService =
          new ApplicationMasterService(rmContext, scheduler);
      TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
          new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
          new ApplicationACLsManager(conf), conf);

      ApplicationId appID = MockApps.newAppID(i * 4 + j + 1);
      asContext.setApplicationId(appID);
      if (individualMaxAppAttempts[i][j] != 0) {
        asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
      }
      appMonitor.submitApplication(asContext, "test");
      RMApp app = rmContext.getRMApps().get(appID);
      Assert.assertEquals("max application attempts doesn't match",
          expectedNums[i][j], app.getMaxAppAttempts());

      // wait for event to be processed
      int timeoutSecs = 0;
      while ((getAppEventType() == RMAppEventType.KILL) &&
          timeoutSecs++ < 20) {
        Thread.sleep(1000);
      }
      setAppEventType(RMAppEventType.KILL);
    }
  }
}
Example #29
Source File: RMActiveServiceContext.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
Example #30
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}