Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 2
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 3
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
 
Example 4
Source Project: hadoop   Source File: TestRMWebApp.java    License: Apache License 2.0 6 votes vote down vote up
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 
Example 5
Source Project: hadoop   Source File: MockRM.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ResourceTrackerService createResourceTrackerService() {

  RMContainerTokenSecretManager containerTokenSecretManager =
      getRMContext().getContainerTokenSecretManager();
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      getRMContext().getNMTokenSecretManager();
  nmTokenSecretManager.rollMasterKey();
  return new ResourceTrackerService(getRMContext(), nodesListManager,
      this.nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager) {

    @Override
    protected void serviceStart() {
      // override to not start rpc handler
    }

    @Override
    protected void serviceStop() {
      // don't do anything
    }
  };
}
 
Example 6
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
 
Example 7
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
 
Example 8
Source Project: hadoop   Source File: TestContainerAllocation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RMContainerTokenSecretManager createContainerTokenSecretManager(
    Configuration conf) {
  return new RMContainerTokenSecretManager(conf) {

    @Override
    public Token createContainerToken(ContainerId containerId,
        NodeId nodeId, String appSubmitter, Resource capability,
        Priority priority, long createTime,
        LogAggregationContext logAggregationContext) {
      numRetries++;
      return super.createContainerToken(containerId, nodeId, appSubmitter,
        capability, priority, createTime, logAggregationContext);
    }
  };
}
 
Example 9
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWithLabels() throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  checkQueueLabels(capacityScheduler);
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 10
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 11
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 12
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 13
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter) {
  this(
    rmDispatcher,
    containerAllocationExpirer,
    amLivelinessMonitor,
    amFinishingMonitor,
    delegationTokenRenewer,
    appTokenSecretManager,
    containerTokenSecretManager,
    nmTokenSecretManager,
    clientToAMTokenSecretManager,
    rmApplicationHistoryWriter,
    null);
}
 
Example 14
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 6 votes vote down vote up
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 
Example 15
Source Project: big-c   Source File: MockRM.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected ResourceTrackerService createResourceTrackerService() {

  RMContainerTokenSecretManager containerTokenSecretManager =
      getRMContext().getContainerTokenSecretManager();
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      getRMContext().getNMTokenSecretManager();
  nmTokenSecretManager.rollMasterKey();
  return new ResourceTrackerService(getRMContext(), nodesListManager,
      this.nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager) {

    @Override
    protected void serviceStart() {
      // override to not start rpc handler
    }

    @Override
    protected void serviceStop() {
      // don't do anything
    }
  };
}
 
Example 16
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test init a queue configuration, children's capacity for a given label
 * doesn't equals to 100%. This expect IllegalArgumentException thrown.
 */
@Test(expected = IllegalArgumentException.class)
public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
    throws IOException {
  nodeLabelManager.addToCluserNodeLabels(ImmutableSet
      .of("red", "blue"));
  
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfiguration(csConf);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
}
 
Example 17
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 18
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRefreshQueues() throws Exception {
  CapacityScheduler cs = new CapacityScheduler();
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  setupQueueConfiguration(conf);
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(conf);
  cs.start();
  cs.reinitialize(conf, rmContext);
  checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);

  conf.setCapacity(A, 80f);
  conf.setCapacity(B, 20f);
  cs.reinitialize(conf, mockContext);
  checkQueueCapacities(cs, 80f, 20f);
  cs.stop();
}
 
Example 19
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
/** Test that parseQueue throws an exception when two leaf queues have the
  *  same name
* @throws IOException
  */
 @Test(expected=IOException.class)
 public void testParseQueue() throws IOException {
   CapacityScheduler cs = new CapacityScheduler();
   cs.setConf(new YarnConfiguration());
   cs.setRMContext(resourceManager.getRMContext());
   CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
   setupQueueConfiguration(conf);
   cs.init(conf);
   cs.start();

   conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a.a1", new String[] {"b1"} );
   conf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);
   conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + ".a.a1.b1", 100.0f);

   cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
     null, new RMContainerTokenSecretManager(conf),
     new NMTokenSecretManagerInRM(conf),
     new ClientToAMTokenSecretManagerInRM(), null));
 }
 
Example 20
Source Project: big-c   Source File: TestContainerAllocation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RMContainerTokenSecretManager createContainerTokenSecretManager(
    Configuration conf) {
  return new RMContainerTokenSecretManager(conf) {

    @Override
    public Token createContainerToken(ContainerId containerId,
        NodeId nodeId, String appSubmitter, Resource capability,
        Priority priority, long createTime,
        LogAggregationContext logAggregationContext) {
      numRetries++;
      return super.createContainerToken(containerId, nodeId, appSubmitter,
        capability, priority, createTime, logAggregationContext);
    }
  };
}
 
Example 21
Source Project: hadoop   Source File: ResourceTrackerService.java    License: Apache License 2.0 5 votes vote down vote up
public ResourceTrackerService(RMContext rmContext,
    NodesListManager nodesListManager,
    NMLivelinessMonitor nmLivelinessMonitor,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager) {
  super(ResourceTrackerService.class.getName());
  this.rmContext = rmContext;
  this.nodesListManager = nodesListManager;
  this.nmLivelinessMonitor = nmLivelinessMonitor;
  this.containerTokenSecretManager = containerTokenSecretManager;
  this.nmTokenSecretManager = nmTokenSecretManager;

}
 
Example 22
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
 
Example 23
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IOException.class)
public void testQueueParsingWithMoveQueue()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  csConf.setQueues("root", new String[] { "a" });
  csConf.setQueues("root.a", new String[] { "x", "y" });
  csConf.setCapacity("root.a", 100);
  csConf.setCapacity("root.a.x", 50);
  csConf.setCapacity("root.a.y", 50);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(nodeLabelManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  
  csConf.setQueues("root", new String[] { "a", "x" });
  csConf.setQueues("root.a", new String[] { "y" });
  csConf.setCapacity("root.x", 50);
  csConf.setCapacity("root.a", 50);
  csConf.setCapacity("root.a.y", 100);
  
  capacityScheduler.reinitialize(csConf, rmContext);
}
 
Example 24
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
 
Example 25
Source Project: hadoop   Source File: TestRMNMRPCResponseId.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {
    @Override
    public void handle(Event event) {
      ; // ignore
    }
  });
  RMContext context =
      new RMContextImpl(dispatcher, null, null, null, null,
        null, new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf), null, null);
  dispatcher.register(RMNodeEventType.class,
      new ResourceManager.NodeEventDispatcher(context));
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  
  context.getContainerTokenSecretManager().rollMasterKey();
  context.getNMTokenSecretManager().rollMasterKey();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, new NMLivelinessMonitor(dispatcher),
      context.getContainerTokenSecretManager(),
      context.getNMTokenSecretManager());
  resourceTrackerService.init(conf);
}
 
Example 26
Source Project: hadoop   Source File: TestNMReconnect.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();

  dispatcher.register(RMNodeEventType.class,
      new TestRMNodeEventDispatcher());

  RMContext context = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, null);
  dispatcher.register(SchedulerEventType.class,
      new InlineDispatcher.EmptyEventHandler());
  dispatcher.register(RMNodeEventType.class,
      new NodeEventDispatcher(context));
  NMLivelinessMonitor nmLivelinessMonitor = new NMLivelinessMonitor(
      dispatcher);
  nmLivelinessMonitor.init(conf);
  nmLivelinessMonitor.start();
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.start();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.start();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, nmLivelinessMonitor, containerTokenSecretManager,
      nmTokenSecretManager);
  
  resourceTrackerService.init(conf);
  resourceTrackerService.start();
}
 
Example 27
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(csConf);
  cs.start();
  cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
    null, null, new RMContainerTokenSecretManager(csConf),
    new NMTokenSecretManagerInRM(csConf),
    new ClientToAMTokenSecretManagerInRM(), null));

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));

  Assert.assertEquals(6 * GB, cs.getClusterResource().getMemory());

  // reconnect n1 with downgraded memory
  n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory());
  cs.stop();
}
 
Example 28
Source Project: big-c   Source File: TestQueueParsing.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithSingleLevel(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 29
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabels(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}
 
Example 30
Source Project: hadoop   Source File: TestQueueParsing.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
    throws IOException {
  YarnConfiguration conf = new YarnConfiguration();
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration(conf);
  setupQueueConfigurationWithLabelsInherit(csConf);

  CapacityScheduler capacityScheduler = new CapacityScheduler();
  RMContextImpl rmContext =
      new RMContextImpl(null, null, null, null, null, null,
          new RMContainerTokenSecretManager(csConf),
          new NMTokenSecretManagerInRM(csConf),
          new ClientToAMTokenSecretManagerInRM(), null);
  
  RMNodeLabelsManager nodeLabelsManager = new NullRMNodeLabelsManager();
  nodeLabelsManager.init(conf);
  nodeLabelsManager.start();
  
  rmContext.setNodeLabelManager(nodeLabelsManager);
  capacityScheduler.setConf(csConf);
  capacityScheduler.setRMContext(rmContext);
  capacityScheduler.init(csConf);
  capacityScheduler.start();
  ServiceOperations.stopQuietly(capacityScheduler);
  ServiceOperations.stopQuietly(nodeLabelsManager);
}