Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example 2
Source Project: hadoop   Source File: RMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example 3
Source Project: hadoop   Source File: RmController.java    License: Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example 4
Source Project: hadoop   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 5
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example 6
Source Project: big-c   Source File: RmController.java    License: Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example 7
Source Project: big-c   Source File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example 8
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAsyncScheduling() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  final int NODES = 100;
  
  // Register nodes
  for (int i=0; i < NODES; ++i) {
    String host = "192.168.1." + i;
    RMNode node =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
  }
  
  // Now directly exercise the scheduling loop
  for (int i=0; i < NODES; ++i) {
    CapacityScheduler.schedule(cs);
  }
}
 
Example 9
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example 10
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal);
  rmconf.set("yarn.resourcemanager.keytab",
      spnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example 11
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
 
Example 12
Source Project: Flink-CEPplus   Source File: YARNSessionFIFOITCase.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo");
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example 13
Source Project: Flink-CEPplus   Source File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	storageDir = FOLDER.newFolder().getAbsolutePath();

	// startYARNWithConfig should be implemented by subclass
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example 14
Source Project: big-c   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRm(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockFairScheduler();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example 15
Source Project: big-c   Source File: TestFairSchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example 16
Source Project: big-c   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager();
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      FifoScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
}
 
Example 17
Source Project: hadoop   Source File: RMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@GET
@Path("/nodes/{nodeId}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeInfo getNode(@PathParam("nodeId") String nodeId) {
  init();
  if (nodeId == null || nodeId.isEmpty()) {
    throw new NotFoundException("nodeId, " + nodeId + ", is empty or null");
  }
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  NodeId nid = ConverterUtils.toNodeId(nodeId);
  RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
  boolean isInactive = false;
  if (ni == null) {
    ni = this.rm.getRMContext().getInactiveRMNodes().get(nid.getHost());
    if (ni == null) {
      throw new NotFoundException("nodeId, " + nodeId + ", is not found");
    }
    isInactive = true;
  }
  NodeInfo nodeInfo = new NodeInfo(ni, sched);
  if (isInactive) {
    nodeInfo.setNodeHTTPAddress(EMPTY);
  }
  return nodeInfo;
}
 
Example 18
Source Project: hadoop   Source File: ClusterMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.reservedGpuCores = metrics.getReservedGpuCores();
  this.availableGpuCores = metrics.getAvailableGpuCores();
  this.allocatedGpuCores = metrics.getAllocatedGpuCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.totalGpuCores = availableGpuCores + allocatedGpuCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 
Example 19
Source Project: hadoop   Source File: SchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example 20
Source Project: hadoop   Source File: NodeInfo.java    License: Apache License 2.0 5 votes vote down vote up
public NodeInfo(RMNode ni, ResourceScheduler sched) {
  NodeId id = ni.getNodeID();
  SchedulerNodeReport report = sched.getNodeReport(id);
  this.numContainers = 0;
  this.usedMemoryMB = 0;
  this.availMemoryMB = 0;
  if (report != null) {
    this.numContainers = report.getNumContainers();
    this.usedMemoryMB = report.getUsedResource().getMemory();
    this.availMemoryMB = report.getAvailableResource().getMemory();
    this.usedVirtualCores = report.getUsedResource().getVirtualCores();
    this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
    this.usedGpuCores = report.getUsedResource().getGpuCores();
    this.availableGpuCores = report.getAvailableResource().getGpuCores();
  }
  this.id = id.toString();
  this.rack = ni.getRackName();
  this.nodeHostName = ni.getHostName();
  this.state = ni.getState();
  this.nodeHTTPAddress = ni.getHttpAddress();
  this.lastHealthUpdate = ni.getLastHealthReportTime();
  this.healthReport = String.valueOf(ni.getHealthReport());
  this.version = ni.getNodeManagerVersion();
  
  // add labels
  Set<String> labelSet = ni.getNodeLabels();
  if (labelSet != null) {
    nodeLabels.addAll(labelSet);
    Collections.sort(nodeLabels);
  }
}
 
Example 21
Source Project: hadoop   Source File: CapacitySchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
  super.init(clock, sched, plans);
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
  if (!(sched instanceof CapacityScheduler)) {
    throw new YarnRuntimeException(
        "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
  }
  this.cs = (CapacityScheduler) sched;
}
 
Example 22
Source Project: hadoop   Source File: CapacityReservationSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is capacity based
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof CapacityScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + CapacityScheduler.class.getCanonicalName());
  }
  capScheduler = (CapacityScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}
 
Example 23
Source Project: big-c   Source File: TestFairSchedulerPreemption.java    License: Apache License 2.0 5 votes vote down vote up
public Configuration createConfiguration() {
  Configuration conf = super.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class,
      ResourceScheduler.class);
  conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example 24
Source Project: hadoop   Source File: AbstractReservationSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 
Example 25
Source Project: big-c   Source File: ClusterMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 
Example 26
Source Project: hadoop   Source File: RMActiveServiceContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
 
Example 27
Source Project: big-c   Source File: TestRMWebServicesFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example 28
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitMaxAppAttempts() throws Exception {
  int[] globalMaxAppAttempts = new int[] { 10, 1 };
  int[][] individualMaxAppAttempts = new int[][]{
      new int[]{ 9, 10, 11, 0 },
      new int[]{ 1, 10, 0, -1 }};
  int[][] expectedNums = new int[][]{
      new int[]{ 9, 10, 10, 10 },
      new int[]{ 1, 1, 1, 1 }};
  for (int i = 0; i < globalMaxAppAttempts.length; ++i) {
    for (int j = 0; j < individualMaxAppAttempts.length; ++j) {
      ResourceScheduler scheduler = mockResourceScheduler();
      Configuration conf = new Configuration();
      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, globalMaxAppAttempts[i]);
      ApplicationMasterService masterService =
          new ApplicationMasterService(rmContext, scheduler);
      TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
          new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
          new ApplicationACLsManager(conf), conf);

      ApplicationId appID = MockApps.newAppID(i * 4 + j + 1);
      asContext.setApplicationId(appID);
      if (individualMaxAppAttempts[i][j] != 0) {
        asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
      }
      appMonitor.submitApplication(asContext, "test");
      RMApp app = rmContext.getRMApps().get(appID);
      Assert.assertEquals("max application attempts doesn't match",
          expectedNums[i][j], app.getMaxAppAttempts());

      // wait for event to be processed
      int timeoutSecs = 0;
      while ((getAppEventType() == RMAppEventType.KILL) &&
          timeoutSecs++ < 20) {
        Thread.sleep(1000);
      }
      setAppEventType(RMAppEventType.KILL);
    }
  }
}
 
Example 29
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  String httpPrefix = "hadoop.http.authentication.";
  rmconf.setStrings(httpPrefix + "type", "kerberos");
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL,
    httpSpnegoPrincipal);
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  // use any file for signature secret
  rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file",
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
    true);
  rmconf.set("hadoop.http.filter.initializers",
    AuthenticationFilterInitializer.class.getName());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.RM_KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
  rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example 30
Source Project: hadoop   Source File: TestRMWebServicesFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}