Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: big-c   Author: yncxcw   File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #2
Source Project: hadoop   Author: naver   File: RMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #3
Source Project: hadoop   Author: naver   File: RmController.java    License: Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #4
Source Project: hadoop   Author: naver   File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example #5
Source Project: hadoop   Author: naver   File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example #6
Source Project: big-c   Author: yncxcw   File: RmController.java    License: Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #7
Source Project: big-c   Author: yncxcw   File: RMContextImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example #8
Source Project: big-c   Author: yncxcw   File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAsyncScheduling() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  final int NODES = 100;
  
  // Register nodes
  for (int i=0; i < NODES; ++i) {
    String host = "192.168.1." + i;
    RMNode node =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
  }
  
  // Now directly exercise the scheduling loop
  for (int i=0; i < NODES; ++i) {
    CapacityScheduler.schedule(cs);
  }
}
 
Example #9
Source Project: hadoop   Author: naver   File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #10
Source Project: big-c   Author: yncxcw   File: TestRMWebServicesHttpStaticUserPermissions.java    License: Apache License 2.0 6 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal);
  rmconf.set("yarn.resourcemanager.keytab",
      spnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #11
Source Project: big-c   Author: yncxcw   File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
 
Example #12
Source Project: Flink-CEPplus   Author: ljygz   File: YARNSessionFIFOITCase.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo");
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	storageDir = FOLDER.newFolder().getAbsolutePath();

	// startYARNWithConfig should be implemented by subclass
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #14
Source Project: big-c   Author: yncxcw   File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRm(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs = mockFairScheduler();
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #15
Source Project: big-c   Author: yncxcw   File: TestFairSchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example #16
Source Project: big-c   Author: yncxcw   File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager();
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      FifoScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
}
 
Example #17
Source Project: hadoop   Author: naver   File: RMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@GET
@Path("/nodes/{nodeId}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeInfo getNode(@PathParam("nodeId") String nodeId) {
  init();
  if (nodeId == null || nodeId.isEmpty()) {
    throw new NotFoundException("nodeId, " + nodeId + ", is empty or null");
  }
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  NodeId nid = ConverterUtils.toNodeId(nodeId);
  RMNode ni = this.rm.getRMContext().getRMNodes().get(nid);
  boolean isInactive = false;
  if (ni == null) {
    ni = this.rm.getRMContext().getInactiveRMNodes().get(nid.getHost());
    if (ni == null) {
      throw new NotFoundException("nodeId, " + nodeId + ", is not found");
    }
    isInactive = true;
  }
  NodeInfo nodeInfo = new NodeInfo(ni, sched);
  if (isInactive) {
    nodeInfo.setNodeHTTPAddress(EMPTY);
  }
  return nodeInfo;
}
 
Example #18
Source Project: hadoop   Author: naver   File: ClusterMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.reservedGpuCores = metrics.getReservedGpuCores();
  this.availableGpuCores = metrics.getAvailableGpuCores();
  this.allocatedGpuCores = metrics.getAllocatedGpuCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.totalGpuCores = availableGpuCores + allocatedGpuCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 
Example #19
Source Project: hadoop   Author: naver   File: SchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example #20
Source Project: hadoop   Author: naver   File: NodeInfo.java    License: Apache License 2.0 5 votes vote down vote up
public NodeInfo(RMNode ni, ResourceScheduler sched) {
  NodeId id = ni.getNodeID();
  SchedulerNodeReport report = sched.getNodeReport(id);
  this.numContainers = 0;
  this.usedMemoryMB = 0;
  this.availMemoryMB = 0;
  if (report != null) {
    this.numContainers = report.getNumContainers();
    this.usedMemoryMB = report.getUsedResource().getMemory();
    this.availMemoryMB = report.getAvailableResource().getMemory();
    this.usedVirtualCores = report.getUsedResource().getVirtualCores();
    this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
    this.usedGpuCores = report.getUsedResource().getGpuCores();
    this.availableGpuCores = report.getAvailableResource().getGpuCores();
  }
  this.id = id.toString();
  this.rack = ni.getRackName();
  this.nodeHostName = ni.getHostName();
  this.state = ni.getState();
  this.nodeHTTPAddress = ni.getHttpAddress();
  this.lastHealthUpdate = ni.getLastHealthReportTime();
  this.healthReport = String.valueOf(ni.getHealthReport());
  this.version = ni.getNodeManagerVersion();
  
  // add labels
  Set<String> labelSet = ni.getNodeLabels();
  if (labelSet != null) {
    nodeLabels.addAll(labelSet);
    Collections.sort(nodeLabels);
  }
}
 
Example #21
Source Project: hadoop   Author: naver   File: CapacitySchedulerPlanFollower.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
  super.init(clock, sched, plans);
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
  if (!(sched instanceof CapacityScheduler)) {
    throw new YarnRuntimeException(
        "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
  }
  this.cs = (CapacityScheduler) sched;
}
 
Example #22
Source Project: hadoop   Author: naver   File: CapacityReservationSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is capacity based
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof CapacityScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + CapacityScheduler.class.getCanonicalName());
  }
  capScheduler = (CapacityScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}
 
Example #23
Source Project: big-c   Author: yncxcw   File: TestFairSchedulerPreemption.java    License: Apache License 2.0 5 votes vote down vote up
public Configuration createConfiguration() {
  Configuration conf = super.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, StubbedFairScheduler.class,
      ResourceScheduler.class);
  conf.setBoolean(FairSchedulerConfiguration.PREEMPTION, true);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example #24
Source Project: hadoop   Author: naver   File: AbstractReservationSystem.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 
Example #25
Source Project: big-c   Author: yncxcw   File: ClusterMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 
Example #26
Source Project: hadoop   Author: naver   File: RMActiveServiceContext.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@Unstable
public RMActiveServiceContext(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setContainerAllocationExpirer(containerAllocationExpirer);
  this.setAMLivelinessMonitor(amLivelinessMonitor);
  this.setAMFinishingMonitor(amFinishingMonitor);
  this.setDelegationTokenRenewer(delegationTokenRenewer);
  this.setAMRMTokenSecretManager(appTokenSecretManager);
  this.setContainerTokenSecretManager(containerTokenSecretManager);
  this.setNMTokenSecretManager(nmTokenSecretManager);
  this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
  this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  this.setScheduler(scheduler);

  RMStateStore nullStore = new NullRMStateStore();
  nullStore.setRMDispatcher(rmDispatcher);
  try {
    nullStore.init(new YarnConfiguration());
    setStateStore(nullStore);
  } catch (Exception e) {
    assert false;
  }
}
 
Example #27
Source Project: big-c   Author: yncxcw   File: TestRMWebServicesFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #28
Source Project: hadoop   Author: naver   File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitMaxAppAttempts() throws Exception {
  int[] globalMaxAppAttempts = new int[] { 10, 1 };
  int[][] individualMaxAppAttempts = new int[][]{
      new int[]{ 9, 10, 11, 0 },
      new int[]{ 1, 10, 0, -1 }};
  int[][] expectedNums = new int[][]{
      new int[]{ 9, 10, 10, 10 },
      new int[]{ 1, 1, 1, 1 }};
  for (int i = 0; i < globalMaxAppAttempts.length; ++i) {
    for (int j = 0; j < individualMaxAppAttempts.length; ++j) {
      ResourceScheduler scheduler = mockResourceScheduler();
      Configuration conf = new Configuration();
      conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, globalMaxAppAttempts[i]);
      ApplicationMasterService masterService =
          new ApplicationMasterService(rmContext, scheduler);
      TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
          new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
          new ApplicationACLsManager(conf), conf);

      ApplicationId appID = MockApps.newAppID(i * 4 + j + 1);
      asContext.setApplicationId(appID);
      if (individualMaxAppAttempts[i][j] != 0) {
        asContext.setMaxAppAttempts(individualMaxAppAttempts[i][j]);
      }
      appMonitor.submitApplication(asContext, "test");
      RMApp app = rmContext.getRMApps().get(appID);
      Assert.assertEquals("max application attempts doesn't match",
          expectedNums[i][j], app.getMaxAppAttempts());

      // wait for event to be processed
      int timeoutSecs = 0;
      while ((getAppEventType() == RMAppEventType.KILL) &&
          timeoutSecs++ < 20) {
        Thread.sleep(1000);
      }
      setAppEventType(RMAppEventType.KILL);
    }
  }
}
 
Example #29
Source Project: hadoop   Author: naver   File: TestRMWebServicesDelegationTokenAuthentication.java    License: Apache License 2.0 5 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  String httpPrefix = "hadoop.http.authentication.";
  rmconf.setStrings(httpPrefix + "type", "kerberos");
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL,
    httpSpnegoPrincipal);
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  // use any file for signature secret
  rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file",
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
    true);
  rmconf.set("hadoop.http.filter.initializers",
    AuthenticationFilterInitializer.class.getName());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.RM_KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
  rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #30
Source Project: hadoop   Author: naver   File: TestRMWebServicesFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}