Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.ResourceManager#getResourceScheduler()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.ResourceManager#getResourceScheduler() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: FairSchedulerAppsBlock.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 
Example 2
Source Project: hadoop   File: RmController.java    License: Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example 3
Source Project: big-c   File: FairSchedulerAppsBlock.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 
Example 4
Source Project: big-c   File: RmController.java    License: Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example 5
Source Project: hadoop   File: CapacitySchedulerPage.java    License: Apache License 2.0 5 votes vote down vote up
@Inject QueuesBlock(ResourceManager rm, CSQInfo info) {
  cs = (CapacityScheduler) rm.getResourceScheduler();
  csqinfo = info;
  RMNodeLabelsManager nodeLabelManager =
      rm.getRMContext().getNodeLabelManager();
  nodeLabelsInfo = nodeLabelManager.pullRMNodeLabelsInfo();      
}
 
Example 6
Source Project: hadoop   File: FifoSchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public FifoSchedulerInfo(final ResourceManager rm) {

    RMContext rmContext = rm.getRMContext();

    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    qName = fs.getQueueInfo("", false, false).getQueueName();
    QueueInfo qInfo = fs.getQueueInfo(qName, true, true);

    this.usedCapacity = qInfo.getCurrentCapacity();
    this.capacity = qInfo.getCapacity();
    this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory();
    this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory();
    this.qstate = qInfo.getQueueState();

    this.numNodes = rmContext.getRMNodes().size();
    this.usedNodeCapacity = 0;
    this.availNodeCapacity = 0;
    this.totalNodeCapacity = 0;
    this.numContainers = 0;

    for (RMNode ni : rmContext.getRMNodes().values()) {
      SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
      this.usedNodeCapacity += report.getUsedResource().getMemory();
      this.availNodeCapacity += report.getAvailableResource().getMemory();
      this.totalNodeCapacity += ni.getTotalCapability().getMemory();
      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
    }
  }
 
Example 7
Source Project: hadoop   File: UserMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public UserMetricsInfo(final ResourceManager rm, final String user) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  QueueMetrics userMetrics = metrics.getUserMetrics(user);
  this.userMetricsAvailable = false;

  if (userMetrics != null) {
    this.userMetricsAvailable = true;

    this.appsSubmitted = userMetrics.getAppsSubmitted();
    this.appsCompleted = userMetrics.getAppsCompleted();
    this.appsPending = userMetrics.getAppsPending();
    this.appsRunning = userMetrics.getAppsRunning();
    this.appsFailed = userMetrics.getAppsFailed();
    this.appsKilled = userMetrics.getAppsKilled();

    this.runningContainers = userMetrics.getAllocatedContainers();
    this.pendingContainers = userMetrics.getPendingContainers();
    this.reservedContainers = userMetrics.getReservedContainers();

    this.reservedMB = userMetrics.getReservedMB();
    this.pendingMB = userMetrics.getPendingMB();
    this.allocatedMB = userMetrics.getAllocatedMB();

    this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
    this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
    this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();

    this.reservedGpuCores = userMetrics.getReservedGpuCores();
    this.pendingGpuCores = userMetrics.getPendingGpuCores();
    this.allocatedGpuCores = userMetrics.getAllocatedGpuCores();
  }
}
 
Example 8
Source Project: hadoop   File: ClusterMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.reservedGpuCores = metrics.getReservedGpuCores();
  this.availableGpuCores = metrics.getAvailableGpuCores();
  this.allocatedGpuCores = metrics.getAllocatedGpuCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.totalGpuCores = availableGpuCores + allocatedGpuCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 
Example 9
Source Project: hadoop   File: SchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example 10
Source Project: big-c   File: FifoSchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public FifoSchedulerInfo(final ResourceManager rm) {

    RMContext rmContext = rm.getRMContext();

    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    qName = fs.getQueueInfo("", false, false).getQueueName();
    QueueInfo qInfo = fs.getQueueInfo(qName, true, true);

    this.usedCapacity = qInfo.getCurrentCapacity();
    this.capacity = qInfo.getCapacity();
    this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory();
    this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory();
    this.qstate = qInfo.getQueueState();

    this.numNodes = rmContext.getRMNodes().size();
    this.usedNodeCapacity = 0;
    this.availNodeCapacity = 0;
    this.totalNodeCapacity = 0;
    this.numContainers = 0;

    for (RMNode ni : rmContext.getRMNodes().values()) {
      SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
      this.usedNodeCapacity += report.getUsedResource().getMemory();
      this.availNodeCapacity += report.getAvailableResource().getMemory();
      this.totalNodeCapacity += ni.getTotalCapability().getMemory();
      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
    }
  }
 
Example 11
Source Project: big-c   File: UserMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public UserMetricsInfo(final ResourceManager rm, final String user) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  QueueMetrics userMetrics = metrics.getUserMetrics(user);
  this.userMetricsAvailable = false;

  if (userMetrics != null) {
    this.userMetricsAvailable = true;

    this.appsSubmitted = userMetrics.getAppsSubmitted();
    this.appsCompleted = userMetrics.getAppsCompleted();
    this.appsPending = userMetrics.getAppsPending();
    this.appsRunning = userMetrics.getAppsRunning();
    this.appsFailed = userMetrics.getAppsFailed();
    this.appsKilled = userMetrics.getAppsKilled();

    this.runningContainers = userMetrics.getAllocatedContainers();
    this.pendingContainers = userMetrics.getPendingContainers();
    this.reservedContainers = userMetrics.getReservedContainers();

    this.reservedMB = userMetrics.getReservedMB();
    this.pendingMB = userMetrics.getPendingMB();
    this.allocatedMB = userMetrics.getAllocatedMB();

    this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
    this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
    this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
  }
}
 
Example 12
Source Project: big-c   File: ClusterMetricsInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 
Example 13
Source Project: big-c   File: SchedulerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example 14
Source Project: hadoop   File: DefaultSchedulerPage.java    License: Apache License 2.0 4 votes vote down vote up
@Inject QueuesBlock(ResourceManager rm) {
  sinfo = new FifoSchedulerInfo(rm);
  fs = (FifoScheduler) rm.getResourceScheduler();
}
 
Example 15
Source Project: hadoop   File: FairSchedulerPage.java    License: Apache License 2.0 4 votes vote down vote up
@Inject QueuesBlock(ResourceManager rm, FSQInfo info) {
  fs = (FairScheduler)rm.getResourceScheduler();
  fsqinfo = info;
}
 
Example 16
Source Project: hadoop   File: TestCapacityScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
  resourceManager = new ResourceManager() {
     @Override
      protected RMNodeLabelsManager createNodeLabelManager() {
        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
        mgr.init(getConfig());
        return mgr;
      }
  };
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  StringBuilder qState = new StringBuilder();
  qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
      .append(CapacitySchedulerConfiguration.DOT)
      .append(CapacitySchedulerConfiguration.STATE);
  csConf.set(qState.toString(), QueueState.STOPPED.name());
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
      .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());

  ResourceScheduler scheduler = resourceManager.getResourceScheduler();

  // Register node1
  String host_0 = "host_0";
  NodeManager nm_0 =
      registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
          Resources.createResource(6 * GB, 1));

  // ResourceRequest priorities
  Priority priority_0 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(0);
  Priority priority_1 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(1);

  // Submit application_0
  Application application_0 =
      new Application("user_0", "a1", resourceManager);
  application_0.submit(); // app + app attempt event sent to scheduler

  application_0.addNodeManager(host_0, 1234, nm_0);

  Resource capability_0_0 = Resources.createResource(3 * GB, 1);
  application_0.addResourceRequestSpec(priority_1, capability_0_0);

  Resource capability_0_1 = Resources.createResource(2 * GB, 1);
  application_0.addResourceRequestSpec(priority_0, capability_0_1);

  Task task_0_0 =
      new Task(application_0, priority_1, new String[] { host_0 });
  application_0.addTask(task_0_0);

  // Send resource requests to the scheduler
  application_0.schedule(); // allocate

  // task_0_0 allocated
  nodeUpdate(nm_0);

  // Get allocations from the scheduler
  application_0.schedule(); // task_0_0
  checkApplicationResourceUsage(3 * GB, application_0);

  checkNodeResourceUsage(3 * GB, nm_0);
  // b2 queue contains 3GB consumption app,
  // add another 3GB will hit max capacity limit on queue b
  scheduler.moveApplication(application_0.getApplicationId(), "b1");

}
 
Example 17
Source Project: big-c   File: DefaultSchedulerPage.java    License: Apache License 2.0 4 votes vote down vote up
@Inject QueuesBlock(ResourceManager rm) {
  sinfo = new FifoSchedulerInfo(rm);
  fs = (FifoScheduler) rm.getResourceScheduler();
}
 
Example 18
Source Project: big-c   File: FairSchedulerPage.java    License: Apache License 2.0 4 votes vote down vote up
@Inject QueuesBlock(ResourceManager rm, FSQInfo info) {
  fs = (FairScheduler)rm.getResourceScheduler();
  fsqinfo = info;
}
 
Example 19
Source Project: big-c   File: CapacitySchedulerPage.java    License: Apache License 2.0 4 votes vote down vote up
@Inject QueuesBlock(ResourceManager rm, CSQInfo info) {
  cs = (CapacityScheduler) rm.getResourceScheduler();
  csqinfo = info;
}
 
Example 20
Source Project: big-c   File: TestCapacityScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
  resourceManager = new ResourceManager() {
     @Override
      protected RMNodeLabelsManager createNodeLabelManager() {
        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
        mgr.init(getConfig());
        return mgr;
      }
  };
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  StringBuilder qState = new StringBuilder();
  qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
      .append(CapacitySchedulerConfiguration.DOT)
      .append(CapacitySchedulerConfiguration.STATE);
  csConf.set(qState.toString(), QueueState.STOPPED.name());
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
      .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());

  ResourceScheduler scheduler = resourceManager.getResourceScheduler();

  // Register node1
  String host_0 = "host_0";
  NodeManager nm_0 =
      registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
          Resources.createResource(6 * GB, 1));

  // ResourceRequest priorities
  Priority priority_0 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(0);
  Priority priority_1 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(1);

  // Submit application_0
  Application application_0 =
      new Application("user_0", "a1", resourceManager);
  application_0.submit(); // app + app attempt event sent to scheduler

  application_0.addNodeManager(host_0, 1234, nm_0);

  Resource capability_0_0 = Resources.createResource(3 * GB, 1);
  application_0.addResourceRequestSpec(priority_1, capability_0_0);

  Resource capability_0_1 = Resources.createResource(2 * GB, 1);
  application_0.addResourceRequestSpec(priority_0, capability_0_1);

  Task task_0_0 =
      new Task(application_0, priority_1, new String[] { host_0 });
  application_0.addTask(task_0_0);

  // Send resource requests to the scheduler
  application_0.schedule(); // allocate

  // task_0_0 allocated
  nodeUpdate(nm_0);

  // Get allocations from the scheduler
  application_0.schedule(); // task_0_0
  checkApplicationResourceUsage(3 * GB, application_0);

  checkNodeResourceUsage(3 * GB, nm_0);
  // b2 queue contains 3GB consumption app,
  // add another 3GB will hit max capacity limit on queue b
  scheduler.moveApplication(application_0.getApplicationId(), "b1");

}