org.apache.hadoop.yarn.util.resource.Resources Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.resource.Resources. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName, nodeManager.getNodeLabels());
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());

  // update this node to node label manager
  if (labelManager != null) {
    labelManager.activateNode(nodeManager.getNodeID(),
        nodeManager.getTotalCapability());
  }
  
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
  int numNodes = numNodeManagers.incrementAndGet();
  updateMaximumAllocation(schedulerNode, true);
  
  LOG.info("Added node " + nodeManager.getNodeAddress() + 
      " clusterResource: " + clusterResource);

  if (scheduleAsynchronously && numNodes == 1) {
    asyncSchedulerThread.beginSchedule();
  }
}
 
Example #2
Source File: FiCaSchedulerApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 
Example #3
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAggregateCapacityTracking() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  assertEquals(1024, scheduler.getClusterResource().getMemory());

  // Add another node
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  assertEquals(1536, scheduler.getClusterResource().getMemory());

  // Remove the first node
  NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(nodeEvent3);
  assertEquals(512, scheduler.getClusterResource().getMemory());
}
 
Example #4
Source File: FSParentQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void updateDemand() {
  // Compute demand by iterating through apps in the queue
  // Limit demand to maxResources
  Resource maxRes = scheduler.getAllocationConfiguration()
      .getMaxResources(getName());
  demand = Resources.createResource(0);
  for (FSQueue childQueue : childQueues) {
    childQueue.updateDemand();
    Resource toAdd = childQueue.getDemand();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Counting resource from " + childQueue.getName() + " " + 
          toAdd + "; Total resource consumption for " + getName() +
          " now " + demand);
    }
    demand = Resources.add(demand, toAdd);
    demand = Resources.componentwiseMin(demand, maxRes);
    if (Resources.equals(demand, maxRes)) {
      break;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("The updated demand for " + getName() + " is " + demand +
        "; the max is " + maxRes);
  }    
}
 
Example #5
Source File: TestReservationQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  // setup a context / conf
  csConf = new CapacitySchedulerConfiguration();
  YarnConfiguration conf = new YarnConfiguration();
  csContext = mock(CapacitySchedulerContext.class);
  when(csContext.getConfiguration()).thenReturn(csConf);
  when(csContext.getConf()).thenReturn(conf);
  when(csContext.getMinimumResourceCapability()).thenReturn(
      Resources.createResource(GB, 1, 1));
  when(csContext.getMaximumResourceCapability()).thenReturn(
      Resources.createResource(16 * GB, 32, 32));
  when(csContext.getClusterResource()).thenReturn(
      Resources.createResource(100 * 16 * GB, 100 * 32, 100 * 32));
  when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
  
  RMContext mockRMContext = TestUtils.getMockRMContext();
  when(csContext.getRMContext()).thenReturn(mockRMContext);

  // create a queue
  PlanQueue pq = new PlanQueue(csContext, "root", null, null);
  reservationQueue = new ReservationQueue(csContext, "a", pq);

}
 
Example #6
Source File: FifoScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 
Example #7
Source File: FairScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Check for queues that need tasks preempted, either because they have been
 * below their guaranteed share for minSharePreemptionTimeout or they have
 * been below their fair share threshold for the fairSharePreemptionTimeout. If
 * such queues exist, compute how many tasks of each type need to be preempted
 * and then select the right ones using preemptTasks.
 */
protected synchronized void preemptTasksIfNecessary() {
  if (!shouldAttemptPreemption()) {
    return;
  }

  long curTime = getClock().getTime();
  if (curTime - lastPreemptCheckTime < preemptionInterval) {
    return;
  }
  lastPreemptCheckTime = curTime;

  Resource resToPreempt = Resources.clone(Resources.none());
  for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
    Resources.addTo(resToPreempt, resToPreempt(sched, curTime));
  }
  if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt,
      Resources.none())) {
    preemptResources(resToPreempt);
  }
}
 
Example #8
Source File: LeafQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public synchronized Resource getUserAMResourceLimitPerPartition(
    String nodePartition) {
  /*
   * The user am resource limit is based on the same approach as the user
   * limit (as it should represent a subset of that). This means that it uses
   * the absolute queue capacity (per partition) instead of the max and is
   * modified by the userlimit and the userlimit factor as is the userlimit
   */
  float effectiveUserLimit = Math.max(userLimit / 100.0f,
      1.0f / Math.max(getActiveUsersManager().getNumActiveUsers(), 1));

  Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
      resourceCalculator,
      labelManager.getResourceByLabel(nodePartition, lastClusterResource),
      queueCapacities.getAbsoluteCapacity(nodePartition), minimumAllocation);

  return Resources.multiplyAndNormalizeUp(resourceCalculator,
      queuePartitionResource,
      queueCapacities.getMaxAMResourcePercentage(nodePartition)
          * effectiveUserLimit * userLimitFactor, minimumAllocation);
}
 
Example #9
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName, nodeManager.getNodeLabels());
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());

  // update this node to node label manager
  if (labelManager != null) {
    labelManager.activateNode(nodeManager.getNodeID(),
        nodeManager.getTotalCapability());
  }
  
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
  int numNodes = numNodeManagers.incrementAndGet();
  updateMaximumAllocation(schedulerNode, true);
  
  LOG.info("Added node " + nodeManager.getNodeAddress() + 
      " clusterResource: " + clusterResource);

  if (scheduleAsynchronously && numNodes == 1) {
    asyncSchedulerThread.beginSchedule();
  }
}
 
Example #10
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static YarnScheduler mockYarnScheduler() {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  when(yarnScheduler.getMinimumResourceCapability()).thenReturn(
      Resources.createResource(
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  when(yarnScheduler.getMaximumResourceCapability()).thenReturn(
      Resources.createResource(
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB));
  when(yarnScheduler.getAppsInQueue(QUEUE_1)).thenReturn(
      Arrays.asList(getApplicationAttemptId(101), getApplicationAttemptId(102)));
  when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
      Arrays.asList(getApplicationAttemptId(103)));
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
  ResourceCalculator rc = new DefaultResourceCalculator();
  when(yarnScheduler.getResourceCalculator()).thenReturn(rc);
  return yarnScheduler;
}
 
Example #11
Source File: YarnTaskSchedulerService.java    From tez with Apache License 2.0 6 votes vote down vote up
private void releaseContainer(ContainerId containerId) {
  Object assignedTask = containerAssignments.remove(containerId);
  if (assignedTask != null) {
    // A task was assigned to this container at some point. Inform the app.
    getContext().containerBeingReleased(containerId);
  }
  HeldContainer delayedContainer = heldContainers.remove(containerId);
  if (delayedContainer != null) {
    Resources.subtractFrom(allocatedResources,
        delayedContainer.getContainer().getResource());
  }
  if (delayedContainer != null || !shouldReuseContainers) {
    amRmClient.releaseAssignedContainer(containerId);
  }
  if (assignedTask != null) {
    // A task was assigned at some point. Add to release list since we are
    // releasing the container.
    releasedContainers.put(containerId, assignedTask);
  }
}
 
Example #12
Source File: TestUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static FiCaSchedulerNode getMockNode(
    String host, String rack, int port, int capability) {
  NodeId nodeId = mock(NodeId.class);
  when(nodeId.getHost()).thenReturn(host);
  when(nodeId.getPort()).thenReturn(port);
  RMNode rmNode = mock(RMNode.class);
  when(rmNode.getNodeID()).thenReturn(nodeId);
  when(rmNode.getTotalCapability()).thenReturn(
      Resources.createResource(capability, 1));
  when(rmNode.getNodeAddress()).thenReturn(host+":"+port);
  when(rmNode.getHostName()).thenReturn(host);
  when(rmNode.getRackName()).thenReturn(rack);
  
  FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
  LOG.info("node = " + host + " avail=" + node.getAvailableResource());
  return node;
}
 
Example #13
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 
Example #14
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 
Example #15
Source File: FifoPolicy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void computeShares(Collection<? extends Schedulable> schedulables,
    Resource totalResources) {
  if (schedulables.isEmpty()) {
    return;
  }

  Schedulable earliest = null;
  for (Schedulable schedulable : schedulables) {
    if (earliest == null ||
        schedulable.getStartTime() < earliest.getStartTime()) {
      earliest = schedulable;
    }
  }
  earliest.setFairShare(Resources.clone(totalResources));
}
 
Example #16
Source File: FSQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Helper method to check if the queue should attempt assigning resources
 * 
 * @return true if check passes (can assign) or false otherwise
 */
protected boolean assignContainerPreCheck(FSSchedulerNode node) {
  if (!Resources.fitsIn(getResourceUsage(),
      scheduler.getAllocationConfiguration().getMaxResources(getName()))
      || node.getReservedContainer() != null) {
    return false;
  }
  return true;
}
 
Example #17
Source File: FSLeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) {
  sched.updateDemand();
  Resource toAdd = sched.getDemand();
  if (LOG.isDebugEnabled()) {
    LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
        + "; Total resource consumption for " + getName() + " now "
        + demand);
  }
  demand = Resources.add(demand, toAdd);
  demand = Resources.componentwiseMin(demand, maxRes);
}
 
Example #18
Source File: TestComputeFairShares.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test that shares are computed accurately even when the number of slots is
 * very large.
 */
@Test
public void testLargeShares() {
  int million = 1000 * 1000;
  scheds.add(new FakeSchedulable());
  scheds.add(new FakeSchedulable());
  scheds.add(new FakeSchedulable());
  scheds.add(new FakeSchedulable());
  ComputeFairShares.computeShares(scheds,
      Resources.createResource(40 * million), ResourceType.MEMORY);
  verifyMemoryShares(10 * million, 10 * million, 10 * million, 10 * million);
}
 
Example #19
Source File: ProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private double getIdealPctOfGuaranteed(TempQueue q) {
  double pctOver = Integer.MAX_VALUE;
  if (q != null && Resources.greaterThan(
      rc, clusterRes, q.guaranteed, Resources.none())) {
    pctOver =
        Resources.divide(rc, clusterRes, q.idealAssigned, q.guaranteed);
  }
  return (pctOver);
}
 
Example #20
Source File: ResourceUsage.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Resource _get(String label, ResourceType type) {
  try {
    readLock.lock();
    UsageByLabel usage = usages.get(label);
    if (null == usage) {
      return Resources.none();
    }
    return normalize(usage.resArr[type.idx]);
  } finally {
    readLock.unlock();
  }
}
 
Example #21
Source File: FairSchedulerConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Resource getMinimumAllocation() {
  int mem = getInt(
      YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  int cpu = getInt(
      YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
  return Resources.createResource(mem, cpu);
}
 
Example #22
Source File: FSLeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Check whether this queue can run this application master under the
 * maxAMShare limit
 *
 * @param amResource
 * @return true if this queue can run
 */
public boolean canRunAppAM(Resource amResource) {
  float maxAMShare =
      scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName());
  if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
    return true;
  }
  Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare);
  Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
  return !policy
      .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource);
}
 
Example #23
Source File: SchedulerNode.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private synchronized void deductAvailableResource(Resource resource) {
  if (resource == null) {
    LOG.error("Invalid deduction of null resource for "
        + rmNode.getNodeAddress());
    return;
  }
  Resources.subtractFrom(availableResource, resource);
  Resources.addTo(usedResource, resource);
}
 
Example #24
Source File: TestFairSchedulerFairShare.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void createClusterWithQueuesAndOneNode(int mem, int vCores, int gCores,
    String policy) throws IOException {
  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"root\" >");
  out.println("   <queue name=\"parentA\" >");
  out.println("       <weight>8</weight>");
  out.println("       <queue name=\"childA1\" />");
  out.println("       <queue name=\"childA2\" />");
  out.println("       <queue name=\"childA3\" />");
  out.println("       <queue name=\"childA4\" />");
  out.println("   </queue>");
  out.println("   <queue name=\"parentB\" >");
  out.println("       <weight>1</weight>");
  out.println("       <queue name=\"childB1\" />");
  out.println("       <queue name=\"childB2\" />");
  out.println("   </queue>");
  out.println("</queue>");
  out.println("<defaultQueueSchedulingPolicy>" + policy
      + "</defaultQueueSchedulingPolicy>");
  out.println("</allocations>");
  out.close();

  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();

  RMNode node1 = MockNodes.newNodeInfo(1,
      Resources.createResource(mem, vCores, gCores), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
}
 
Example #25
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testContinuousSchedulingWithNodeRemoved() throws Exception {
  // Disable continuous scheduling, will invoke continuous scheduling once manually
  scheduler.init(conf);
  scheduler.start();
  Assert.assertTrue("Continuous scheduling should be disabled.",
      !scheduler.isContinuousSchedulingEnabled());

  // Add two nodes
  RMNode node1 =
      MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8, 8), 1,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8, 8), 2,
          "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  Assert.assertEquals("We should have two alive nodes.",
      2, scheduler.getNumClusterNodes());

  // Remove one node
  NodeRemovedSchedulerEvent removeNode1 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(removeNode1);
  Assert.assertEquals("We should only have one alive node.",
      1, scheduler.getNumClusterNodes());

  // Invoke the continuous scheduling once
  try {
    scheduler.continuousSchedulingAttempt();
  } catch (Exception e) {
    fail("Exception happened when doing continuous scheduling. " +
      e.toString());
  }
}
 
Example #26
Source File: CapacitySchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected Resource getPlanResources(
    Plan plan, Queue queue, Resource clusterResources) {
  PlanQueue planQueue = (PlanQueue)queue;
  float planAbsCap = planQueue.getAbsoluteCapacity();
  Resource planResources = Resources.multiply(clusterResources, planAbsCap);
  plan.setTotalCapacity(planResources);
  return planResources;
}
 
Example #27
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public synchronized boolean unreserve(FiCaSchedulerNode node, Priority priority) {
  Map<NodeId, RMContainer> reservedContainers =
    this.reservedContainers.get(priority);

  if (reservedContainers != null) {
    RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());

    // unreserve is now triggered in new scenarios (preemption)
    // as a consequence reservedcontainer might be null, adding NP-checks
    if (reservedContainer != null
        && reservedContainer.getContainer() != null
        && reservedContainer.getContainer().getResource() != null) {

      if (reservedContainers.isEmpty()) {
        this.reservedContainers.remove(priority);
      }
      // Reset the re-reservation count
      resetReReservations(priority);

      Resource resource = reservedContainer.getContainer().getResource();
      Resources.subtractFrom(currentReservation, resource);

      LOG.info("Application " + getApplicationId() + " unreserved "
          + " on node " + node + ", currently has " + reservedContainers.size()
          + " at priority " + priority + "; currentReservation "
          + currentReservation);
      return true;
    }
  }
  return false;
}
 
Example #28
Source File: TestComputeFairShares.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** 
 * Basic test - pools with different demands that are all higher than their
 * fair share (of 10 slots) should each get their fair share.
 */
@Test
public void testEqualSharing() {
  scheds.add(new FakeSchedulable());
  scheds.add(new FakeSchedulable());
  scheds.add(new FakeSchedulable());
  scheds.add(new FakeSchedulable());
  ComputeFairShares.computeShares(scheds,
      Resources.createResource(40), ResourceType.MEMORY);
  verifyMemoryShares(10, 10, 10, 10);
}
 
Example #29
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
  ResourceCalculator resourceCalculator = new DominantResourceCalculator();
  
  Resource minResource = Resources.createResource(1024, 1);
  Resource maxResource = Resources.createResource(10240, 10);
  Resource clusterResource = Resources.createResource(10 * 1024, 10);
  
  ResourceRequest ask = new ResourceRequestPBImpl();

  // case negative memory/vcores
  ask.setCapability(Resources.createResource(-1024, -1));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(minResource, ask.getCapability());

  // case zero memory/vcores
  ask.setCapability(Resources.createResource(0, 0));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(minResource, ask.getCapability());
  assertEquals(1, ask.getCapability().getVirtualCores());
  assertEquals(1024, ask.getCapability().getMemory());

  // case non-zero memory & zero cores
  ask.setCapability(Resources.createResource(1536, 0));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(Resources.createResource(2048, 1), ask.getCapability());
  assertEquals(1, ask.getCapability().getVirtualCores());
  assertEquals(2048, ask.getCapability().getMemory());
}
 
Example #30
Source File: FairSchedulerConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Resource getMaximumAllocation() {
  int mem = getInt(
      YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
  int cpu = getInt(
      YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
  return Resources.createResource(mem, cpu);
}