org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.releaseResource(clusterResource,
        rmContainer.getContainer().getResource(),
        node.getLabels());
    LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.detachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example #2
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Preempt a running container according to the priority
 */
@Override
public RMContainer preemptContainer() {
  if (LOG.isDebugEnabled()) {
    LOG.debug("App " + getName() + " is going to preempt a running " +
        "container");
  }

  RMContainer toBePreempted = null;
  for (RMContainer container : getLiveContainers()) {
    if (!getPreemptionContainers().contains(container) &&
        (toBePreempted == null ||
            comparator.compare(toBePreempted, container) > 0)) {
      toBePreempted = container;
    }
  }
  return toBePreempted;
}
 
Example #3
Source File: ProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Compare by reversed priority order first, and then reversed containerId
 * order
 * @param containers
 */
@VisibleForTesting
static void sortContainers(List<RMContainer> containers){
  Collections.sort(containers, new Comparator<RMContainer>() {
    @Override
    public int compare(RMContainer a, RMContainer b) {
      Comparator<Priority> c = new org.apache.hadoop.yarn.server
          .resourcemanager.resource.Priority.Comparator();
      int priorityComp = c.compare(b.getContainer().getPriority(),
                                   a.getContainer().getPriority());
      if (priorityComp != 0) {
        return priorityComp;
      }
      return b.getContainerId().compareTo(a.getContainerId());
    }
  });
}
 
Example #4
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void unreserveInternal(
    Priority priority, FSSchedulerNode node) {
  Map<NodeId, RMContainer> reservedContainers = 
      this.reservedContainers.get(priority);
  RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());
  if (reservedContainers.isEmpty()) {
    this.reservedContainers.remove(priority);
  }
  
  // Reset the re-reservation count
  resetReReservations(priority);

  Resource resource = reservedContainer.getContainer().getResource();
  this.attemptResourceUsage.decReserved(resource);

  LOG.info("Application " + getApplicationId() + " unreserved " + " on node "
      + node + ", currently has " + reservedContainers.size()
      + " at priority " + priority + "; currentReservation "
      + this.attemptResourceUsage.getReserved());
}
 
Example #5
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void removeNode(RMNode nodeInfo) {
  FiCaSchedulerNode node = getNode(nodeInfo.getNodeID());
  if (node == null) {
    return;
  }
  // Kill running containers
  for(RMContainer container : node.getRunningContainers()) {
    completedContainer(container, 
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(), 
            SchedulerUtils.LOST_CONTAINER),
            RMContainerEventType.KILL);
  }
  
  //Remove the node
  this.nodes.remove(nodeInfo.getNodeID());
  updateMaximumAllocation(node, false);
  
  // Update cluster metrics
  Resources.subtractFrom(clusterResource, node.getRMNode().getTotalCapability());
}
 
Example #6
Source File: LeafQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    allocateResource(clusterResource, application, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
    LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
        + " resource=" + rmContainer.getContainer().getResource()
        + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
        + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
        + queueUsage.getUsed() + " cluster=" + clusterResource);
    // Inform the parent queue
    getParent().attachContainer(clusterResource, application, rmContainer);
  }
}
 
Example #7
Source File: TestRMApplicationHistoryWriter.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static RMContainer createRMContainer(ContainerId containerId) {
  RMContainer container = mock(RMContainer.class);
  when(container.getContainerId()).thenReturn(containerId);
  when(container.getAllocatedNode()).thenReturn(
    NodeId.newInstance("test host", -100));
  when(container.getAllocatedResource()).thenReturn(
    Resource.newInstance(-1, -1, -1));
  when(container.getAllocatedPriority()).thenReturn(Priority.UNDEFINED);
  when(container.getCreationTime()).thenReturn(0L);
  when(container.getFinishTime()).thenReturn(1L);
  when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info");
  when(container.getLogURL()).thenReturn("test log url");
  when(container.getContainerExitStatus()).thenReturn(-1);
  when(container.getContainerState()).thenReturn(ContainerState.COMPLETE);
  return container;
}
 
Example #8
Source File: ResourceManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(ContainerPreemptEvent event) {
  ApplicationAttemptId aid = event.getAppId();
  RMContainer container = event.getContainer();
  switch (event.getType()) {
  case DROP_RESERVATION:
    scheduler.dropContainerReservation(container);
    break;
  case PREEMPT_CONTAINER:
    scheduler.preemptContainer(aid, container);
    break;
  case KILL_CONTAINER:
    scheduler.killContainer(container);
    break;
  }
}
 
Example #9
Source File: ParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
  }
  if (parent != null) {
    parent.recoverContainer(clusterResource, attempt, rmContainer);
  }
}
 
Example #10
Source File: SchedulerApplicationAttempt.java    From big-c with Apache License 2.0 6 votes vote down vote up
public synchronized void recoverContainer(RMContainer rmContainer) {
  // recover app scheduling info
  appSchedulingInfo.recoverContainer(rmContainer);

  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  LOG.info("SchedulerAttempt " + getApplicationAttemptId()
    + " is recovering container " + rmContainer.getContainerId());
  liveContainers.put(rmContainer.getContainerId(), rmContainer);
  Resources.addTo(currentConsumption, rmContainer.getContainer()
    .getResource());
  // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
  // is called.
  // newlyAllocatedContainers.add(rmContainer);
  // schedulingOpportunities
  // lastScheduledContainer
}
 
Example #11
Source File: TestProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
FiCaSchedulerApp mockApp(int qid, int id, int used, int pending, int reserved,
    int gran) {
  FiCaSchedulerApp app = mock(FiCaSchedulerApp.class);

  ApplicationId appId = ApplicationId.newInstance(TS, id);
  ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(appId, 0);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getApplicationAttemptId()).thenReturn(appAttId);

  int cAlloc = 0;
  Resource unit = Resource.newInstance(gran, 0, 0);
  List<RMContainer> cReserved = new ArrayList<RMContainer>();
  for (int i = 0; i < reserved; i += gran) {
    cReserved.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER
        .getValue()));
    ++cAlloc;
  }
  when(app.getReservedContainers()).thenReturn(cReserved);

  List<RMContainer> cLive = new ArrayList<RMContainer>();
  for (int i = 0; i < used; i += gran) {
    if(setAMContainer && i == 0){
      cLive.add(mockContainer(appAttId, cAlloc, unit, priority.AMCONTAINER
          .getValue()));
    }else if(setLabeledContainer && i ==1){
      cLive.add(mockContainer(appAttId, cAlloc, unit,
          priority.LABELEDCONTAINER.getValue()));
      ++used;
    }
    else{
      cLive.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER
          .getValue()));
    }
    ++cAlloc;
  }
  when(app.getLiveContainers()).thenReturn(cLive);
  return app;
}
 
Example #12
Source File: SchedulerApplicationAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Has the application reserved the given <code>node</code> at the
 * given <code>priority</code>?
 * @param node node to be checked
 * @param priority priority of reserved container
 * @return true is reserved, false if not
 */
public synchronized boolean isReserved(SchedulerNode node, Priority priority) {
  Map<NodeId, RMContainer> reservedContainers = 
      this.reservedContainers.get(priority);
  if (reservedContainers != null) {
    return reservedContainers.containsKey(node.getNodeID());
  }
  return false;
}
 
Example #13
Source File: SystemMetricsPublisher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public void containerFinished(RMContainer container, long finishedTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new ContainerFinishedEvent(
            container.getContainerId(),
            container.getDiagnosticsInfo(),
            container.getContainerExitStatus(),
            container.getContainerState(),
            finishedTime));
  }
}
 
Example #14
Source File: FSLeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public RMContainer preemptContainer() {
  RMContainer toBePreempted = null;

  // If this queue is not over its fair share, reject
  if (!preemptContainerPreCheck()) {
    return toBePreempted;
  }

  if (LOG.isDebugEnabled()) {
    LOG.debug("Queue " + getName() + " is going to preempt a container " +
        "from its applications.");
  }

  // Choose the app that is most over fair share
  Comparator<Schedulable> comparator = policy.getComparator();
  FSAppAttempt candidateSched = null;
  readLock.lock();
  try {
    for (FSAppAttempt sched : runnableApps) {
      if (candidateSched == null ||
          comparator.compare(sched, candidateSched) > 0) {
        candidateSched = sched;
      }
    }
  } finally {
    readLock.unlock();
  }

  // Preempt from the selected app
  if (candidateSched != null) {
    toBePreempted = candidateSched.preemptContainer();
  }
  return toBePreempted;
}
 
Example #15
Source File: FairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private synchronized void removeNode(RMNode rmNode) {
  FSSchedulerNode node = getFSSchedulerNode(rmNode.getNodeID());
  // This can occur when an UNHEALTHY node reconnects
  if (node == null) {
    return;
  }
  Resources.subtractFrom(clusterResource, rmNode.getTotalCapability());
  updateRootQueueMetrics();

  // Remove running containers
  List<RMContainer> runningContainers = node.getRunningContainers();
  for (RMContainer container : runningContainers) {
    completedContainer(container,
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(),
            SchedulerUtils.LOST_CONTAINER),
        RMContainerEventType.KILL);
  }

  // Remove reservations, if any
  RMContainer reservedContainer = node.getReservedContainer();
  if (reservedContainer != null) {
    completedContainer(reservedContainer,
        SchedulerUtils.createAbnormalContainerStatus(
            reservedContainer.getContainerId(),
            SchedulerUtils.LOST_CONTAINER),
        RMContainerEventType.KILL);
  }

  nodes.remove(rmNode.getNodeID());
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
  updateMaximumAllocation(node, false);
  LOG.info("Removed node " + rmNode.getNodeAddress() +
      " cluster capacity: " + clusterResource);
}
 
Example #16
Source File: CSAssignment.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public CSAssignment(FiCaSchedulerApp application, RMContainer excessReservation) {
  this.resource = excessReservation.getContainer().getResource();
  this.type = NodeType.NODE_LOCAL;
  this.application = application;
  this.excessReservation = excessReservation;
  this.skipped = false;
}
 
Example #17
Source File: TestContainerAllocation.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerTokenGeneratedOnPullRequest() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

  RMContainer container =
      rm1.getResourceScheduler().getRMContainer(containerId2);
  // no container token is generated.
  Assert.assertEquals(containerId2, container.getContainerId());
  Assert.assertNull(container.getContainer().getContainerToken());

  // acquire the container.
  List<Container> containers =
      am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId2, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  rm1.stop();
}
 
Example #18
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
synchronized public void containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  
  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);
  
  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);
  
  // Remove from the list of containers
  liveContainers.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  this.attemptResourceUsage.decUsed(containerResource);

  // remove from preemption map if it is completed
  preemptionMap.remove(rmContainer);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;
}
 
Example #19
Source File: FiCaSchedulerApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
synchronized public NodeId getNodeIdToUnreserve(Priority priority,
    Resource resourceNeedUnreserve, ResourceCalculator rc,
    Resource clusterResource) {

  // first go around make this algorithm simple and just grab first
  // reservation that has enough resources
  Map<NodeId, RMContainer> reservedContainers = this.reservedContainers
      .get(priority);

  if ((reservedContainers != null) && (!reservedContainers.isEmpty())) {
    for (Map.Entry<NodeId, RMContainer> entry : reservedContainers.entrySet()) {
      NodeId nodeId = entry.getKey();
      Resource containerResource = entry.getValue().getContainer().getResource();
      
      // make sure we unreserve one with at least the same amount of
      // resources, otherwise could affect capacity limits
      if (Resources.lessThanOrEqual(rc, clusterResource,
          resourceNeedUnreserve, containerResource)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("unreserving node with reservation size: "
              + containerResource
              + " in order to allocate container with size: " + resourceNeedUnreserve);
        }
        return nodeId;
      }
    }
  }
  return null;
}
 
Example #20
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public synchronized boolean unreserve(FiCaSchedulerNode node, Priority priority) {
  Map<NodeId, RMContainer> reservedContainers =
    this.reservedContainers.get(priority);

  if (reservedContainers != null) {
    RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());

    // unreserve is now triggered in new scenarios (preemption)
    // as a consequence reservedcontainer might be null, adding NP-checks
    if (reservedContainer != null
        && reservedContainer.getContainer() != null
        && reservedContainer.getContainer().getResource() != null) {

      if (reservedContainers.isEmpty()) {
        this.reservedContainers.remove(priority);
      }
      // Reset the re-reservation count
      resetReReservations(priority);

      Resource resource = reservedContainer.getContainer().getResource();
      Resources.subtractFrom(currentReservation, resource);

      LOG.info("Application " + getApplicationId() + " unreserved "
          + " on node " + node + ", currently has " + reservedContainers.size()
          + " at priority " + priority + "; currentReservation "
          + currentReservation);
      return true;
    }
  }
  return false;
}
 
Example #21
Source File: MockRM.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public boolean waitForState(MockNM nm, ContainerId containerId,
    RMContainerState containerState, int timeoutMillisecs) throws Exception {
  RMContainer container = getResourceScheduler().getRMContainer(containerId);
  int timeoutSecs = 0;
  while(container == null && timeoutSecs++ < timeoutMillisecs / 100) {
    nm.nodeHeartbeat(true);
    container = getResourceScheduler().getRMContainer(containerId);
    System.out.println("Waiting for container " + containerId + " to be allocated.");
    Thread.sleep(100);
    
    if (timeoutMillisecs <= timeoutSecs * 100) {
      return false;
    }
  }
  Assert.assertNotNull("Container shouldn't be null", container);
  while (!containerState.equals(container.getState())
      && timeoutSecs++ < timeoutMillisecs / 100) {
    System.out.println("Container : " + containerId + " State is : "
        + container.getState() + " Waiting for state : " + containerState);
    nm.nodeHeartbeat(true);
    Thread.sleep(100);
    
    if (timeoutMillisecs <= timeoutSecs * 100) {
      return false;
    }
  }
  
  System.out.println("Container State is : " + container.getState());
  Assert.assertEquals("Container state is not correct (timedout)",
    containerState, container.getState());
  return true;
}
 
Example #22
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Remove the reservation on {@code node} at the given {@link Priority}.
 * This dispatches SchedulerNode handlers as well.
 */
public void unreserve(Priority priority, FSSchedulerNode node) {
  RMContainer rmContainer = node.getReservedContainer();
  unreserveInternal(priority, node);
  node.unreserveResource(this);
  getMetrics().unreserveResource(
      getUser(), rmContainer.getContainer().getResource());
}
 
Example #23
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void dropContainerReservation(RMContainer container) {
  if(LOG.isDebugEnabled()){
    LOG.debug("DROP_RESERVATION:" + container.toString());
  }
  completedContainer(container,
      SchedulerUtils.createAbnormalContainerStatus(
          container.getContainerId(),
          SchedulerUtils.UNRESERVED_CONTAINER),
      RMContainerEventType.KILL);
}
 
Example #24
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    allocateResource(clusterResource, attempt, rmContainer.getContainer()
        .getResource(), node.getLabels());
  }
  getParent().recoverContainer(clusterResource, attempt, rmContainer);
}
 
Example #25
Source File: SchedulerNode.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * The Scheduler has allocated containers on this node to the given
 * application.
 * 
 * @param rmContainer
 *          allocated container
 */
public synchronized void allocateContainer(RMContainer rmContainer) {
  Container container = rmContainer.getContainer();
  deductAvailableResource(container.getResource());
  ++numContainers;

  launchedContainers.put(container.getId(), rmContainer);

  LOG.info("Assigned container " + container.getId() + " of capacity "
      + container.getResource() + " on host " + rmNode.getNodeAddress()
      + ", which has " + numContainers + " containers, "
      + getUsedResource() + " used and " + getAvailableResource()
      + " available after allocation");
}
 
Example #26
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LeafQueue stubLeafQueue(LeafQueue queue) {
  
  // Mock some methods for ease in these unit tests
  
  // 1. LeafQueue.createContainer to return dummy containers
  doAnswer(
      new Answer<Container>() {
        @Override
        public Container answer(InvocationOnMock invocation) 
            throws Throwable {
          final FiCaSchedulerApp application = 
              (FiCaSchedulerApp)(invocation.getArguments()[0]);
          final ContainerId containerId =                 
              TestUtils.getMockContainerId(application);

          Container container = TestUtils.getMockContainer(
              containerId,
              ((FiCaSchedulerNode)(invocation.getArguments()[1])).getNodeID(), 
              (Resource)(invocation.getArguments()[2]),
              ((Priority)invocation.getArguments()[3]));
          return container;
        }
      }
    ).
    when(queue).createContainer(
            any(FiCaSchedulerApp.class), 
            any(FiCaSchedulerNode.class), 
            any(Resource.class),
            any(Priority.class)
            );
  
  // 2. Stub out LeafQueue.parent.completedContainer
  CSQueue parent = queue.getParent();
  doNothing().when(parent).completedContainer(
      any(Resource.class), any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class), 
      any(RMContainer.class), any(ContainerStatus.class), 
      any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());
  
  return queue;
}
 
Example #27
Source File: TestContainerAllocation.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void checkTaskContainersHost(ApplicationAttemptId attemptId,
    ContainerId containerId, ResourceManager rm, String host) {
  YarnScheduler scheduler = rm.getRMContext().getScheduler();
  SchedulerAppReport appReport = scheduler.getSchedulerAppInfo(attemptId);

  Assert.assertTrue(appReport.getLiveContainers().size() > 0);
  for (RMContainer c : appReport.getLiveContainers()) {
    if (c.getContainerId().equals(containerId)) {
      Assert.assertEquals(host, c.getAllocatedNode().getHost());
    }
  }
}
 
Example #28
Source File: FairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void warnOrKillContainer(RMContainer container) {
  ApplicationAttemptId appAttemptId = container.getApplicationAttemptId();
  FSAppAttempt app = getSchedulerApp(appAttemptId);
  FSLeafQueue queue = app.getQueue();
  LOG.info("Preempting container (prio=" + container.getContainer().getPriority() +
      "res=" + container.getContainer().getResource() +
      ") from queue " + queue.getName());
  
  Long time = app.getContainerPreemptionTime(container);

  if (time != null) {
    // if we asked for preemption more than maxWaitTimeBeforeKill ms ago,
    // proceed with kill
    if (time + waitTimeBeforeKill < getClock().getTime()) {
      ContainerStatus status =
        SchedulerUtils.createPreemptedContainerStatus(
          container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER);

      recoverResourceRequestForContainer(container);
      // TODO: Not sure if this ever actually adds this to the list of cleanup
      // containers on the RMNode (see SchedulerNode.releaseContainer()).
      completedContainer(container, status, RMContainerEventType.KILL);
      LOG.info("Killing container" + container +
          " (after waiting for premption for " +
          (getClock().getTime() - time) + "ms)");
    }
  } else {
    // track the request in the FSAppAttempt itself
    app.addPreemption(container, getClock().getTime());
  }
}
 
Example #29
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Resource assignNodeLocalContainers(Resource clusterResource,
    ResourceRequest nodeLocalResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.NODE_LOCAL, 
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        nodeLocalResourceRequest, NodeType.NODE_LOCAL, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 
Example #30
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void releaseContainers(List<ContainerId> containers,
    SchedulerApplicationAttempt attempt) {
  for (ContainerId containerId : containers) {
    RMContainer rmContainer = getRMContainer(containerId);
    if (rmContainer == null) {
      if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp()
          < nmExpireInterval) {
        LOG.info(containerId + " doesn't exist. Add the container"
            + " to the release request cache as it maybe on recovery.");
        synchronized (attempt) {
          attempt.getPendingRelease().add(containerId);
        }
      } else {
        RMAuditLogger.logFailure(attempt.getUser(),
          AuditConstants.RELEASE_CONTAINER,
          "Unauthorized access or invalid container", "Scheduler",
          "Trying to release container not owned by app or with invalid id.",
          attempt.getApplicationId(), containerId);
      }
    }
    
    LOG.info("Container FINISHED by AbstractYarnScheduler by releaseContainers: " + containerId);
    completedContainer(rmContainer,
      SchedulerUtils.createAbnormalContainerStatus(containerId,
        SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED);
  }
}