Java Code Examples for org.apache.hadoop.yarn.api.records.Priority

The following examples show how to use org.apache.hadoop.yarn.api.records.Priority. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example 2
Source Project: ambari-metrics   Source File: ContainerHistoryData.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Unstable
public static ContainerHistoryData newInstance(ContainerId containerId,
    Resource allocatedResource, NodeId assignedNode, Priority priority,
    long startTime, long finishTime, String diagnosticsInfo,
    int containerExitCode, ContainerState containerState) {
  ContainerHistoryData containerHD = new ContainerHistoryData();
  containerHD.setContainerId(containerId);
  containerHD.setAllocatedResource(allocatedResource);
  containerHD.setAssignedNode(assignedNode);
  containerHD.setPriority(priority);
  containerHD.setStartTime(startTime);
  containerHD.setFinishTime(finishTime);
  containerHD.setDiagnosticsInfo(diagnosticsInfo);
  containerHD.setContainerExitStatus(containerExitCode);
  containerHD.setContainerState(containerState);
  return containerHD;
}
 
Example 3
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private int assignRackLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getRMNode().getRackName());
  if (request != null) {
    // Don't allocate on this rack if the application doens't need containers
    ResourceRequest offSwitchRequest =
        application.getResourceRequest(priority, ResourceRequest.ANY);
    if (offSwitchRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.RACK_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.RACK_LOCAL);
  }
  return assignedContainers;
}
 
Example 4
Source Project: incubator-gobblin   Source File: YarnService.java    License: Apache License 2.0 6 votes vote down vote up
private void requestContainer(Optional<String> preferredNode) {
  Priority priority = Records.newRecord(Priority.class);
  priority.setPriority(0);

  Resource capability = Records.newRecord(Resource.class);
  int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
  capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ?
      this.requestedContainerMemoryMbs : maxMemoryCapacity);
  int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
  capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ?
      this.requestedContainerCores : maxCoreCapacity);

  String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null;
  this.amrmClientAsync.addContainerRequest(
      new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority));
}
 
Example 5
Source Project: hadoop   Source File: AMRMClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Instantiates a {@link ContainerRequest} with the given constraints.
 * 
 * @param capability
 *          The {@link Resource} to be requested for each container.
 * @param nodes
 *          Any hosts to request that the containers are placed on.
 * @param racks
 *          Any racks to request that the containers are placed on. The
 *          racks corresponding to any hosts requested will be automatically
 *          added to this list.
 * @param priority
 *          The priority at which to request the containers. Higher
 *          priorities have lower numerical values.
 * @param relaxLocality
 *          If true, containers for this request may be assigned on hosts
 *          and racks other than the ones explicitly requested.
 * @param nodeLabelsExpression
 *          Set node labels to allocate resource, now we only support
 *          asking for only a single node label
 */
public ContainerRequest(Resource capability, String[] nodes,
    String[] racks, Priority priority, boolean relaxLocality,
    String nodeLabelsExpression) {
  // Validate request
  Preconditions.checkArgument(capability != null,
      "The Resource to be requested for each container " +
          "should not be null ");
  Preconditions.checkArgument(priority != null,
      "The priority at which to request containers should not be null ");
  Preconditions.checkArgument(
          !(!relaxLocality && (racks == null || racks.length == 0) 
              && (nodes == null || nodes.length == 0)),
          "Can't turn off locality relaxation on a " + 
          "request with no location constraints");
  this.capability = capability;
  this.nodes = (nodes != null ? ImmutableList.copyOf(nodes) : null);
  this.racks = (racks != null ? ImmutableList.copyOf(racks) : null);
  this.priority = priority;
  this.relaxLocality = relaxLocality;
  this.nodeLabelsExpression = nodeLabelsExpression;
}
 
Example 6
Source Project: hadoop   Source File: LeafQueue.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 
Example 7
Source Project: incubator-tez   Source File: YarnTaskSchedulerService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void allocateTask(
    Object task,
    Resource capability,
    String[] hosts,
    String[] racks,
    Priority priority,
    Object containerSignature,
    Object clientCookie) {

  // XXX Have ContainerContext implement an interface defined by TaskScheduler.
  // TODO check for nulls etc
  // TODO extra memory allocation
  CRCookie cookie = new CRCookie(task, clientCookie, containerSignature);
  CookieContainerRequest request = new CookieContainerRequest(
    capability, hosts, racks, priority, cookie);

  addRequestAndTrigger(task, request, hosts, racks);
}
 
Example 8
Source Project: tez   Source File: YarnTaskSchedulerService.java    License: Apache License 2.0 6 votes vote down vote up
private CookieContainerRequest getMatchingRequestWithPriority(
    Container container,
    String location) {
  Priority priority = container.getPriority();
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> requestsList =
      amRmClient.getMatchingRequests(priority, location, capability);

  if (!requestsList.isEmpty()) {
    // pick first one
    for (Collection<CookieContainerRequest> requests : requestsList) {
      for (CookieContainerRequest cookieContainerRequest : requests) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          return cookieContainerRequest;
        }
      }
    }
  }

  return null;
}
 
Example 9
Source Project: hadoop   Source File: FiCaSchedulerApp.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 
Example 10
Source Project: big-c   Source File: FiCaSchedulerApp.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 
Example 11
Source Project: tez   Source File: DagAwareYarnTaskScheduler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Removes a vertex from a RequestPriorityStats.
 *
 * Removing a vertex is more expensive than adding a vertex. The stats contain bitmasks which only store on/off
 * values rather than reference counts. Therefore we must rebuild the descendants bitmasks from the remaining
 * vertices in the request stats. Once the new descendants mask is computed we then need to rebuild the
 * allowedVertices BitSet for all lower priority request stats in case the removal of this vertex unblocks lower
 * priority requests of a descendant vertex.
 *
 * Rebuilding allowedVertices for the lower priorities involves starting with the allowedVertices mask at the
 * current priority then masking off the descendants at each priority level encountered, accumulating the results.
 * Any descendants of a level will be blocked at all lower levels. See the addVertexToRequestStats documentation
 * for details on how vertices map to the descendants and allowedVertices bit masks.
 */
private void removeVertexFromRequestStats(Priority priority, RequestPriorityStats stats, Integer vertexIndexInt) {
  stats.vertexTaskCount.remove(vertexIndexInt);
  int vertexIndex = vertexIndexInt;
  stats.vertices.clear(vertexIndex);

  // Rebuild the descendants BitSet for the remaining vertices at this priority.
  stats.descendants.clear();
  for (Integer vIndex : stats.vertexTaskCount.keySet()) {
    stats.descendants.or(vertexDescendants.get(vIndex));
  }

  // The allowedVertices for all lower priorities need to be recalculated where the vertex descendants at each
  // level are removed from the list of allowed vertices at all subsequent levels.
  Collection<RequestPriorityStats> tailStats = priorityStats.tailMap(priority, false).values();
  if (!tailStats.isEmpty()) {
    BitSet cumulativeAllowed = new BitSet(vertexDescendants.size());
    cumulativeAllowed.or(stats.allowedVertices);
    cumulativeAllowed.andNot(stats.descendants);
    for (RequestPriorityStats s : tailStats) {
      s.allowedVertices.clear();
      s.allowedVertices.or(cumulativeAllowed);
      cumulativeAllowed.andNot(s.descendants);
    }
  }
}
 
Example 12
Source Project: hadoop   Source File: TestContainerAllocation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected RMContainerTokenSecretManager createContainerTokenSecretManager(
    Configuration conf) {
  return new RMContainerTokenSecretManager(conf) {

    @Override
    public Token createContainerToken(ContainerId containerId,
        NodeId nodeId, String appSubmitter, Resource capability,
        Priority priority, long createTime,
        LogAggregationContext logAggregationContext) {
      numRetries++;
      return super.createContainerToken(containerId, nodeId, appSubmitter,
        capability, priority, createTime, logAggregationContext);
    }
  };
}
 
Example 13
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example 14
Source Project: big-c   Source File: AppSchedulingInfo.java    License: Apache License 2.0 5 votes vote down vote up
synchronized private void checkForDeactivation() {
  boolean deactivate = true;
  for (Priority priority : getPriorities()) {
    ResourceRequest request = getResourceRequest(priority, ResourceRequest.ANY);
    if (request != null) {
      if (request.getNumContainers() > 0) {
        deactivate = false;
        break;
      }
    }
  }
  if (deactivate) {
    activeUsersManager.deactivateApplication(user, applicationId);
  }
}
 
Example 15
Source Project: big-c   Source File: NMContainerStatus.java    License: Apache License 2.0 5 votes vote down vote up
public static NMContainerStatus newInstance(ContainerId containerId,
    ContainerState containerState, Resource allocatedResource,
    String diagnostics, int containerExitStatus, Priority priority,
    long creationTime) {
  NMContainerStatus status =
      Records.newRecord(NMContainerStatus.class);
  status.setContainerId(containerId);
  status.setContainerState(containerState);
  status.setAllocatedResource(allocatedResource);
  status.setDiagnostics(diagnostics);
  status.setContainerExitStatus(containerExitStatus);
  status.setPriority(priority);
  status.setCreationTime(creationTime);
  return status;
}
 
Example 16
Source Project: tez   Source File: DagAwareYarnTaskScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@GuardedBy("this")
@Nullable
private TaskRequest tryAssignReuseContainerAppRunning(HeldContainer hc) {
  if (!hc.isAssignable()) {
    LOG.debug("Skipping scheduling of container {} because it state is {}", hc.getId(), hc.getState());
    return null;
  }

  TaskRequest assignedRequest = tryAssignReuseContainerForAffinity(hc);
  if (assignedRequest != null) {
    return assignedRequest;
  }

  for (Entry<Priority,RequestPriorityStats> entry : requestTracker.getStatsEntries()) {
    Priority priority = entry.getKey();
    RequestPriorityStats stats = entry.getValue();
    if (!stats.allowedVertices.intersects(stats.vertices)) {
      LOG.debug("Skipping requests at priority {} because all requesting vertices are blocked by higher priority requests",
          priority);
      continue;
    }

    String matchLocation = hc.getMatchingLocation();
    if (stats.localityCount <= 0) {
      LOG.debug("Overriding locality match of container {} to ANY since there are no locality requests at priority {}",
          hc.getId(), priority);
      matchLocation = ResourceRequest.ANY;
    }
    assignedRequest = tryAssignReuseContainerForPriority(hc, matchLocation,
        priority, stats.allowedVertices);
    if (assignedRequest != null) {
      break;
    }
  }
  return assignedRequest;
}
 
Example 17
Source Project: hadoop   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
private Resource assignNodeLocalContainers(Resource clusterResource,
    ResourceRequest nodeLocalResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.NODE_LOCAL, 
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        nodeLocalResourceRequest, NodeType.NODE_LOCAL, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 
Example 18
private ContainerRequest setupContainerReqForRM() {
  // Priority for worker containers - priorities are intra-application
  Priority priority = Records.newRecord(Priority.class);
  priority.setPriority(0);
  // Resource requirements for worker containers
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(containerMem);
  //capability.setVirtualCores(1);
  ContainerRequest containerReq = new ContainerRequest(
      capability,
      null /* hosts String[] */,
      null /* racks String [] */,
      priority);
  return containerReq;
}
 
Example 19
Source Project: flink   Source File: TestingContainer.java    License: Apache License 2.0 5 votes vote down vote up
TestingContainer(
	final ContainerId containerId,
	final NodeId nodeId,
	final Resource resource,
	final Priority priority) {

	this.containerId = containerId;
	this.nodeId = nodeId;
	this.resource = resource;
	this.priority = priority;
}
 
Example 20
Source Project: hadoop   Source File: RMContainerRequestor.java    License: Apache License 2.0 5 votes vote down vote up
public ContainerRequest(TaskAttemptId attemptID,
    Resource capability, String[] hosts, String[] racks,
    Priority priority, long requestTimeMs,String nodeLabelExpression) {
  this.attemptID = attemptID;
  this.capability = capability;
  this.hosts = hosts;
  this.racks = racks;
  this.priority = priority;
  this.requestTimeMs = requestTimeMs;
  this.nodeLabelExpression = nodeLabelExpression;
}
 
Example 21
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Token newContainerToken(ContainerId cId, String host,
    int port, String user, Resource r, long expiryTime, int masterKeyId,
    byte[] password, long rmIdentifier) throws IOException {
  ContainerTokenIdentifier identifier =
      new ContainerTokenIdentifier(cId, host + ":" + port, user, r,
        expiryTime, masterKeyId, rmIdentifier, Priority.newInstance(0), 0);
  return newContainerToken(BuilderUtils.newNodeId(host, port), password,
      identifier);
}
 
Example 22
Source Project: incubator-tez   Source File: TezAMRMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized List<? extends Collection<T>>
  getMatchingRequestsForTopPriority(
      String resourceName, Resource capability) {
  // Sort based on reverse order. By default, Priority ordering is based on
  // highest numeric value being considered to be lowest priority.
  Iterator<Priority> iter =
    knownRequestsByPriority.descendingKeySet().iterator();
  if (!iter.hasNext()) {
    return Collections.emptyList();
  }
  Priority p = iter.next();
  LocalityRequestCounter lrc = knownRequestsByPriority.get(p);
  if (lrc.localityRequests.get() == 0) {
    // Fallback to ANY if there are no pending requests that require
    // locality matching
    if (LOG.isDebugEnabled()) {
      LOG.debug("Over-ridding location request for matching containers as"
        + " there are no pending requests that require locality at this"
        + " priority"
        + ", priority=" + p
        + ", localityRequests=" + lrc.localityRequests
        + ", noLocalityRequests=" + lrc.noLocalityRequests);
    }
    resourceName = ResourceRequest.ANY;
  }
  List<? extends Collection<T>> matched =
    getMatchingRequests(p, resourceName, capability);
  if (matched != null && !matched.isEmpty()) {
    return matched;
  }
  return Collections.emptyList();
}
 
Example 23
Source Project: hadoop   Source File: TestReservations.java    License: Apache License 2.0 5 votes vote down vote up
static LeafQueue stubLeafQueue(LeafQueue queue) {

    // Mock some methods for ease in these unit tests

    // 1. LeafQueue.createContainer to return dummy containers
    doAnswer(new Answer<Container>() {
      @Override
      public Container answer(InvocationOnMock invocation) throws Throwable {
        final FiCaSchedulerApp application = (FiCaSchedulerApp) (invocation
            .getArguments()[0]);
        final ContainerId containerId = TestUtils
            .getMockContainerId(application);

        Container container = TestUtils.getMockContainer(containerId,
            ((FiCaSchedulerNode) (invocation.getArguments()[1])).getNodeID(),
            (Resource) (invocation.getArguments()[2]),
            ((Priority) invocation.getArguments()[3]));
        return container;
      }
    }).when(queue).createContainer(any(FiCaSchedulerApp.class),
        any(FiCaSchedulerNode.class), any(Resource.class), any(Priority.class));

    // 2. Stub out LeafQueue.parent.completedContainer
    CSQueue parent = queue.getParent();
    doNothing().when(parent).completedContainer(any(Resource.class),
        any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class),
        any(RMContainer.class), any(ContainerStatus.class),
        any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());

    return queue;
  }
 
Example 24
Source Project: hadoop   Source File: TestLeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
static LeafQueue stubLeafQueue(LeafQueue queue) {
  
  // Mock some methods for ease in these unit tests
  
  // 1. LeafQueue.createContainer to return dummy containers
  doAnswer(
      new Answer<Container>() {
        @Override
        public Container answer(InvocationOnMock invocation) 
            throws Throwable {
          final FiCaSchedulerApp application = 
              (FiCaSchedulerApp)(invocation.getArguments()[0]);
          final ContainerId containerId =                 
              TestUtils.getMockContainerId(application);

          Container container = TestUtils.getMockContainer(
              containerId,
              ((FiCaSchedulerNode)(invocation.getArguments()[1])).getNodeID(), 
              (Resource)(invocation.getArguments()[2]),
              ((Priority)invocation.getArguments()[3]));
          return container;
        }
      }
    ).
    when(queue).createContainer(
            any(FiCaSchedulerApp.class), 
            any(FiCaSchedulerNode.class), 
            any(Resource.class),
            any(Priority.class)
            );
  
  // 2. Stub out LeafQueue.parent.completedContainer
  CSQueue parent = queue.getParent();
  doNothing().when(parent).completedContainer(
      any(Resource.class), any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class), 
      any(RMContainer.class), any(ContainerStatus.class), 
      any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());
  
  return queue;
}
 
Example 25
Source Project: hadoop   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Container newContainer(ContainerId containerId, NodeId nodeId,
    String nodeHttpAddress, Resource resource, Priority priority,
    Token containerToken) {
  Container container = recordFactory.newRecordInstance(Container.class);
  container.setId(containerId);
  container.setNodeId(nodeId);
  container.setNodeHttpAddress(nodeHttpAddress);
  container.setResource(resource);
  container.setPriority(priority);
  container.setContainerToken(containerToken);
  return container;
}
 
Example 26
Source Project: incubator-tez   Source File: TestTezAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=10000)
public void testMatchingRequestsForTopPriority() {
  String[] hosts = { "host1" };
  String[] racks = { "rack1" };
  AMRMClient.ContainerRequest req1 = new AMRMClient.ContainerRequest(
    Resource.newInstance(2048, 1), hosts, racks,
    Priority.newInstance(1));
  AMRMClient.ContainerRequest req2 = new AMRMClient.ContainerRequest(
    Resource.newInstance(1024, 1), hosts, racks,
    Priority.newInstance(2));
  AMRMClient.ContainerRequest req3 = new AMRMClient.ContainerRequest(
    Resource.newInstance(1024, 1), hosts, racks,
    Priority.newInstance(3));
  amrmClient.addContainerRequest(req1);
  amrmClient.addContainerRequest(req2);
  amrmClient.addContainerRequest(req3);

  Assert.assertTrue(amrmClient.getMatchingRequestsForTopPriority("host1",
    Resource.newInstance(1024, 1)).isEmpty());

  List<? extends Collection<AMRMClient.ContainerRequest>> ret =
    amrmClient.getMatchingRequestsForTopPriority("host1",
      Resource.newInstance(2048, 1));
  Assert.assertFalse(ret.isEmpty());
  Assert.assertEquals(req1, ret.get(0).iterator().next());

  amrmClient.removeContainerRequest(req1);

  ret = amrmClient.getMatchingRequestsForTopPriority("host1",
      Resource.newInstance(1024, 1));
  Assert.assertFalse(ret.isEmpty());
  Assert.assertEquals(req2, ret.get(0).iterator().next());
}
 
Example 27
Source Project: hadoop   Source File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=30000)
public void testAskWithInvalidNodeLabels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();

  // specified exp with more than one node labels
  verifyAddRequestFailed(client,
      new ContainerRequest(Resource.newInstance(1024, 1), null, null,
          Priority.UNDEFINED, true, "x && y"));
}
 
Example 28
Source Project: hadoop   Source File: TestProtocolRecords.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRegisterNodeManagerRequest() {
  ApplicationId appId = ApplicationId.newInstance(123456789, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, 1);

  NMContainerStatus containerReport =
      NMContainerStatus.newInstance(containerId,
        ContainerState.RUNNING, Resource.newInstance(1024, 1, 2), "diagnostics",
        0, Priority.newInstance(10), 1234);
  List<NMContainerStatus> reports = Arrays.asList(containerReport);
  RegisterNodeManagerRequest request =
      RegisterNodeManagerRequest.newInstance(
        NodeId.newInstance("1.1.1.1", 1000), 8080,
          Resource.newInstance(1024, 1, 2), "NM-version-id", reports,
          Arrays.asList(appId));
  RegisterNodeManagerRequest requestProto =
      new RegisterNodeManagerRequestPBImpl(
        ((RegisterNodeManagerRequestPBImpl) request).getProto());
  Assert.assertEquals(containerReport, requestProto
    .getNMContainerStatuses().get(0));
  Assert.assertEquals(8080, requestProto.getHttpPort());
  Assert.assertEquals("NM-version-id", requestProto.getNMVersion());
  Assert.assertEquals(NodeId.newInstance("1.1.1.1", 1000),
    requestProto.getNodeId());
  Assert.assertEquals(Resource.newInstance(1024, 1, 2),
    requestProto.getResource());
  Assert.assertEquals(1, requestProto.getRunningApplications().size());
  Assert.assertEquals(appId, requestProto.getRunningApplications().get(0)); 
}
 
Example 29
Source Project: big-c   Source File: FSAppAttempt.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create and return a container object reflecting an allocation for the
 * given appliction on the given node with the given capability and
 * priority.
 */
public Container createContainer(
    FSSchedulerNode node, Resource capability, Priority priority) {

  NodeId nodeId = node.getRMNode().getNodeID();
  ContainerId containerId = BuilderUtils.newContainerId(
      getApplicationAttemptId(), getNewContainerId());

  // Create the container
  Container container =
      BuilderUtils.newContainer(containerId, nodeId, node.getRMNode()
          .getHttpAddress(), capability, priority, null);

  return container;
}
 
Example 30
Source Project: incubator-tez   Source File: LocalTaskSchedulerService.java    License: Apache License 2.0 5 votes vote down vote up
public Container createContainer(Resource capability, Priority priority) {
  ApplicationAttemptId appAttemptId = appContext.getApplicationAttemptId();
  ContainerId containerId = ContainerId.newInstance(appAttemptId, nextId.getAndIncrement());
  NodeId nodeId = NodeId.newInstance("127.0.0.1", 0);
  String nodeHttpAddress = "127.0.0.1:0";

  Container container = Container.newInstance(containerId,
      nodeId,
      nodeHttpAddress,
      capability,
      priority,
      null);

  return container;
}