Java Code Examples for org.apache.hadoop.yarn.api.records.Resource

The following examples show how to use org.apache.hadoop.yarn.api.records.Resource. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-myriad   Source File: OfferUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Transforms a collection of mesos offers into {@link Resource}.
 *
 * @param offers collection of mesos offers
 * @return a single resource object equivalent to the cumulative sum of mesos offers
 */
public static Resource getYarnResourcesFromMesosOffers(Collection<Offer> offers) {
  double cpus = 0.0;
  double mem = 0.0;

  for (Protos.Offer offer : offers) {
    for (Protos.Resource resource : offer.getResourcesList()) {
      if (resource.getName().equalsIgnoreCase("cpus")) {
        cpus += resource.getScalar().getValue();
      } else if (resource.getName().equalsIgnoreCase("mem")) {
        mem += resource.getScalar().getValue();
      }
    }
  }
  return Resource.newInstance((int) mem, (int) cpus);
}
 
Example 2
RMContainer mockContainer(ApplicationAttemptId appAttId, int id,
    Resource r, int cpriority) {
  ContainerId cId = ContainerId.newContainerId(appAttId, id);
  Container c = mock(Container.class);
  when(c.getResource()).thenReturn(r);
  when(c.getPriority()).thenReturn(Priority.create(cpriority));
  RMContainer mC = mock(RMContainer.class);
  when(mC.getContainerId()).thenReturn(cId);
  when(mC.getContainer()).thenReturn(c);
  when(mC.getApplicationAttemptId()).thenReturn(appAttId);
  if (priority.AMCONTAINER.getValue() == cpriority) {
    when(mC.isAMContainer()).thenReturn(true);
  }
  if (priority.LABELEDCONTAINER.getValue() == cpriority) {
    when(mC.getAllocatedNode()).thenReturn(NodeId.newInstance("node1", 0));
  }
  return mC;
}
 
Example 3
Source Project: big-c   Source File: RMAppAttemptBlock.java    License: Apache License 2.0 6 votes vote down vote up
private Resource getTotalResource(List<ResourceRequest> requests) {
  Resource totalResource = Resource.newInstance(0, 0);
  if (requests == null) {
    return totalResource;
  }
  for (ResourceRequest request : requests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    if (request.getResourceName().equals(ResourceRequest.ANY)) {
      Resources.addTo(
        totalResource,
        Resources.multiply(request.getCapability(),
          request.getNumContainers()));
    }
  }
  return totalResource;
}
 
Example 4
Source Project: Bats   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example 5
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  //优先级是否已经达到要求以分配reservation 的 containers
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 
Example 6
Source Project: hadoop   Source File: TestInMemoryReservationAllocation.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBlocks() {
  ReservationId reservationID =
      ReservationId.newInstance(rand.nextLong(), rand.nextLong());
  int[] alloc = { 10, 10, 10, 10, 10, 10 };
  int start = 100;
  ReservationDefinition rDef =
      createSimpleReservationDefinition(start, start + alloc.length + 1,
          alloc.length);
  Map<ReservationInterval, ReservationRequest> allocations =
      generateAllocation(start, alloc, false, false);
  ReservationAllocation rAllocation =
      new InMemoryReservationAllocation(reservationID, rDef, user, planName,
          start, start + alloc.length + 1, allocations, resCalc, minAlloc);
  doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
  Assert.assertFalse(rAllocation.containsGangs());
  for (int i = 0; i < alloc.length; i++) {
    Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])),
        rAllocation.getResourcesAtTime(start + i));
  }
}
 
Example 7
Source Project: hadoop   Source File: QueueMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public void allocateResources(String user, int containers, Resource res,
    boolean decrPending) {
  allocatedContainers.incr(containers);
  aggregateContainersAllocated.incr(containers);
  allocatedMB.incr(res.getMemory() * containers);
  allocatedVCores.incr(res.getVirtualCores() * containers);
  allocatedGCores.incr(res.getGpuCores() * containers);
  if (decrPending) {
    _decrPendingResources(containers, res);
  }
  QueueMetrics userMetrics = getUserMetrics(user);
  if (userMetrics != null) {
    userMetrics.allocateResources(user, containers, res, decrPending);
  }
  if (parent != null) {
    parent.allocateResources(user, containers, res, decrPending);
  }
}
 
Example 8
Source Project: big-c   Source File: CgroupsLCEResourcesHandler.java    License: Apache License 2.0 6 votes vote down vote up
private void setupLimits(ContainerId containerId,
                         Resource containerResource) throws IOException {
  String containerName = containerId.toString();

  if (isCpuWeightEnabled()) {
    int containerVCores = containerResource.getVirtualCores();
    createCgroup(CONTROLLER_CPU, containerName);
    int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
    updateCgroup(CONTROLLER_CPU, containerName, "shares",
        String.valueOf(cpuShares));
    if (strictResourceUsageMode) {
      int nodeVCores =
          conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
      if (nodeVCores != containerVCores) {
        float containerCPU =
            (containerVCores * yarnProcessors) / (float) nodeVCores;
        int[] limits = getOverallLimits(containerCPU);
        updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
          String.valueOf(limits[0]));
        updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US,
          String.valueOf(limits[1]));
      }
    }
  }
}
 
Example 9
/**
 * Gets the resource over usage.
 * This method returns any over usage of resources(vcores and memory) by yarn applications. Criterion for declaring a resource over used is
 * the threshold(memory and vcores threshold) provided by the user. <br>
 * It returns such application IDs along with their respective resource consumptions.
 * 
 * @param memoryThresholdMB the memory threshold mb
 * @param vcoresThreshold the vcores threshold
 * @param rmCommunicator 
 * @return the resource over usage
 */
public Map<String, Map<String, Object>> getResourceOverUsage(
		int memoryThresholdMB, int vcoresThreshold, RMCommunicator rmCommunicator) {
	Resource usedResources = null;
	Map<String, Map<String, Object>> appResourceReports = new HashMap<>(5);
	Map<String, Object> resourceMap = null;
	try {
		for (ApplicationReport report : rmCommunicator.getRunningApplications()) {
			usedResources = report.getApplicationResourceUsageReport().getUsedResources();
			if (usedResources.getMemory() > memoryThresholdMB || usedResources.getVirtualCores() > vcoresThreshold) {
				resourceMap = new HashMap<>(5);
				resourceMap.put(USED_MEMORY, usedResources.getMemory());
				resourceMap.put(USED_VCORES, usedResources.getVirtualCores());
				resourceMap.put(APP_TYPE, report.getApplicationType());
				resourceMap.put(QUEUE, report.getQueue());
				resourceMap.put(USER, report.getUser());
				appResourceReports.put(report.getApplicationId().toString(), resourceMap);
			}
			
		}
	} catch (YarnException | IOException e) {
		LOGGER.error("unable to get running containers", e.getMessage());
	}
	return appResourceReports;
}
 
Example 10
Source Project: big-c   Source File: TestContainerLauncherImpl.java    License: Apache License 2.0 5 votes vote down vote up
private Token createNewContainerToken(ContainerId contId,
    String containerManagerAddr) {
  long currentTime = System.currentTimeMillis();
  return MRApp.newContainerToken(NodeId.newInstance("127.0.0.1",
      1234), "password".getBytes(), new ContainerTokenIdentifier(
      contId, containerManagerAddr, "user",
      Resource.newInstance(1024, 1),
      currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0));
}
 
Example 11
Source Project: big-c   Source File: TestRMNodeLabelsManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testLabelsToNodesOnNodeActiveDeactive() throws Exception {
  // Activate a node without assigning any labels
  mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
  Assert.assertTrue(mgr.getLabelsToNodes().isEmpty());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Add labels and replace labels on node
  mgr.addToCluserNodeLabels(toSet("p1"));
  mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
  // p1 -> n1, n1:1
  Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Activate a node for which host to label mapping exists
  mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0));
  // p1 -> n1, n1:1, n1:2
  Assert.assertEquals(3, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Deactivate a node. n1:1 will be removed from the map
  mgr.deactivateNode(NodeId.newInstance("n1", 1));
  // p1 -> n1, n1:2
  Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
}
 
Example 12
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void updateClusterResource(Resource clusterResource,
    ResourceLimits currentResourceLimits) {
  updateCurrentResourceLimits(currentResourceLimits, clusterResource);
  lastClusterResource = clusterResource;
  updateAbsoluteCapacityResource(clusterResource);
  
  // Update headroom info based on new cluster resource value
  // absoluteMaxCapacity now,  will be replaced with absoluteMaxAvailCapacity
  // during allocation
  setQueueResourceLimitsInfo(clusterResource);
  
  // Update metrics
  CSQueueUtils.updateQueueStatistics(
      resourceCalculator, this, getParent(), clusterResource, 
      minimumAllocation);

  // queue metrics are updated, more resource may be available
  // activate the pending applications if possible
  activateApplications();

  // Update application properties
  for (FiCaSchedulerApp application : activeApplications) {
    synchronized (application) {
      computeUserLimitAndSetHeadroom(application, clusterResource, 
          Resources.none(), null);
    }
  }
}
 
Example 13
Source Project: twill   Source File: ApplicationMasterService.java    License: Apache License 2.0 5 votes vote down vote up
private Resource createCapability(ResourceSpecification resourceSpec) {
  Resource capability = Records.newRecord(Resource.class);

  if (!YarnUtils.setVirtualCores(capability, resourceSpec.getVirtualCores())) {
    LOG.debug("Virtual cores limit not supported.");
  }

  capability.setMemory(resourceSpec.getMemorySize());
  return capability;
}
 
Example 14
Source Project: incubator-tez   Source File: TestTaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  conf = new Configuration();
  taskAttemptListener = mock(TaskAttemptListener.class);
  taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  credentials = new Credentials();
  clock = new SystemClock();
  locationHint = new TaskLocationHint(null, null);

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  dagId = TezDAGID.getInstance(appId, 1);
  vertexId = TezVertexID.getInstance(dagId, 1);
  appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
  mockContainerId = mock(ContainerId.class);
  mockContainer = mock(Container.class);
  mockAMContainer = mock(AMContainer.class);
  mockNodeId = mock(NodeId.class);
  when(mockContainer.getId()).thenReturn(mockContainerId);
  when(mockContainer.getNodeId()).thenReturn(mockNodeId);
  when(mockAMContainer.getContainer()).thenReturn(mockContainer);
  when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
  taskResource = Resource.newInstance(1024, 1);
  localResources = new HashMap<String, LocalResource>();
  environment = new HashMap<String, String>();
  javaOpts = "";
  leafVertex = false;
  containerContext = new ContainerContext(localResources, credentials,
      environment, javaOpts);
  Vertex vertex = mock(Vertex.class);
  eventHandler = new TestEventHandler();
  
  mockTask = new MockTaskImpl(vertexId, partition,
      eventHandler, conf, taskAttemptListener, clock,
      taskHeartbeatHandler, appContext, leafVertex, locationHint,
      taskResource, containerContext, vertex);
}
 
Example 15
Source Project: submarine   Source File: TensorFlowRunJobParameters.java    License: Apache License 2.0 5 votes vote down vote up
private Resource determinePSResource(Parameter parametersHolder,
    int nPS, ClientContext clientContext)
    throws ParseException, YarnException, IOException {
  if (nPS > 0) {
    String psResourceStr =
        parametersHolder.getOptionValue(CliConstants.PS_RES);
    if (psResourceStr == null) {
      throw new ParseException("--" + CliConstants.PS_RES + " is absent.");
    }
    return ResourceUtils.createResourceFromString(psResourceStr);
  }
  return null;
}
 
Example 16
Source Project: hadoop   Source File: ProtocolHATestBase.java    License: Apache License 2.0 5 votes vote down vote up
public AllocateResponse createFakeAllocateResponse() {
  return AllocateResponse.newInstance(-1,
      new ArrayList<ContainerStatus>(),
      new ArrayList<Container>(), new ArrayList<NodeReport>(),
      Resource.newInstance(1024, 2, 2), null, 1,
      null, new ArrayList<NMToken>());
}
 
Example 17
Source Project: hadoop   Source File: TestUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
 
Example 18
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
private void setQueueResourceLimitsInfo(
    Resource clusterResource) {
  synchronized (queueResourceLimitsInfo) {
    queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom
        .getLimit());
    queueResourceLimitsInfo.setClusterResource(clusterResource);
  }
}
 
Example 19
Source Project: big-c   Source File: TestLeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
    throws Exception {
  CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
  Resource clusterResource = Resources
      .createResource(100 * 16 * GB, 100 * 32);
  CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
  when(csContext.getRMContext()).thenReturn(rmContext);
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
  ParentQueue root = new ParentQueue(csContext, 
      CapacitySchedulerConfiguration.ROOT, null, null);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
  LeafQueue a = new LeafQueue(csContext, A, root, null);
  assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
  
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
  LeafQueue newA = new LeafQueue(csContext, A, root, null);
  a.reinitialize(newA, clusterResource);
  assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));

  Resource newClusterResource = Resources.createResource(100 * 20 * GB,
      100 * 32);
  a.updateClusterResource(newClusterResource, 
      new ResourceLimits(newClusterResource));
  //  100 * 20 * 0.2 = 400
  assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
}
 
Example 20
Source Project: big-c   Source File: NodeInfo.java    License: Apache License 2.0 5 votes vote down vote up
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
    Resource perNode, String rackName, String healthReport,
    int cmdPort, String hostName, NodeState state) {
  this.nodeId = nodeId;
  this.nodeAddr = nodeAddr;
  this.httpAddress = httpAddress;
  this.perNode = perNode;
  this.rackName = rackName;
  this.healthReport = healthReport;
  this.cmdPort = cmdPort;
  this.hostName = hostName;
  this.state = state;
  toCleanUpApplications = new ArrayList<ApplicationId>();
  toCleanUpContainers = new ArrayList<ContainerId>();
}
 
Example 21
Source Project: hadoop   Source File: DominantResourceCalculator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public float ratio(Resource a, Resource b) {
  float max = Math.max(
      (float) a.getMemory() / b.getMemory(),
      (float) a.getVirtualCores() / b.getVirtualCores());
  if (b.getGpuCores() != 0) {
    max = Math.max(max,
        (float) a.getGpuCores() / b.getGpuCores());
  }
  return max;
}
 
Example 22
Source Project: hadoop   Source File: InMemoryPlan.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setTotalCapacity(Resource cap) {
  writeLock.lock();
  try {
    totalCapacity = Resources.clone(cap);
  } finally {
    writeLock.unlock();
  }
}
 
Example 23
Source Project: flink   Source File: WorkerSpecContainerResourceAdapter.java    License: Apache License 2.0 5 votes vote down vote up
Optional<Resource> tryComputeContainerResource(final WorkerResourceSpec workerResourceSpec) {
	final InternalContainerResource internalContainerResource = workerSpecToContainerResource.computeIfAbsent(
		Preconditions.checkNotNull(workerResourceSpec),
		this::createAndMapContainerResource);
	if (internalContainerResource != null) {
		return Optional.of(internalContainerResource.toResource());
	} else {
		return Optional.empty();
	}
}
 
Example 24
Source Project: incubator-myriad   Source File: NMHeartBeatHandler.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 
Example 25
Source Project: big-c   Source File: ContainerStartDataPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Resource getAllocatedResource() {
  if (this.resource != null) {
    return this.resource;
  }
  ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAllocatedResource()) {
    return null;
  }
  this.resource = convertFromProtoFormat(p.getAllocatedResource());
  return this.resource;
}
 
Example 26
Source Project: hadoop   Source File: TestInMemoryReservationAllocation.java    License: Apache License 2.0 5 votes vote down vote up
private ReservationDefinition createSimpleReservationDefinition(long arrival,
    long deadline, long duration) {
  // create a request with a single atomic ask
  ReservationRequest r =
      ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), 1, 1,
          duration);
  ReservationDefinition rDef = new ReservationDefinitionPBImpl();
  ReservationRequests reqs = new ReservationRequestsPBImpl();
  reqs.setReservationResources(Collections.singletonList(r));
  reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
  rDef.setReservationRequests(reqs);
  rDef.setArrival(arrival);
  rDef.setDeadline(deadline);
  return rDef;
}
 
Example 27
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
    registerNode(String hostName, int containerManagerPort, int httpPort,
        String rackName, Resource capability)
        throws IOException, YarnException {
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm =
      new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
          hostName, containerManagerPort, httpPort, rackName, capability,
          resourceManager);
  NodeAddedSchedulerEvent nodeAddEvent1 = 
      new NodeAddedSchedulerEvent(resourceManager.getRMContext()
          .getRMNodes().get(nm.getNodeId()));
  resourceManager.getResourceScheduler().handle(nodeAddEvent1);
  return nm;
}
 
Example 28
Source Project: big-c   Source File: TestApplicationLimits.java    License: Apache License 2.0 5 votes vote down vote up
private FiCaSchedulerApp getMockApplication(int appId, String user,
  Resource amResource) {
  FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
  ApplicationAttemptId applicationAttemptId =
      TestUtils.getMockApplicationAttemptId(appId, 0);
  doReturn(applicationAttemptId.getApplicationId()).
      when(application).getApplicationId();
  doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
  doReturn(user).when(application).getUser();
  doReturn(amResource).when(application).getAMResource();
  return application;
}
 
Example 29
Source Project: tez   Source File: TestTaskSchedulerHelpers.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 
Example 30
Source Project: tez   Source File: TestDAGImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testDAGInitFailedDuetoInvalidResource() {
  // cluster maxContainerCapability is less than the vertex resource request
  ClusterInfo clusterInfo = new ClusterInfo(Resource.newInstance(512,10));
  doReturn(clusterInfo).when(appContext).getClusterInfo();
  dag.handle(
      new DAGEvent(dag.getID(), DAGEventType.DAG_INIT));
  dispatcher.await();
  Assert.assertEquals(DAGState.FAILED, dag.getState());
  Assert.assertEquals(DAGTerminationCause.INIT_FAILURE, dag.getTerminationCause());
  Assert.assertTrue(StringUtils.join(dag.getDiagnostics(), ",")
      .contains("Vertex's TaskResource is beyond the cluster container capability"));
}