org.apache.hadoop.yarn.api.records.Resource Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.Resource. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LeafQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  //优先级是否已经达到要求以分配reservation 的 containers
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 
Example #2
Source File: OfferUtils.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
/**
 * Transforms a collection of mesos offers into {@link Resource}.
 *
 * @param offers collection of mesos offers
 * @return a single resource object equivalent to the cumulative sum of mesos offers
 */
public static Resource getYarnResourcesFromMesosOffers(Collection<Offer> offers) {
  double cpus = 0.0;
  double mem = 0.0;

  for (Protos.Offer offer : offers) {
    for (Protos.Resource resource : offer.getResourcesList()) {
      if (resource.getName().equalsIgnoreCase("cpus")) {
        cpus += resource.getScalar().getValue();
      } else if (resource.getName().equalsIgnoreCase("mem")) {
        mem += resource.getScalar().getValue();
      }
    }
  }
  return Resource.newInstance((int) mem, (int) cpus);
}
 
Example #3
Source File: RMAppAttemptBlock.java    From big-c with Apache License 2.0 6 votes vote down vote up
private Resource getTotalResource(List<ResourceRequest> requests) {
  Resource totalResource = Resource.newInstance(0, 0);
  if (requests == null) {
    return totalResource;
  }
  for (ResourceRequest request : requests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    if (request.getResourceName().equals(ResourceRequest.ANY)) {
      Resources.addTo(
        totalResource,
        Resources.multiply(request.getCapability(),
          request.getNumContainers()));
    }
  }
  return totalResource;
}
 
Example #4
Source File: CgroupsLCEResourcesHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void setupLimits(ContainerId containerId,
                         Resource containerResource) throws IOException {
  String containerName = containerId.toString();

  if (isCpuWeightEnabled()) {
    int containerVCores = containerResource.getVirtualCores();
    createCgroup(CONTROLLER_CPU, containerName);
    int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
    updateCgroup(CONTROLLER_CPU, containerName, "shares",
        String.valueOf(cpuShares));
    if (strictResourceUsageMode) {
      int nodeVCores =
          conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
      if (nodeVCores != containerVCores) {
        float containerCPU =
            (containerVCores * yarnProcessors) / (float) nodeVCores;
        int[] limits = getOverallLimits(containerCPU);
        updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
          String.valueOf(limits[0]));
        updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US,
          String.valueOf(limits[1]));
      }
    }
  }
}
 
Example #5
Source File: QueueMetrics.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void allocateResources(String user, int containers, Resource res,
    boolean decrPending) {
  allocatedContainers.incr(containers);
  aggregateContainersAllocated.incr(containers);
  allocatedMB.incr(res.getMemory() * containers);
  allocatedVCores.incr(res.getVirtualCores() * containers);
  allocatedGCores.incr(res.getGpuCores() * containers);
  if (decrPending) {
    _decrPendingResources(containers, res);
  }
  QueueMetrics userMetrics = getUserMetrics(user);
  if (userMetrics != null) {
    userMetrics.allocateResources(user, containers, res, decrPending);
  }
  if (parent != null) {
    parent.allocateResources(user, containers, res, decrPending);
  }
}
 
Example #6
Source File: TestProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
RMContainer mockContainer(ApplicationAttemptId appAttId, int id,
    Resource r, int cpriority) {
  ContainerId cId = ContainerId.newContainerId(appAttId, id);
  Container c = mock(Container.class);
  when(c.getResource()).thenReturn(r);
  when(c.getPriority()).thenReturn(Priority.create(cpriority));
  RMContainer mC = mock(RMContainer.class);
  when(mC.getContainerId()).thenReturn(cId);
  when(mC.getContainer()).thenReturn(c);
  when(mC.getApplicationAttemptId()).thenReturn(appAttId);
  if (priority.AMCONTAINER.getValue() == cpriority) {
    when(mC.isAMContainer()).thenReturn(true);
  }
  if (priority.LABELEDCONTAINER.getValue() == cpriority) {
    when(mC.getAllocatedNode()).thenReturn(NodeId.newInstance("node1", 0));
  }
  return mC;
}
 
Example #7
Source File: ClusterAnalysisMetrics.java    From jumbune with GNU Lesser General Public License v3.0 6 votes vote down vote up
/**
 * Gets the resource over usage.
 * This method returns any over usage of resources(vcores and memory) by yarn applications. Criterion for declaring a resource over used is
 * the threshold(memory and vcores threshold) provided by the user. <br>
 * It returns such application IDs along with their respective resource consumptions.
 * 
 * @param memoryThresholdMB the memory threshold mb
 * @param vcoresThreshold the vcores threshold
 * @param rmCommunicator 
 * @return the resource over usage
 */
public Map<String, Map<String, Object>> getResourceOverUsage(
		int memoryThresholdMB, int vcoresThreshold, RMCommunicator rmCommunicator) {
	Resource usedResources = null;
	Map<String, Map<String, Object>> appResourceReports = new HashMap<>(5);
	Map<String, Object> resourceMap = null;
	try {
		for (ApplicationReport report : rmCommunicator.getRunningApplications()) {
			usedResources = report.getApplicationResourceUsageReport().getUsedResources();
			if (usedResources.getMemory() > memoryThresholdMB || usedResources.getVirtualCores() > vcoresThreshold) {
				resourceMap = new HashMap<>(5);
				resourceMap.put(USED_MEMORY, usedResources.getMemory());
				resourceMap.put(USED_VCORES, usedResources.getVirtualCores());
				resourceMap.put(APP_TYPE, report.getApplicationType());
				resourceMap.put(QUEUE, report.getQueue());
				resourceMap.put(USER, report.getUser());
				appResourceReports.put(report.getApplicationId().toString(), resourceMap);
			}
			
		}
	} catch (YarnException | IOException e) {
		LOGGER.error("unable to get running containers", e.getMessage());
	}
	return appResourceReports;
}
 
Example #8
Source File: TestInMemoryReservationAllocation.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testBlocks() {
  ReservationId reservationID =
      ReservationId.newInstance(rand.nextLong(), rand.nextLong());
  int[] alloc = { 10, 10, 10, 10, 10, 10 };
  int start = 100;
  ReservationDefinition rDef =
      createSimpleReservationDefinition(start, start + alloc.length + 1,
          alloc.length);
  Map<ReservationInterval, ReservationRequest> allocations =
      generateAllocation(start, alloc, false, false);
  ReservationAllocation rAllocation =
      new InMemoryReservationAllocation(reservationID, rDef, user, planName,
          start, start + alloc.length + 1, allocations, resCalc, minAlloc);
  doAssertions(rAllocation, reservationID, rDef, allocations, start, alloc);
  Assert.assertFalse(rAllocation.containsGangs());
  for (int i = 0; i < alloc.length; i++) {
    Assert.assertEquals(Resource.newInstance(1024 * (alloc[i]), (alloc[i]), (alloc[i])),
        rAllocation.getResourcesAtTime(start + i));
  }
}
 
Example #9
Source File: ResourceRequestHandler.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example #10
Source File: TestApplicationLimits.java    From big-c with Apache License 2.0 5 votes vote down vote up
private FiCaSchedulerApp getMockApplication(int appId, String user,
  Resource amResource) {
  FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
  ApplicationAttemptId applicationAttemptId =
      TestUtils.getMockApplicationAttemptId(appId, 0);
  doReturn(applicationAttemptId.getApplicationId()).
      when(application).getApplicationId();
  doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
  doReturn(user).when(application).getUser();
  doReturn(amResource).when(application).getAMResource();
  return application;
}
 
Example #11
Source File: TestUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
 
Example #12
Source File: TestTaskSchedulerHelpers.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
    String appHostName, int appHostPort, String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = mock(Resource.class);
  Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  return mockRegResponse;
}
 
Example #13
Source File: TestContainerLauncherImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Token createNewContainerToken(ContainerId contId,
    String containerManagerAddr) {
  long currentTime = System.currentTimeMillis();
  return MRApp.newContainerToken(NodeId.newInstance("127.0.0.1",
      1234), "password".getBytes(), new ContainerTokenIdentifier(
      contId, containerManagerAddr, "user",
      Resource.newInstance(1024, 1),
      currentTime + 10000L, 123, currentTime, Priority.newInstance(0), 0));
}
 
Example #14
Source File: TestTaskImpl.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  conf = new Configuration();
  taskAttemptListener = mock(TaskAttemptListener.class);
  taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
  credentials = new Credentials();
  clock = new SystemClock();
  locationHint = new TaskLocationHint(null, null);

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
  dagId = TezDAGID.getInstance(appId, 1);
  vertexId = TezVertexID.getInstance(dagId, 1);
  appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
  mockContainerId = mock(ContainerId.class);
  mockContainer = mock(Container.class);
  mockAMContainer = mock(AMContainer.class);
  mockNodeId = mock(NodeId.class);
  when(mockContainer.getId()).thenReturn(mockContainerId);
  when(mockContainer.getNodeId()).thenReturn(mockNodeId);
  when(mockAMContainer.getContainer()).thenReturn(mockContainer);
  when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
  taskResource = Resource.newInstance(1024, 1);
  localResources = new HashMap<String, LocalResource>();
  environment = new HashMap<String, String>();
  javaOpts = "";
  leafVertex = false;
  containerContext = new ContainerContext(localResources, credentials,
      environment, javaOpts);
  Vertex vertex = mock(Vertex.class);
  eventHandler = new TestEventHandler();
  
  mockTask = new MockTaskImpl(vertexId, partition,
      eventHandler, conf, taskAttemptListener, clock,
      taskHeartbeatHandler, appContext, leafVertex, locationHint,
      taskResource, containerContext, vertex);
}
 
Example #15
Source File: TensorFlowRunJobParameters.java    From submarine with Apache License 2.0 5 votes vote down vote up
private Resource determinePSResource(Parameter parametersHolder,
    int nPS, ClientContext clientContext)
    throws ParseException, YarnException, IOException {
  if (nPS > 0) {
    String psResourceStr =
        parametersHolder.getOptionValue(CliConstants.PS_RES);
    if (psResourceStr == null) {
      throw new ParseException("--" + CliConstants.PS_RES + " is absent.");
    }
    return ResourceUtils.createResourceFromString(psResourceStr);
  }
  return null;
}
 
Example #16
Source File: RegisterApplicationMasterResponsePBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void setMaximumResourceCapability(Resource capability) {
  maybeInitBuilder();
  if(maximumResourceCapability == null) {
    builder.clearMaximumCapability();
  }
  this.maximumResourceCapability = capability;
}
 
Example #17
Source File: ApplicationMasterService.java    From twill with Apache License 2.0 5 votes vote down vote up
private Resource createCapability(ResourceSpecification resourceSpec) {
  Resource capability = Records.newRecord(Resource.class);

  if (!YarnUtils.setVirtualCores(capability, resourceSpec.getVirtualCores())) {
    LOG.debug("Virtual cores limit not supported.");
  }

  capability.setMemory(resourceSpec.getMemorySize());
  return capability;
}
 
Example #18
Source File: ProtocolHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public AllocateResponse createFakeAllocateResponse() {
  return AllocateResponse.newInstance(-1,
      new ArrayList<ContainerStatus>(),
      new ArrayList<Container>(), new ArrayList<NodeReport>(),
      Resource.newInstance(1024, 2, 2), null, 1,
      null, new ArrayList<NMToken>());
}
 
Example #19
Source File: NodeReportPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setUsed(Resource used) {
  maybeInitBuilder();
  if (used == null)
    builder.clearUsed();
  this.used = used;
}
 
Example #20
Source File: TestRMNodeLabelsManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testLabelsToNodesOnNodeActiveDeactive() throws Exception {
  // Activate a node without assigning any labels
  mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0));
  Assert.assertTrue(mgr.getLabelsToNodes().isEmpty());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Add labels and replace labels on node
  mgr.addToCluserNodeLabels(toSet("p1"));
  mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
  // p1 -> n1, n1:1
  Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Activate a node for which host to label mapping exists
  mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0));
  // p1 -> n1, n1:1, n1:2
  Assert.assertEquals(3, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));

  // Deactivate a node. n1:1 will be removed from the map
  mgr.deactivateNode(NodeId.newInstance("n1", 1));
  // p1 -> n1, n1:2
  Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
  assertLabelsToNodesEquals(
      mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
}
 
Example #21
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
    registerNode(String hostName, int containerManagerPort, int httpPort,
        String rackName, Resource capability)
        throws IOException, YarnException {
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm =
      new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
          hostName, containerManagerPort, httpPort, rackName, capability,
          resourceManager);
  NodeAddedSchedulerEvent nodeAddEvent1 = 
      new NodeAddedSchedulerEvent(resourceManager.getRMContext()
          .getRMNodes().get(nm.getNodeId()));
  resourceManager.getResourceScheduler().handle(nodeAddEvent1);
  return nm;
}
 
Example #22
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void setQueueResourceLimitsInfo(
    Resource clusterResource) {
  synchronized (queueResourceLimitsInfo) {
    queueResourceLimitsInfo.setQueueCurrentLimit(cachedResourceLimitsForHeadroom
        .getLimit());
    queueResourceLimitsInfo.setClusterResource(clusterResource);
  }
}
 
Example #23
Source File: TestInMemoryReservationAllocation.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private ReservationDefinition createSimpleReservationDefinition(long arrival,
    long deadline, long duration) {
  // create a request with a single atomic ask
  ReservationRequest r =
      ReservationRequest.newInstance(Resource.newInstance(1024, 1, 1), 1, 1,
          duration);
  ReservationDefinition rDef = new ReservationDefinitionPBImpl();
  ReservationRequests reqs = new ReservationRequestsPBImpl();
  reqs.setReservationResources(Collections.singletonList(r));
  reqs.setInterpreter(ReservationRequestInterpreter.R_ALL);
  rDef.setReservationRequests(reqs);
  rDef.setArrival(arrival);
  rDef.setDeadline(deadline);
  return rDef;
}
 
Example #24
Source File: ContainerStartDataPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public Resource getAllocatedResource() {
  if (this.resource != null) {
    return this.resource;
  }
  ContainerStartDataProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAllocatedResource()) {
    return null;
  }
  this.resource = convertFromProtoFormat(p.getAllocatedResource());
  return this.resource;
}
 
Example #25
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
    throws Exception {
  CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
  Resource clusterResource = Resources
      .createResource(100 * 16 * GB, 100 * 32);
  CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
  when(csContext.getRMContext()).thenReturn(rmContext);
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
  ParentQueue root = new ParentQueue(csContext, 
      CapacitySchedulerConfiguration.ROOT, null, null);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
  LeafQueue a = new LeafQueue(csContext, A, root, null);
  assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
  
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
  LeafQueue newA = new LeafQueue(csContext, A, root, null);
  a.reinitialize(newA, clusterResource);
  assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));

  Resource newClusterResource = Resources.createResource(100 * 20 * GB,
      100 * 32);
  a.updateClusterResource(newClusterResource, 
      new ResourceLimits(newClusterResource));
  //  100 * 20 * 0.2 = 400
  assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
}
 
Example #26
Source File: NodeInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
    Resource perNode, String rackName, String healthReport,
    int cmdPort, String hostName, NodeState state) {
  this.nodeId = nodeId;
  this.nodeAddr = nodeAddr;
  this.httpAddress = httpAddress;
  this.perNode = perNode;
  this.rackName = rackName;
  this.healthReport = healthReport;
  this.cmdPort = cmdPort;
  this.hostName = hostName;
  this.state = state;
  toCleanUpApplications = new ArrayList<ApplicationId>();
  toCleanUpContainers = new ArrayList<ContainerId>();
}
 
Example #27
Source File: NMHeartBeatHandler.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 
Example #28
Source File: DominantResourceCalculator.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public float ratio(Resource a, Resource b) {
  float max = Math.max(
      (float) a.getMemory() / b.getMemory(),
      (float) a.getVirtualCores() / b.getVirtualCores());
  if (b.getGpuCores() != 0) {
    max = Math.max(max,
        (float) a.getGpuCores() / b.getGpuCores());
  }
  return max;
}
 
Example #29
Source File: InMemoryPlan.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void setTotalCapacity(Resource cap) {
  writeLock.lock();
  try {
    totalCapacity = Resources.clone(cap);
  } finally {
    writeLock.unlock();
  }
}
 
Example #30
Source File: WorkerSpecContainerResourceAdapter.java    From flink with Apache License 2.0 5 votes vote down vote up
Optional<Resource> tryComputeContainerResource(final WorkerResourceSpec workerResourceSpec) {
	final InternalContainerResource internalContainerResource = workerSpecToContainerResource.computeIfAbsent(
		Preconditions.checkNotNull(workerResourceSpec),
		this::createAndMapContainerResource);
	if (internalContainerResource != null) {
		return Optional.of(internalContainerResource.toResource());
	} else {
		return Optional.empty();
	}
}