Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#setVirtualCores()

The following examples show how to use org.apache.hadoop.yarn.api.records.Resource#setVirtualCores() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example 2
Source Project: hadoop   File: TestYarnServerApiClasses.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test RegisterNodeManagerRequestPBImpl.
 */

@Test
public void testRegisterNodeManagerRequestPBImpl() {
  RegisterNodeManagerRequestPBImpl original = new RegisterNodeManagerRequestPBImpl();
  original.setHttpPort(8080);
  original.setNodeId(getNodeId());
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(10000);
  resource.setVirtualCores(2);
  resource.setGpuCores(3);
  original.setResource(resource);
  RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl(
      original.getProto());

  assertEquals(8080, copy.getHttpPort());
  assertEquals(9090, copy.getNodeId().getPort());
  assertEquals(10000, copy.getResource().getMemory());
  assertEquals(2, copy.getResource().getVirtualCores());
  assertEquals(3, copy.getResource().getGpuCores());

}
 
Example 3
@Test(timeout = 15000)
public void testSubmitApplicationOnHA() throws Exception {
  ApplicationSubmissionContext appContext =
      Records.newRecord(ApplicationSubmissionContext.class);
  appContext.setApplicationId(cluster.createFakeAppId());
  ContainerLaunchContext amContainer =
      Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(10);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  appContext.setResource(capability);
  ApplicationId appId = client.submitApplication(appContext);
  Assert.assertTrue(getActiveRM().getRMContext().getRMApps()
      .containsKey(appId));
}
 
Example 4
Source Project: PoseidonX   File: JstormAMUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 构建对于container的申请
 * @param containerType
 * @param mem
 * @return
 */
public static AMRMClient.ContainerRequest setupContainerAskForRM(int containerType, int mem) {
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(containerType);
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(mem);
    capability.setVirtualCores(1);
    AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability, null, null, pri);
    LOG.error("#### 申请container:[" + request.toString()+"]");
    return request;
}
 
Example 5
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource createResource(int memory, int cores, int gcores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  resource.setGpuCores(gcores);
  return resource;
}
 
Example 6
Source Project: reef   File: YarnResourceRequestHandler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized Resource getResource(final ResourceRequestEvent resourceRequestEvent) {
  final Resource result = Records.newRecord(Resource.class);
  final int memory = getMemory(resourceRequestEvent.getMemorySize().get());
  final int core = resourceRequestEvent.getVirtualCores().get();
  LOG.log(Level.FINEST, "Resource requested: memory = {0}, virtual core count = {1}.", new Object[]{memory, core});
  result.setMemory(memory);
  result.setVirtualCores(core);
  return result;
}
 
Example 7
Source Project: hadoop   File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource newResource(int memory, int vCores) {
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(vCores);
  resource.setGpuCores(0);
  return resource;
}
 
Example 8
Source Project: hadoop   File: ComputeFairShares.java    License: Apache License 2.0 5 votes vote down vote up
private static void setResourceValue(int val, Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    resource.setMemory(val);
    break;
  case CPU:
    resource.setVirtualCores(val);
    break;
  case GPU:
    resource.setGpuCores(val);
    break;
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 
Example 9
Source Project: big-c   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource subtractFroms(Resource lhs, Resource rhs){
if(lhs.getMemory() > rhs.getMemory()){  
  lhs.setMemory(lhs.getMemory() - rhs.getMemory());
}else{
  lhs.setMemory(0);	
}

if(lhs.getVirtualCores() > rhs.getVirtualCores()){
  lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); 
}else{
  lhs.setVirtualCores(0);	
}

return lhs;
 }
 
Example 10
Source Project: ignite   File: ApplicationMaster.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Runs application master.
 *
 * @throws Exception If failed.
 */
public void run() throws Exception {
    // Register with ResourceManager
    rmClient.registerApplicationMaster("", 0, "");

    log.log(Level.INFO, "Application master registered.");

    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    try {
        // Check ignite cluster.
        while (!nmClient.isInState(Service.STATE.STOPPED)) {
            int runningCnt = containers.size();

            if (runningCnt < props.instances() && checkAvailableResource()) {
                // Resource requirements for worker containers.
                Resource capability = Records.newRecord(Resource.class);

                capability.setMemory((int)props.totalMemoryPerNode());
                capability.setVirtualCores((int)props.cpusPerNode());

                for (int i = 0; i < props.instances() - runningCnt; ++i) {
                    // Make container requests to ResourceManager
                    AMRMClient.ContainerRequest containerAsk =
                        new AMRMClient.ContainerRequest(capability, null, null, priority);

                    rmClient.addContainerRequest(containerAsk);

                    log.log(Level.INFO, "Making request. Memory: {0}, cpu {1}.",
                        new Object[]{props.totalMemoryPerNode(), props.cpusPerNode()});
                }
            }

            TimeUnit.MILLISECONDS.sleep(schedulerTimeout);
        }
    }
    catch (InterruptedException ignored) {
        // Un-register with ResourceManager
        rmClient.unregisterApplicationMaster(FinalApplicationStatus.KILLED, "", "");

        log.log(Level.WARNING, "Application master killed.");
    }
    catch (Exception e) {
        // Un-register with ResourceManager
        rmClient.unregisterApplicationMaster(FinalApplicationStatus.FAILED, "", "");

        log.log(Level.SEVERE, "Application master failed.", e);
    }
}
 
Example 11
private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception {
  appContext.setMaxAppAttempts(MAX_ATTEMPT);

  Map<String, LocalResource> localResources = new HashMap<>();
  Set<Path> shippedPaths = new HashSet<>();
  collectLocalResources(localResources, shippedPaths);
  final ContainerLaunchContext amContainer = setupApplicationMasterContainer(
      this.getYarnSessionClusterEntrypoint(),
      false,
      true,
      false, (int) job.taskManagerMemoryMb());

  amContainer.setLocalResources(localResources);

  final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator));
  final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString())
      .collect(Collectors.joining(","));

  // Setup CLASSPATH and environment variables for ApplicationMaster
  ApplicationId appId = appContext.getApplicationId();
  final Map<String, String> appMasterEnv = setUpAmEnvironment(
      appId,
      classPath,
      shippedFiles,
      getDynamicPropertiesEncoded()
  );

  amContainer.setEnvironment(appMasterEnv);

  // Set up resource type requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(getFlinkConfiguration()
      .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY));
  capability.setVirtualCores(1);

  appContext.setApplicationName(job.name());
  appContext.setApplicationType(ATHENAX_APPLICATION_TYPE);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setApplicationTags(Collections.singleton(job.metadata().serialize()));
  if (job.queue() != null) {
    appContext.setQueue(job.queue());
  }

  LOG.info("Submitting application master {}", appId);
  yarnClient.submitApplication(appContext);

  PollDeploymentStatus poll = new PollDeploymentStatus(appId);
  YARN_POLL_EXECUTOR.submit(poll);
  try {
    return poll.result.get();
  } catch (ExecutionException e) {
    LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause());
    yarnClient.killApplication(appId);
    throw (Exception) e.getCause();
  }
}
 
Example 12
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource addTo(Resource lhs, Resource rhs) {
  lhs.setMemory(lhs.getMemory() + rhs.getMemory());
  lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
  lhs.setGpuCores(lhs.getGpuCores() + rhs.getGpuCores());
  return lhs;
}
 
Example 13
Source Project: tez   File: DagAwareYarnTaskScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void deductFrom(Resource total, Resource toSubtract) {
  super.deductFrom(total, toSubtract);
  total.setVirtualCores(total.getVirtualCores() - toSubtract.getVirtualCores());
}
 
Example 14
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource multiplyTo(Resource lhs, double by) {
  lhs.setMemory((int)(lhs.getMemory() * by));
  lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
  lhs.setGpuCores((int) (lhs.getGpuCores() * by));
  return lhs;
}
 
Example 15
Source Project: big-c   File: BuilderUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource newResource(int memory, int vCores) {
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(vCores);
  return resource;
}
 
Example 16
Source Project: hadoop   File: TestNodeManagerMetrics.java    License: Apache License 2.0 4 votes vote down vote up
@Test public void testNames() {
  DefaultMetricsSystem.initialize("NodeManager");
  NodeManagerMetrics metrics = NodeManagerMetrics.create();
  Resource total = Records.newRecord(Resource.class);
  total.setMemory(8*GiB);
  total.setVirtualCores(16);
  total.setGpuCores(16);
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(512); //512MiB
  resource.setVirtualCores(2);
  resource.setGpuCores(1);


  metrics.addResource(total);

  for (int i = 10; i-- > 0;) {
    // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
    metrics.launchedContainer();
    metrics.allocateContainer(resource);
  }

  metrics.initingContainer();
  metrics.endInitingContainer();
  metrics.runningContainer();
  metrics.endRunningContainer();
  // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
  metrics.completedContainer();
  metrics.releaseContainer(resource);

  metrics.failedContainer();
  metrics.releaseContainer(resource);

  metrics.killedContainer();
  metrics.releaseContainer(resource);

  metrics.initingContainer();
  metrics.runningContainer();

  Assert.assertTrue(!metrics.containerLaunchDuration.changed());
  metrics.addContainerLaunchDuration(1);
  Assert.assertTrue(metrics.containerLaunchDuration.changed());

  // availableGB is expected to be floored,
  // while allocatedGB is expected to be ceiled.
  // allocatedGB: 3.5GB allocated memory is shown as 4GB
  // availableGB: 4.5GB available memory is shown as 4GB
  checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2, 7, 9);
}
 
Example 17
Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 
Example 18
@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
 
Example 19
Source Project: big-c   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource createResource(int memory, int cores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  return resource;
}
 
Example 20
Source Project: big-c   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource addTo(Resource lhs, Resource rhs) {
  lhs.setMemory(lhs.getMemory() + rhs.getMemory());
  lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
  return lhs;
}