Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#setMemory()

/**
 * Test RegisterNodeManagerRequestPBImpl.
 */

@Test
public void testRegisterNodeManagerRequestPBImpl() {
  RegisterNodeManagerRequestPBImpl original = new RegisterNodeManagerRequestPBImpl();
  original.setHttpPort(8080);
  original.setNodeId(getNodeId());
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(10000);
  resource.setVirtualCores(2);
  original.setResource(resource);
  RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl(
      original.getProto());

  assertEquals(8080, copy.getHttpPort());
  assertEquals(9090, copy.getNodeId().getPort());
  assertEquals(10000, copy.getResource().getMemory());
  assertEquals(2, copy.getResource().getVirtualCores());

}
 

public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
 

private static void setResourceValue(int val, Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    resource.setMemory(val);
    break;
  case CPU:
    resource.setVirtualCores(val);
    break;
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 

@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 

private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) {
  int maxMemory = response.getMaximumResourceCapability().getMemory();
  int updatedMemory = capability.getMemory();

  if (updatedMemory > maxMemory) {
    capability.setMemory(maxMemory);
  }

  return capability;
}
 

public void run(VMConfig vmConfig, Configuration conf) throws Exception {
  try {
    vmConfig.overrideHadoopConfiguration(conf);
    System.out.println("Create YarnClient") ;
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    System.out.println("Create YarnClientApplication via YarnClient") ;
    YarnClientApplication app = yarnClient.createApplication();
    String appId =  app.getApplicationSubmissionContext().getApplicationId().toString() ;
    System.out.println("Application Id = " + appId) ;
    System.out.println("Set up the container launch context for the application master") ;
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    StringBuilder sb = new StringBuilder();
    List<String> commands = Collections.singletonList(
        sb.append(vmConfig.buildCommand()).
        append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
        append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr")
        .toString()
    );
    amContainer.setCommands(commands) ;

    System.out.println("Setup the app classpath and resources") ;
    if(vmConfig.getVmResources().size() > 0) {
      amContainer.setLocalResources(new VMResources(conf, vmConfig));
    }
    
    System.out.println("Setup the classpath for ApplicationMaster, environment = " + vmConfig.getEnvironment()) ;
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    boolean jvmEnv = vmConfig.getEnvironment() != VMConfig.Environment.YARN;
    Util.setupAppMasterEnv(jvmEnv , conf, appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    System.out.println("Set up resource type requirements for ApplicationMaster") ;
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemory(256);
    resource.setVirtualCores(1);

    System.out.println("Finally, set-up ApplicationSubmissionContext for the application");
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(vmConfig.getName()); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(resource);
    appContext.setQueue("default"); // queue 

    // Submit application
    ApplicationId applicationId = appContext.getApplicationId();
    System.out.println("Submitting application " + applicationId);
    yarnClient.submitApplication(appContext);
  } catch(Exception ex) {
    ex.printStackTrace(); 
    throw ex ;
  }
}
 

public static Resource createResource(int memory, int cores, int gcores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  resource.setGpuCores(gcores);
  return resource;
}
 

public static Resource subtractFroms(Resource lhs, Resource rhs){
if(lhs.getMemory() > rhs.getMemory()){  
  lhs.setMemory(lhs.getMemory() - rhs.getMemory());
}else{
  lhs.setMemory(0);	
}

if(lhs.getVirtualCores() > rhs.getVirtualCores()){
  lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); 
}else{
  lhs.setVirtualCores(0);	
}

return lhs;
 }
 

public static Resource multiplyTo(Resource lhs, double by) {
  lhs.setMemory((int)(lhs.getMemory() * by));
  lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
  return lhs;
}
 

public static Resource multiplyTo(Resource lhs, double by) {
  lhs.setMemory((int)(lhs.getMemory() * by));
  lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
  lhs.setGpuCores((int) (lhs.getGpuCores() * by));
  return lhs;
}
 

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 

public static Resource newResource(int memory, int vCores) {
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(vCores);
  return resource;
}
 

@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
 

@Test public void testNames() {
  DefaultMetricsSystem.initialize("NodeManager");
  NodeManagerMetrics metrics = NodeManagerMetrics.create();
  Resource total = Records.newRecord(Resource.class);
  total.setMemory(8*GiB);
  total.setVirtualCores(16);
  total.setGpuCores(16);
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(512); //512MiB
  resource.setVirtualCores(2);
  resource.setGpuCores(1);


  metrics.addResource(total);

  for (int i = 10; i-- > 0;) {
    // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
    metrics.launchedContainer();
    metrics.allocateContainer(resource);
  }

  metrics.initingContainer();
  metrics.endInitingContainer();
  metrics.runningContainer();
  metrics.endRunningContainer();
  // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
  metrics.completedContainer();
  metrics.releaseContainer(resource);

  metrics.failedContainer();
  metrics.releaseContainer(resource);

  metrics.killedContainer();
  metrics.releaseContainer(resource);

  metrics.initingContainer();
  metrics.runningContainer();

  Assert.assertTrue(!metrics.containerLaunchDuration.changed());
  metrics.addContainerLaunchDuration(1);
  Assert.assertTrue(metrics.containerLaunchDuration.changed());

  // availableGB is expected to be floored,
  // while allocatedGB is expected to be ceiled.
  // allocatedGB: 3.5GB allocated memory is shown as 4GB
  // availableGB: 4.5GB available memory is shown as 4GB
  checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2, 7, 9);
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

public static Resource newResource(int mem) {
  Resource rs = recordFactory.newRecordInstance(Resource.class);
  rs.setMemory(mem);
  return rs;
}
 

public static Resource newResource(int mem) {
  Resource rs = recordFactory.newRecordInstance(Resource.class);
  rs.setMemory(mem);
  return rs;
}
 

public static Resource newAvailResource(Resource total, Resource used) {
  Resource rs = recordFactory.newRecordInstance(Resource.class);
  rs.setMemory(total.getMemory() - used.getMemory());
  return rs;
}
 

@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
 

public static Resource subtractFrom(Resource lhs, Resource rhs) {
  lhs.setMemory(lhs.getMemory() - rhs.getMemory());
  lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores());
  return lhs;
}