Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#setMemory()

The following examples show how to use org.apache.hadoop.yarn.api.records.Resource#setMemory() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   File: MockAM.java    License: Apache License 2.0 6 votes vote down vote up
public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
 
Example 2
Source Project: big-c   File: TestYarnServerApiClasses.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test RegisterNodeManagerRequestPBImpl.
 */

@Test
public void testRegisterNodeManagerRequestPBImpl() {
  RegisterNodeManagerRequestPBImpl original = new RegisterNodeManagerRequestPBImpl();
  original.setHttpPort(8080);
  original.setNodeId(getNodeId());
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(10000);
  resource.setVirtualCores(2);
  original.setResource(resource);
  RegisterNodeManagerRequestPBImpl copy = new RegisterNodeManagerRequestPBImpl(
      original.getProto());

  assertEquals(8080, copy.getHttpPort());
  assertEquals(9090, copy.getNodeId().getPort());
  assertEquals(10000, copy.getResource().getMemory());
  assertEquals(2, copy.getResource().getVirtualCores());

}
 
Example 3
Source Project: big-c   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource subtractFroms(Resource lhs, Resource rhs){
if(lhs.getMemory() > rhs.getMemory()){  
  lhs.setMemory(lhs.getMemory() - rhs.getMemory());
}else{
  lhs.setMemory(0);	
}

if(lhs.getVirtualCores() > rhs.getVirtualCores()){
  lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores()); 
}else{
  lhs.setVirtualCores(0);	
}

return lhs;
 }
 
Example 4
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
public static Resource createResource(int memory, int cores, int gcores) {
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(cores);
  resource.setGpuCores(gcores);
  return resource;
}
 
Example 5
Source Project: big-c   File: ComputeFairShares.java    License: Apache License 2.0 5 votes vote down vote up
private static void setResourceValue(int val, Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    resource.setMemory(val);
    break;
  case CPU:
    resource.setVirtualCores(val);
    break;
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 
Example 6
Source Project: twill   File: Hadoop21YarnAppClient.java    License: Apache License 2.0 5 votes vote down vote up
private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) {
  int maxMemory = response.getMaximumResourceCapability().getMemory();
  int updatedMemory = capability.getMemory();

  if (updatedMemory > maxMemory) {
    capability.setMemory(maxMemory);
  }

  return capability;
}
 
Example 7
Source Project: big-c   File: TestTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 
Example 8
public void run(VMConfig vmConfig, Configuration conf) throws Exception {
  try {
    vmConfig.overrideHadoopConfiguration(conf);
    System.out.println("Create YarnClient") ;
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    System.out.println("Create YarnClientApplication via YarnClient") ;
    YarnClientApplication app = yarnClient.createApplication();
    String appId =  app.getApplicationSubmissionContext().getApplicationId().toString() ;
    System.out.println("Application Id = " + appId) ;
    System.out.println("Set up the container launch context for the application master") ;
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    StringBuilder sb = new StringBuilder();
    List<String> commands = Collections.singletonList(
        sb.append(vmConfig.buildCommand()).
        append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
        append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr")
        .toString()
    );
    amContainer.setCommands(commands) ;

    System.out.println("Setup the app classpath and resources") ;
    if(vmConfig.getVmResources().size() > 0) {
      amContainer.setLocalResources(new VMResources(conf, vmConfig));
    }
    
    System.out.println("Setup the classpath for ApplicationMaster, environment = " + vmConfig.getEnvironment()) ;
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    boolean jvmEnv = vmConfig.getEnvironment() != VMConfig.Environment.YARN;
    Util.setupAppMasterEnv(jvmEnv , conf, appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    System.out.println("Set up resource type requirements for ApplicationMaster") ;
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemory(256);
    resource.setVirtualCores(1);

    System.out.println("Finally, set-up ApplicationSubmissionContext for the application");
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(vmConfig.getName()); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(resource);
    appContext.setQueue("default"); // queue 

    // Submit application
    ApplicationId applicationId = appContext.getApplicationId();
    System.out.println("Submitting application " + applicationId);
    yarnClient.submitApplication(appContext);
  } catch(Exception ex) {
    ex.printStackTrace(); 
    throw ex ;
  }
}
 
Example 9
Source Project: big-c   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource multiplyTo(Resource lhs, double by) {
  lhs.setMemory((int)(lhs.getMemory() * by));
  lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
  return lhs;
}
 
Example 10
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource multiplyTo(Resource lhs, double by) {
  lhs.setMemory((int)(lhs.getMemory() * by));
  lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
  lhs.setGpuCores((int) (lhs.getGpuCores() * by));
  return lhs;
}
 
Example 11
Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 
Example 12
Source Project: big-c   File: BuilderUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource newResource(int memory, int vCores) {
  Resource resource = recordFactory.newRecordInstance(Resource.class);
  resource.setMemory(memory);
  resource.setVirtualCores(vCores);
  return resource;
}
 
Example 13
@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
 
Example 14
Source Project: hadoop   File: TestNodeManagerMetrics.java    License: Apache License 2.0 4 votes vote down vote up
@Test public void testNames() {
  DefaultMetricsSystem.initialize("NodeManager");
  NodeManagerMetrics metrics = NodeManagerMetrics.create();
  Resource total = Records.newRecord(Resource.class);
  total.setMemory(8*GiB);
  total.setVirtualCores(16);
  total.setGpuCores(16);
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(512); //512MiB
  resource.setVirtualCores(2);
  resource.setGpuCores(1);


  metrics.addResource(total);

  for (int i = 10; i-- > 0;) {
    // allocate 10 containers(allocatedGB: 5GiB, availableGB: 3GiB)
    metrics.launchedContainer();
    metrics.allocateContainer(resource);
  }

  metrics.initingContainer();
  metrics.endInitingContainer();
  metrics.runningContainer();
  metrics.endRunningContainer();
  // Releasing 3 containers(allocatedGB: 3.5GiB, availableGB: 4.5GiB)
  metrics.completedContainer();
  metrics.releaseContainer(resource);

  metrics.failedContainer();
  metrics.releaseContainer(resource);

  metrics.killedContainer();
  metrics.releaseContainer(resource);

  metrics.initingContainer();
  metrics.runningContainer();

  Assert.assertTrue(!metrics.containerLaunchDuration.changed());
  metrics.addContainerLaunchDuration(1);
  Assert.assertTrue(metrics.containerLaunchDuration.changed());

  // availableGB is expected to be floored,
  // while allocatedGB is expected to be ceiled.
  // allocatedGB: 3.5GB allocated memory is shown as 4GB
  // availableGB: 4.5GB available memory is shown as 4GB
  checkMetrics(10, 1, 1, 1, 1, 1, 4, 7, 4, 14, 2, 7, 9);
}
 
Example 15
Source Project: hadoop-mini-clusters   File: Client.java    License: Apache License 2.0 4 votes vote down vote up
public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 
Example 16
Source Project: big-c   File: MockNodes.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource newResource(int mem) {
  Resource rs = recordFactory.newRecordInstance(Resource.class);
  rs.setMemory(mem);
  return rs;
}
 
Example 17
Source Project: hadoop   File: MockNodes.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource newResource(int mem) {
  Resource rs = recordFactory.newRecordInstance(Resource.class);
  rs.setMemory(mem);
  return rs;
}
 
Example 18
Source Project: hadoop   File: MockNodes.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource newAvailResource(Resource total, Resource used) {
  Resource rs = recordFactory.newRecordInstance(Resource.class);
  rs.setMemory(total.getMemory() - used.getMemory());
  return rs;
}
 
Example 19
@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}
 
Example 20
Source Project: big-c   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static Resource subtractFrom(Resource lhs, Resource rhs) {
  lhs.setMemory(lhs.getMemory() - rhs.getMemory());
  lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores());
  return lhs;
}