Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#getVirtualCores()

The following examples show how to use org.apache.hadoop.yarn.api.records.Resource#getVirtualCores() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   File: ResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(Resource other) {
  int diff = this.getMemory() - other.getMemory();
  if (diff == 0) {
    diff = this.getVirtualCores() - other.getVirtualCores();
  }
  return diff;
}
 
Example 2
Source Project: big-c   File: RMContainerImpl.java    License: Apache License 2.0 5 votes vote down vote up
private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }
  
  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = (long)(resource.getMemory()*container.utilization)
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = (long)(resource.getVirtualCores()*container.utilization)
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    
    if (container.suspendTime.size() >0 && container.resumeTime.size() >0 && container.suspendTime.size() == container.resumeTime.size()){
    	double acc=0;
    	for(int i=0; i < container.suspendTime.size();i++){
    		
    		acc = acc + (container.resumeTime.get(i) - container.suspendTime.get(i));
    	}
    	container.utilization = acc/usedMillis;  	
    }
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds);
  }
}
 
Example 3
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 
Example 4
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 
Example 5
/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
  //NOOP prevent YARN warning changing to same size
  if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
    return;
  }
  if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
    LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
    return;
  }
  yarnSchedulerLock.lock();
  try {
    if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
      Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
      rmNode.getTotalCapability().setMemory(zeroed.getMemory());
      rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
      LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.",
          rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
    } else {
      rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
      rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
      }
    }
    // updates the scheduler with the new capacity for the NM.
    // the event is handled by the scheduler asynchronously
    rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
        rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  } finally {
    yarnSchedulerLock.unlock();
  }
}
 
Example 6
Source Project: stratosphere   File: Client.java    License: Apache License 2.0 5 votes vote down vote up
private void showClusterMetrics(YarnClient yarnClient)
		throws YarnException, IOException {
	YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
	System.out.println("NodeManagers in Cluster " + metrics.getNumNodeManagers());
	List<NodeReport> nodes = yarnClient.getNodeReports();
	final String format = "|%-16s |%-16s %n";
	System.out.printf("|Property         |Value          %n");
	System.out.println("+---------------------------------------+");
	int totalMemory = 0;
	int totalCores = 0;
	for(NodeReport rep : nodes) {
		final Resource res = rep.getCapability();
		totalMemory += res.getMemory();
		totalCores += res.getVirtualCores();
		System.out.format(format, "NodeID", rep.getNodeId());
		System.out.format(format, "Memory", res.getMemory()+" MB");
		System.out.format(format, "vCores", res.getVirtualCores());
		System.out.format(format, "HealthReport", rep.getHealthReport());
		System.out.format(format, "Containers", rep.getNumContainers());
		System.out.println("+---------------------------------------+");
	}
	System.out.println("Summary: totalMemory "+totalMemory+" totalCores "+totalCores);
	List<QueueInfo> qInfo = yarnClient.getAllQueues();
	for(QueueInfo q : qInfo) {
		System.out.println("Queue: "+q.getQueueName()+", Current Capacity: "+q.getCurrentCapacity()+" Max Capacity: "+q.getMaximumCapacity()+" Applications: "+q.getApplications().size());
	}
	yarnClient.stop();
	System.exit(0);
}
 
Example 7
@Override
public boolean isInvalidDivisor(Resource r) {
  if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f || r.getGpuCores() == 0.0f) {
    return true;
  }
  return false;
}
 
Example 8
boolean canFit(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  
  if(mem0 <= mem1 && cpu0 <= cpu1) { 
    return true;
  }
  return false; 
}
 
Example 9
Source Project: big-c   File: SchedulerNode.java    License: Apache License 2.0 5 votes vote down vote up
synchronized void addAvailableResource(Resource resource) {
  if (resource == null) {
    LOG.error("Invalid resource addition of null resource for "
        + rmNode.getNodeAddress());
    return;
  }
  //chekc if the add resource operation is good to go
  if(resource.getMemory() < 0 && resource.getMemory()+availableResource.getMemory() < 0){
    LOG.error("Invalid resource addition of null resource for "
  	          + rmNode.getNodeAddress());
    return; 	 	
  }
  
  if(resource.getVirtualCores() < 0 && resource.getVirtualCores() + availableResource.getVirtualCores() < 0){
  LOG.error("Invalid resource addition of null resource for "
	          + rmNode.getNodeAddress());
	  return; 	
  }
  
  if(resource.getMemory() > 0 && resource.getMemory()+availableResource.getMemory() > totalResourceCapability.getMemory()){
  LOG.error("Invalid resource addition of null resource for "
  	          + rmNode.getNodeAddress());
  	  return; 		
  }
  
  if(resource.getVirtualCores() > 0 && resource.getVirtualCores() + availableResource.getVirtualCores() > totalResourceCapability.getVirtualCores()){
  LOG.error("Invalid resource addition of null resource for "
  	          + rmNode.getNodeAddress());
  	  return; 		    	
  }
  
  Resources.addTo(availableResource, resource);
  Resources.subtractFrom(usedResource, resource);
}
 
Example 10
Source Project: ignite   File: ApplicationMaster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return {@code True} if cluster contains available resources.
 */
private boolean checkAvailableResource() {
    Resource availableRes = rmClient.getAvailableResources();

    return availableRes == null || availableRes.getMemory() >= props.totalMemoryPerNode()
        && availableRes.getVirtualCores() >= props.cpusPerNode();
}
 
Example 11
Source Project: hadoop   File: AMRMClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compare(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  if(mem0 == mem1) {
    if(cpu0 == cpu1) {
      if(gpu0 == gpu1) {
        return 0;
      }
      if(gpu0 < gpu1) {
        return 1;
      }
      return -1;
    }
    if(cpu0 < cpu1) {
      return 1;
    }
    return -1;
  }
  if(mem0 < mem1) { 
    return 1;
  }
  return -1;
}
 
Example 12
@VisibleForTesting
protected boolean offerWithinResourceLimits(Resource currentResources, Resource offeredResources) {
  int newMemory = currentResources.getMemory() + offeredResources.getMemory();
  int newCores = currentResources.getVirtualCores() + offeredResources.getVirtualCores();
     
  return (newMemory <= conf.getJvmMaxMemoryMB() && newCores <= conf.getMaxCpus());
}
 
Example 13
Source Project: hadoop   File: ComputeFairShares.java    License: Apache License 2.0 5 votes vote down vote up
private static int getResourceValue(Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    return resource.getMemory();
  case CPU:
    return resource.getVirtualCores();
  case GPU:
    return resource.getGpuCores();
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 
Example 14
/**
 * Get the per queue setting for the maximum limit to allocate to
 * each container request.
 *
 * @param queue
 *          name of the queue
 * @return setting specified per queue else falls back to the cluster setting
 */
public Resource getMaximumAllocationPerQueue(String queue) {
  String queuePrefix = getQueuePrefix(queue);
  int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
      (int)UNDEFINED);
  int maxAllocationVcoresPerQueue = getInt(
      queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
  if (LOG.isDebugEnabled()) {
    LOG.debug("max alloc mb per queue for " + queue + " is "
        + maxAllocationMbPerQueue);
    LOG.debug("max alloc vcores per queue for " + queue + " is "
        + maxAllocationVcoresPerQueue);
  }
  Resource clusterMax = getMaximumAllocation();
  if (maxAllocationMbPerQueue == (int)UNDEFINED) {
    LOG.info("max alloc mb per queue for " + queue + " is undefined");
    maxAllocationMbPerQueue = clusterMax.getMemory();
  }
  if (maxAllocationVcoresPerQueue == (int)UNDEFINED) {
     LOG.info("max alloc vcore per queue for " + queue + " is undefined");
    maxAllocationVcoresPerQueue = clusterMax.getVirtualCores();
  }
  Resource result = Resources.createResource(maxAllocationMbPerQueue,
      maxAllocationVcoresPerQueue);
  if (maxAllocationMbPerQueue > clusterMax.getMemory()
      || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()) {
    throw new IllegalArgumentException(
        "Queue maximum allocation cannot be larger than the cluster setting"
        + " for queue " + queue
        + " max allocation per queue: " + result
        + " cluster setting: " + clusterMax);
  }
  return result;
}
 
Example 15
Source Project: tez   File: DagAwareYarnTaskScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public boolean anyAvailable(Resource rsrc) {
  return super.anyAvailable(rsrc) || rsrc.getVirtualCores() > 0;
}
 
Example 16
Source Project: big-c   File: CapacityOverTimePolicy.java    License: Apache License 2.0 4 votes vote down vote up
public IntegralResource(Resource resource) {
  this.memory = resource.getMemory();
  this.vcores = resource.getVirtualCores();
}
 
Example 17
Source Project: big-c   File: ContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
/** 
 *  Return a command to execute the given command in OS shell.
 *  On Windows, the passed in groupId can be used to launch
 *  and associate the given groupId in a process group. On
 *  non-Windows, groupId is ignored. 
 */
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf, Resource resource) {
  boolean containerSchedPriorityIsSet = false;
  int containerSchedPriorityAdjustment = 
      YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;

  if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != 
      null) {
    containerSchedPriorityIsSet = true;
    containerSchedPriorityAdjustment = conf 
        .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 
        YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
  }

  if (Shell.WINDOWS) {
    int cpuRate = -1;
    int memory = -1;
    if (resource != null) {
      if (conf
          .getBoolean(
              YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
              YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
        memory = resource.getMemory();
      }

      if (conf.getBoolean(
          YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
          YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
        int containerVCores = resource.getVirtualCores();
        int nodeVCores = conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
        // cap overall usage to the number of cores allocated to YARN
        int nodeCpuPercentage = Math
            .min(
                conf.getInt(
                    YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
                    YarnConfiguration.DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT),
                100);
        nodeCpuPercentage = Math.max(0, nodeCpuPercentage);
        if (nodeCpuPercentage == 0) {
          String message = "Illegal value for "
              + YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
              + ". Value cannot be less than or equal to 0.";
          throw new IllegalArgumentException(message);
        }
        float yarnVCores = (nodeCpuPercentage * nodeVCores) / 100.0f;
        // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
        // should be set as 20 * 100. The following setting is equal to:
        // 100 * (100 * (vcores / Total # of cores allocated to YARN))
        cpuRate = Math.min(10000,
            (int) ((containerVCores * 10000) / yarnVCores));
      }
    }
    return new String[] { Shell.WINUTILS, "task", "create", "-m",
        String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId,
        "cmd /c " + command };
  } else {
    List<String> retCommand = new ArrayList<String>();
    if (containerSchedPriorityIsSet) {
      retCommand.addAll(Arrays.asList("nice", "-n",
          Integer.toString(containerSchedPriorityAdjustment)));
    }
    retCommand.addAll(Arrays.asList("bash", command));
    return retCommand.toArray(new String[retCommand.size()]);
  }

}
 
Example 18
Source Project: hadoop   File: ResourceInfo.java    License: Apache License 2.0 4 votes vote down vote up
public ResourceInfo(Resource res) {
  memory = res.getMemory();
  vCores = res.getVirtualCores();
  gCores = res.getGpuCores();
}
 
Example 19
Source Project: big-c   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static boolean fitsIn(Resource smaller, Resource bigger) {
  return smaller.getMemory() <= bigger.getMemory() &&
      smaller.getVirtualCores() <= bigger.getVirtualCores();
}
 
Example 20
Source Project: tez   File: DagAwareYarnTaskScheduler.java    License: Apache License 2.0 4 votes vote down vote up
boolean canFit(Resource capability) {
  Resource cr = container.getResource();
  return cr.getMemory() >= capability.getMemory() && cr.getVirtualCores() >= capability.getVirtualCores();
}