Java Code Examples for org.apache.hadoop.yarn.api.records.NodeState#RUNNING

The following examples show how to use org.apache.hadoop.yarn.api.records.NodeState#RUNNING . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ResourceRequestHandler.java    From Bats with Apache License 2.0 6 votes vote down vote up 
/**
 * Assign host to container given affinity and anti-affinity constraints and resource availibility on node
 * @param host
 * @param antiHosts
 * @param antiPreferredHosts
 * @param grpObj
 * @param nodeLocalSet
 * @param aggrMemory
 * @param vCores
 * @return
 */
public String assignHost(String host, List<String> antiHosts, List<String> antiPreferredHosts, HostOperatorSet grpObj, Set<PTOperator> nodeLocalSet, int aggrMemory, int vCores)
{
  for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) {
    if (nodeEntry.getValue().getNodeState() == NodeState.RUNNING) {
      int memAvailable = nodeEntry.getValue().getCapability().getMemory() - nodeEntry.getValue().getUsed().getMemory();
      int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores() - nodeEntry.getValue().getUsed().getVirtualCores();
      if (memAvailable >= aggrMemory && vCoresAvailable >= vCores && !antiHosts.contains(nodeEntry.getKey()) && !antiPreferredHosts.contains(nodeEntry.getKey())) {
        host = nodeEntry.getKey();
        grpObj.setHost(host);
        nodeLocalMapping.put(nodeLocalSet, host);

        return host;
      }
    }
  }
  return null;
}
Example 2
Source File: RMNodeImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;

  // Switch the last heartbeatresponse.
  rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();
  NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus();
  rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
  rmNode.setLastHealthReportTime(
      remoteNodeHealthStatus.getLastHealthReportTime());
  if (remoteNodeHealthStatus.getIsNodeHealthy()) {
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeAddedSchedulerEvent(rmNode));
    rmNode.context.getDispatcher().getEventHandler().handle(
            new NodesListManagerEvent(
                NodesListManagerEventType.NODE_USABLE, rmNode));
    // ??? how about updating metrics before notifying to ensure that
    // notifiers get update metadata because they will very likely query it
    // upon notification
    // Update metrics
    rmNode.updateMetricsForRejoinedNode(NodeState.UNHEALTHY);
    return NodeState.RUNNING;
  }

  return NodeState.UNHEALTHY;
}
Example 3
Source File: SLSRunner.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void waitForNodesRunning() throws InterruptedException {
  long startTimeMS = System.currentTimeMillis();
  while (true) {
    int numRunningNodes = 0;
    for (RMNode node : rm.getRMContext().getRMNodes().values()) {
      if (node.getState() == NodeState.RUNNING) {
        numRunningNodes ++;
      }
    }
    if (numRunningNodes == numNMs) {
      break;
    }
    LOG.info(MessageFormat.format("SLSRunner is waiting for all " +
            "nodes RUNNING. {0} of {1} NMs initialized.",
            numRunningNodes, numNMs));
    Thread.sleep(1000);
  }
  LOG.info(MessageFormat.format("SLSRunner takes {0} ms to launch all nodes.",
          (System.currentTimeMillis() - startTimeMS)));
}
Example 4
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;

  // Switch the last heartbeatresponse.
  rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();
  NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus();
  rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
  rmNode.setLastHealthReportTime(
      remoteNodeHealthStatus.getLastHealthReportTime());
  if (remoteNodeHealthStatus.getIsNodeHealthy()) {
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeAddedSchedulerEvent(rmNode));
    rmNode.context.getDispatcher().getEventHandler().handle(
            new NodesListManagerEvent(
                NodesListManagerEventType.NODE_USABLE, rmNode));
    // ??? how about updating metrics before notifying to ensure that
    // notifiers get update metadata because they will very likely query it
    // upon notification
    // Update metrics
    rmNode.updateMetricsForRejoinedNode(NodeState.UNHEALTHY);
    return NodeState.RUNNING;
  }

  return NodeState.UNHEALTHY;
}
Example 5
Source File: SLSRunner.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void waitForNodesRunning() throws InterruptedException {
  long startTimeMS = System.currentTimeMillis();
  while (true) {
    int numRunningNodes = 0;
    for (RMNode node : rm.getRMContext().getRMNodes().values()) {
      if (node.getState() == NodeState.RUNNING) {
        numRunningNodes ++;
      }
    }
    if (numRunningNodes == numNMs) {
      break;
    }
    LOG.info(MessageFormat.format("SLSRunner is waiting for all " +
            "nodes RUNNING. {0} of {1} NMs initialized.",
            numRunningNodes, numNMs));
    Thread.sleep(1000);
  }
  LOG.info(MessageFormat.format("SLSRunner takes {0} ms to launch all nodes.",
          (System.currentTimeMillis() - startTimeMS)));
}
Example 6
Source File: ResourceRequestHandler.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up 
/**
 * Assign host to container given affinity and anti-affinity constraints and resource availibility on node
 * @param host
 * @param antiHosts
 * @param antiPreferredHosts
 * @param grpObj
 * @param nodeLocalSet
 * @param aggrMemory
 * @param vCores
 * @return
 */
public String assignHost(String host, List<String> antiHosts, List<String> antiPreferredHosts, HostOperatorSet grpObj, Set<PTOperator> nodeLocalSet, int aggrMemory, int vCores)
{
  for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) {
    if (nodeEntry.getValue().getNodeState() == NodeState.RUNNING) {
      int memAvailable = nodeEntry.getValue().getCapability().getMemory() - nodeEntry.getValue().getUsed().getMemory();
      int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores() - nodeEntry.getValue().getUsed().getVirtualCores();
      if (memAvailable >= aggrMemory && vCoresAvailable >= vCores && !antiHosts.contains(nodeEntry.getKey()) && !antiPreferredHosts.contains(nodeEntry.getKey())) {
        host = nodeEntry.getKey();
        grpObj.setHost(host);
        nodeLocalMapping.put(nodeLocalSet, host);

        return host;
      }
    }
  }
  return null;
}
Example 7
Source File: ClusterAnalysisMetrics.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up 
/**
 * Gets the num containers running.
 * @param rmCommunicator 
 *
 * @return the num containers running
 */
public int getNumContainersRunning(RMCommunicator rmCommunicator) {
	int containers = 0;
	try { 			
		for (NodeReport report : rmCommunicator.getNodeReports()) {
			if(NodeState.RUNNING == report.getNodeState()) {
				containers += report.getNumContainers();
			}
		}
	} catch (YarnException | IOException e) {
           LOGGER.error("unable to get running containers", e.getMessage()); 
	}
	return containers;
}
Example 8
Source File: RMNodeImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;

  // Switch the last heartbeatresponse.
  rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();

  NodeHealthStatus remoteNodeHealthStatus = 
      statusEvent.getNodeHealthStatus();
  rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
  rmNode.setLastHealthReportTime(
      remoteNodeHealthStatus.getLastHealthReportTime());
  if (!remoteNodeHealthStatus.getIsNodeHealthy()) {
    LOG.info("Node " + rmNode.nodeId + " reported UNHEALTHY with details: "
        + remoteNodeHealthStatus.getHealthReport());
    rmNode.nodeUpdateQueue.clear();
    // Inform the scheduler
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeRemovedSchedulerEvent(rmNode));
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodesListManagerEvent(
            NodesListManagerEventType.NODE_UNUSABLE, rmNode));
    // Update metrics
    rmNode.updateMetricsForDeactivatedNode(rmNode.getState(),
        NodeState.UNHEALTHY);
    return NodeState.UNHEALTHY;
  }

  rmNode.handleContainerStatus(statusEvent.getContainers());

  if(rmNode.nextHeartBeat) {
    rmNode.nextHeartBeat = false;
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeUpdateSchedulerEvent(rmNode));
  }

  // Update DTRenewer in secure mode to keep these apps alive. Today this is
  // needed for log-aggregation to finish long after the apps are gone.
  if (UserGroupInformation.isSecurityEnabled()) {
    rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications(
      statusEvent.getKeepAliveAppIds());
  }

  return NodeState.RUNNING;
}
Example 9
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;

  // Switch the last heartbeatresponse.
  rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();

  NodeHealthStatus remoteNodeHealthStatus = 
      statusEvent.getNodeHealthStatus();
  rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
  rmNode.setLastHealthReportTime(
      remoteNodeHealthStatus.getLastHealthReportTime());
  if (!remoteNodeHealthStatus.getIsNodeHealthy()) {
    LOG.info("Node " + rmNode.nodeId + " reported UNHEALTHY with details: "
        + remoteNodeHealthStatus.getHealthReport());
    rmNode.nodeUpdateQueue.clear();
    // Inform the scheduler
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeRemovedSchedulerEvent(rmNode));
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodesListManagerEvent(
            NodesListManagerEventType.NODE_UNUSABLE, rmNode));
    // Update metrics
    rmNode.updateMetricsForDeactivatedNode(rmNode.getState(),
        NodeState.UNHEALTHY);
    return NodeState.UNHEALTHY;
  }

  rmNode.handleContainerStatus(statusEvent.getContainers());

  if(rmNode.nextHeartBeat) {
    rmNode.nextHeartBeat = false;
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeUpdateSchedulerEvent(rmNode));
  }

  // Update DTRenewer in secure mode to keep these apps alive. Today this is
  // needed for log-aggregation to finish long after the apps are gone.
  if (UserGroupInformation.isSecurityEnabled()) {
    rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications(
      statusEvent.getKeepAliveAppIds());
  }

  return NodeState.RUNNING;
}