Java Code Examples for org.apache.hadoop.yarn.api.records.NodeState

The following examples show how to use org.apache.hadoop.yarn.api.records.NodeState. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestResourceTrackerService.java    License: Apache License 2.0 6 votes vote down vote up
private void checkUnealthyNMCount(MockRM rm, MockNM nm1, boolean health,
    int count) throws Exception {
  
  int waitCount = 0;
  while((rm.getRMContext().getRMNodes().get(nm1.getNodeId())
      .getState() != NodeState.UNHEALTHY) == health
      && waitCount++ < 20) {
    synchronized (this) {
      wait(100);
    }
  }
  Assert.assertFalse((rm.getRMContext().getRMNodes().get(nm1.getNodeId())
      .getState() != NodeState.UNHEALTHY) == health);
  Assert.assertEquals("Unhealthy metrics not incremented", count,
      ClusterMetrics.getMetrics().getUnhealthyNMs());
}
 
Example 2
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException {
	List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

	int totalFreeMemory = 0;
	int containerLimit = 0;
	int[] nodeManagersFree = new int[nodes.size()];

	for (int i = 0; i < nodes.size(); i++) {
		NodeReport rep = nodes.get(i);
		int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0);
		nodeManagersFree[i] = free;
		totalFreeMemory += free;
		if (free > containerLimit) {
			containerLimit = free;
		}
	}
	return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree);
}
 
Example 3
Source Project: flink   Source File: AbstractYarnClusterDescriptor.java    License: Apache License 2.0 6 votes vote down vote up
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException {
	List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

	int totalFreeMemory = 0;
	int containerLimit = 0;
	int[] nodeManagersFree = new int[nodes.size()];

	for (int i = 0; i < nodes.size(); i++) {
		NodeReport rep = nodes.get(i);
		int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0);
		nodeManagersFree[i] = free;
		totalFreeMemory += free;
		if (free > containerLimit) {
			containerLimit = free;
		}
	}
	return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree);
}
 
Example 4
Source Project: hadoop   Source File: RMNodeImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void updateMetricsForRejoinedNode(NodeState previousNodeState) {
  ClusterMetrics metrics = ClusterMetrics.getMetrics();
  metrics.incrNumActiveNodes();

  switch (previousNodeState) {
  case LOST:
    metrics.decrNumLostNMs();
    break;
  case REBOOTED:
    metrics.decrNumRebootedNMs();
    break;
  case DECOMMISSIONED:
    metrics.decrDecommisionedNMs();
    break;
  case UNHEALTHY:
    metrics.decrNumUnhealthyNMs();
    break;
  default:
    LOG.debug("Unexpected previous node state");    
  }
}
 
Example 5
Source Project: hadoop   Source File: JobImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
  // rerun previously successful map tasks
  List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
  if(taskAttemptIdList != null) {
    String mesg = "TaskAttempt killed because it ran on unusable node "
        + nodeId;
    for(TaskAttemptId id : taskAttemptIdList) {
      if(TaskType.MAP == id.getTaskId().getTaskType()) {
        // reschedule only map tasks because their outputs maybe unusable
        LOG.info(mesg + ". AttemptId:" + id);
        eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
      }
    }
  }
  // currently running task attempts on unusable nodes are handled in
  // RMContainerAllocator
}
 
Example 6
Source Project: big-c   Source File: MockNodes.java    License: Apache License 2.0 6 votes vote down vote up
public static List<RMNode> newNodes(int racks, int nodesPerRack,
                                      Resource perNode) {
  List<RMNode> list = Lists.newArrayList();
  for (int i = 0; i < racks; ++i) {
    for (int j = 0; j < nodesPerRack; ++j) {
      if (j == (nodesPerRack - 1)) {
        // One unhealthy node per rack.
        list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY));
      }
      if (j == 0) {
        // One node with label
        list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x")));
      } else {
        list.add(newNodeInfo(i, perNode));
      }
    }
  }
  return list;
}
 
Example 7
Source Project: tez   Source File: TestAMNodeTracker.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout=5000)
public void testHealthUpdateUnknownNode() {
  AppContext appContext = mock(AppContext.class);

  AMNodeTracker amNodeTracker = new AMNodeTracker(eventHandler, appContext);
  doReturn(amNodeTracker).when(appContext).getNodeTracker();
  amNodeTracker.init(new Configuration(false));
  amNodeTracker.start();

  NodeId nodeId = NodeId.newInstance("unknownhost", 2342);

  NodeReport nodeReport = generateNodeReport(nodeId, NodeState.UNHEALTHY);
  amNodeTracker.handle(new AMNodeEventStateChanged(nodeReport, 0));
  dispatcher.await();

  amNodeTracker.stop();
  // No exceptions - the status update was ignored. Not bothering to capture
  // the log message for verification.
}
 
Example 8
Source Project: big-c   Source File: JobImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
  // rerun previously successful map tasks
  List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
  if(taskAttemptIdList != null) {
    String mesg = "TaskAttempt killed because it ran on unusable node "
        + nodeId;
    for(TaskAttemptId id : taskAttemptIdList) {
      if(TaskType.MAP == id.getTaskId().getTaskType()) {
        // reschedule only map tasks because their outputs maybe unusable
        LOG.info(mesg + ". AttemptId:" + id);
        eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
      }
    }
  }
  // currently running task attempts on unusable nodes are handled in
  // RMContainerAllocator
}
 
Example 9
Source Project: hadoop   Source File: TestRMWebServicesNodes.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNodesQueryNew() throws JSONException, Exception {
  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1235", 5121);
  rm.sendNodeStarted(nm1);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("nodes").queryParam("states", NodeState.NEW.toString())
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  assertEquals("incorrect number of elements", 1, nodeArray.length());
  JSONObject info = nodeArray.getJSONObject(0);

  verifyNodeInfo(info, nm2);
}
 
Example 10
Source Project: hadoop   Source File: TestRMWebServicesNodes.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNodesQueryRunning() throws JSONException, Exception {
  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1235", 5121);
  rm.sendNodeStarted(nm1);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("nodes").queryParam("states", "running")
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  assertEquals("incorrect number of elements", 1, nodeArray.length());
}
 
Example 11
Source Project: big-c   Source File: RMNodeImpl.java    License: Apache License 2.0 6 votes vote down vote up
public void handle(RMNodeEvent event) {
  LOG.debug("Processing " + event.getNodeId() + " of type " + event.getType());
  try {
    writeLock.lock();
    NodeState oldState = getState();
    try {
       stateMachine.doTransition(event.getType(), event);
    } catch (InvalidStateTransitonException e) {
      LOG.error("Can't handle this event at current state", e);
      LOG.error("Invalid event " + event.getType() + 
          " on Node  " + this.nodeId);
    }
    if (oldState != getState()) {
      LOG.info(nodeId + " Node Transitioned from " + oldState + " to "
               + getState());
    }
  }
  
  finally {
    writeLock.unlock();
  }
}
 
Example 12
Source Project: hadoop   Source File: TestRMWebServicesNodes.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testQueryAll() throws Exception {
  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1235", 5121);
  MockNM nm3 = rm.registerNode("h3:1236", 5122);
  rm.sendNodeStarted(nm1);
  rm.sendNodeStarted(nm3);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
  rm.sendNodeLost(nm3);

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("nodes")
      .queryParam("states", Joiner.on(',').join(EnumSet.allOf(NodeState.class)))
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  assertEquals("incorrect number of elements", 3, nodeArray.length());
}
 
Example 13
Source Project: hadoop   Source File: MockNodes.java    License: Apache License 2.0 6 votes vote down vote up
public static List<RMNode> newNodes(int racks, int nodesPerRack,
                                      Resource perNode) {
  List<RMNode> list = Lists.newArrayList();
  for (int i = 0; i < racks; ++i) {
    for (int j = 0; j < nodesPerRack; ++j) {
      if (j == (nodesPerRack - 1)) {
        // One unhealthy node per rack.
        list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY));
      }
      if (j == 0) {
        // One node with label
        list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x")));
      } else {
        list.add(newNodeInfo(i, perNode));
      }
    }
  }
  return list;
}
 
Example 14
Source Project: hadoop   Source File: MockNodes.java    License: Apache License 2.0 6 votes vote down vote up
public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
    Resource perNode, String rackName, String healthReport,
    long lastHealthReportTime, int cmdPort, String hostName, NodeState state,
    Set<String> labels) {
  this.nodeId = nodeId;
  this.nodeAddr = nodeAddr;
  this.httpAddress = httpAddress;
  this.perNode = perNode;
  this.rackName = rackName;
  this.healthReport = healthReport;
  this.lastHealthReportTime = lastHealthReportTime;
  this.cmdPort = cmdPort;
  this.hostName = hostName;
  this.state = state;
  this.labels = labels;
}
 
Example 15
Source Project: big-c   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRunningDecommission() {
  RMNodeImpl node = getRunningNode();
  ClusterMetrics cm = ClusterMetrics.getMetrics();
  int initialActive = cm.getNumActiveNMs();
  int initialLost = cm.getNumLostNMs();
  int initialUnhealthy = cm.getUnhealthyNMs();
  int initialDecommissioned = cm.getNumDecommisionedNMs();
  int initialRebooted = cm.getNumRebootedNMs();
  node.handle(new RMNodeEvent(node.getNodeID(),
      RMNodeEventType.DECOMMISSION));
  Assert.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
  Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
  Assert.assertEquals("Unhealthy Nodes",
      initialUnhealthy, cm.getUnhealthyNMs());
  Assert.assertEquals("Decommissioned Nodes",
      initialDecommissioned + 1, cm.getNumDecommisionedNMs());
  Assert.assertEquals("Rebooted Nodes",
      initialRebooted, cm.getNumRebootedNMs());
  Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
 
Example 16
/**
 * Get the available v cores in cluster.
 * @param rmCommunicator 
 *
 * @return the available v cores in cluster
 * @throws IOException Signals that an I/O exception has occurred.
 */
private int getAvailableVCoresInCluster(RMCommunicator rmCommunicator) throws IOException {
	List<NodeReport> nodeReports = null;
	try {
		nodeReports = rmCommunicator.getNodeReports();
	} catch (YarnException e) {
		LOGGER.error(JumbuneRuntimeException.throwYarnException(e.getStackTrace()));
	}
	Set<String> hostname = new HashSet<String>();
	
	int totalVCores = 0;
	int usedVCores = 0;
	for(NodeReport report: nodeReports){
		if(!hostname.contains(report.getHttpAddress())  && report.getNodeState().equals(NodeState.RUNNING)){
		hostname.add(report.getHttpAddress());
		totalVCores += report.getCapability().getVirtualCores();
		if(report.getUsed()!=null){
			usedVCores += report.getUsed().getVirtualCores();
			}
		}
	}
	int availableVCores = totalVCores - usedVCores;
	return availableVCores ;
}
 
Example 17
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRunningExpire() {
  RMNodeImpl node = getRunningNode();
  ClusterMetrics cm = ClusterMetrics.getMetrics();
  int initialActive = cm.getNumActiveNMs();
  int initialLost = cm.getNumLostNMs();
  int initialUnhealthy = cm.getUnhealthyNMs();
  int initialDecommissioned = cm.getNumDecommisionedNMs();
  int initialRebooted = cm.getNumRebootedNMs();
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
  Assert.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
  Assert.assertEquals("Lost Nodes", initialLost + 1, cm.getNumLostNMs());
  Assert.assertEquals("Unhealthy Nodes",
      initialUnhealthy, cm.getUnhealthyNMs());
  Assert.assertEquals("Decommissioned Nodes",
      initialDecommissioned, cm.getNumDecommisionedNMs());
  Assert.assertEquals("Rebooted Nodes",
      initialRebooted, cm.getNumRebootedNMs());
  Assert.assertEquals(NodeState.LOST, node.getState());
}
 
Example 18
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnhealthyExpire() {
  RMNodeImpl node = getUnhealthyNode();
  ClusterMetrics cm = ClusterMetrics.getMetrics();
  int initialActive = cm.getNumActiveNMs();
  int initialLost = cm.getNumLostNMs();
  int initialUnhealthy = cm.getUnhealthyNMs();
  int initialDecommissioned = cm.getNumDecommisionedNMs();
  int initialRebooted = cm.getNumRebootedNMs();
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
  Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
  Assert.assertEquals("Lost Nodes", initialLost + 1, cm.getNumLostNMs());
  Assert.assertEquals("Unhealthy Nodes",
      initialUnhealthy - 1, cm.getUnhealthyNMs());
  Assert.assertEquals("Decommissioned Nodes",
      initialDecommissioned, cm.getNumDecommisionedNMs());
  Assert.assertEquals("Rebooted Nodes",
      initialRebooted, cm.getNumRebootedNMs());
  Assert.assertEquals(NodeState.LOST, node.getState());
}
 
Example 19
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRunningRebooting() {
  RMNodeImpl node = getRunningNode();
  ClusterMetrics cm = ClusterMetrics.getMetrics();
  int initialActive = cm.getNumActiveNMs();
  int initialLost = cm.getNumLostNMs();
  int initialUnhealthy = cm.getUnhealthyNMs();
  int initialDecommissioned = cm.getNumDecommisionedNMs();
  int initialRebooted = cm.getNumRebootedNMs();
  node.handle(new RMNodeEvent(node.getNodeID(),
      RMNodeEventType.REBOOTING));
  Assert.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
  Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
  Assert.assertEquals("Unhealthy Nodes",
      initialUnhealthy, cm.getUnhealthyNMs());
  Assert.assertEquals("Decommissioned Nodes",
      initialDecommissioned, cm.getNumDecommisionedNMs());
  Assert.assertEquals("Rebooted Nodes",
      initialRebooted + 1, cm.getNumRebootedNMs());
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example 20
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReconnect() {
  RMNodeImpl node = getRunningNode();
  ClusterMetrics cm = ClusterMetrics.getMetrics();
  int initialActive = cm.getNumActiveNMs();
  int initialLost = cm.getNumLostNMs();
  int initialUnhealthy = cm.getUnhealthyNMs();
  int initialDecommissioned = cm.getNumDecommisionedNMs();
  int initialRebooted = cm.getNumRebootedNMs();
  node.handle(new RMNodeReconnectEvent(node.getNodeID(), node, null, null));
  Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
  Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
  Assert.assertEquals("Unhealthy Nodes",
      initialUnhealthy, cm.getUnhealthyNMs());
  Assert.assertEquals("Decommissioned Nodes",
      initialDecommissioned, cm.getNumDecommisionedNMs());
  Assert.assertEquals("Rebooted Nodes",
      initialRebooted, cm.getNumRebootedNMs());
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example 21
/**
 * Get the total memory available in cluster.
 * @param rmCommunicator 
 *
 * @return the total memory available in cluster
 * @throws YarnException the yarn exception
 * @throws IOException Signals that an I/O exception has occurred.
 */
private int getTotalMemoryAvailableInCluster(RMCommunicator rmCommunicator) throws YarnException, IOException{
	List<NodeReport> reports=rmCommunicator.getNodeReports();
	int availableMemory=0;
	
	Set<String> hostname = new HashSet<String>();
	
	for (NodeReport nodeReport : reports) {
		if(!hostname.contains(nodeReport.getHttpAddress())  && nodeReport.getNodeState().equals(NodeState.RUNNING)){
		hostname.add(nodeReport.getHttpAddress());
		availableMemory+=nodeReport.getCapability().getMemory() - 
				(nodeReport.getUsed()==null?0:nodeReport.getUsed().getMemory());
		}
	}
	return availableMemory;
}
 
Example 22
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example 23
Source Project: big-c   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example 24
Source Project: hadoop   Source File: NodeCLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Lists the nodes matching the given node states
 * 
 * @param nodeStates
 * @throws YarnException
 * @throws IOException
 */
private void listClusterNodes(Set<NodeState> nodeStates) 
          throws YarnException, IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
  List<NodeReport> nodesReport = client.getNodeReports(
                                     nodeStates.toArray(new NodeState[0]));
  writer.println("Total Nodes:" + nodesReport.size());
  writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
      "Number-of-Running-Containers");
  for (NodeReport nodeReport : nodesReport) {
    writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
        .getNodeState(), nodeReport.getHttpAddress(), nodeReport
        .getNumContainers());
  }
  writer.flush();
}
 
Example 25
Source Project: hadoop   Source File: TestYarnCLI.java    License: Apache License 2.0 6 votes vote down vote up
private List<NodeReport> getNodeReports(int noOfNodes, NodeState state,
    boolean emptyNodeLabel) {
  List<NodeReport> nodeReports = new ArrayList<NodeReport>();

  for (int i = 0; i < noOfNodes; i++) {
    Set<String> nodeLabels = null;
    if (!emptyNodeLabel) {
      // node labels is not ordered, but when we output it, it should be
      // ordered
      nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
    }
    NodeReport nodeReport = NodeReport.newInstance(NodeId
      .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
        "rack1", Records.newRecord(Resource.class), Records
            .newRecord(Resource.class), 0, "", 0, nodeLabels);
    nodeReports.add(nodeReport);
  }
  return nodeReports;
}
 
Example 26
Source Project: big-c   Source File: JobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void transition(JobImpl job, JobEvent event) {
  JobUpdatedNodesEvent updateEvent = (JobUpdatedNodesEvent) event;
  for(NodeReport nr: updateEvent.getUpdatedNodes()) {
    NodeState nodeState = nr.getNodeState();
    if(nodeState.isUnusable()) {
      // act on the updates
      job.actOnUnusableNode(nr.getNodeId(), nodeState);
    }
  }
}
 
Example 27
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
 
Example 28
Source Project: flink   Source File: AbstractYarnClusterDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
 
Example 29
Source Project: hadoop   Source File: NodeInfo.java    License: Apache License 2.0 5 votes vote down vote up
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
    Resource perNode, String rackName, String healthReport,
    int cmdPort, String hostName, NodeState state) {
  this.nodeId = nodeId;
  this.nodeAddr = nodeAddr;
  this.httpAddress = httpAddress;
  this.perNode = perNode;
  this.rackName = rackName;
  this.healthReport = healthReport;
  this.cmdPort = cmdPort;
  this.hostName = hostName;
  this.state = state;
  toCleanUpApplications = new ArrayList<ApplicationId>();
  toCleanUpContainers = new ArrayList<ContainerId>();
}
 
Example 30
Source Project: big-c   Source File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}