Java Code Examples for org.apache.hadoop.yarn.api.records.NodeReport

The following examples show how to use org.apache.hadoop.yarn.api.records.NodeReport. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException {
	List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);

	int totalFreeMemory = 0;
	int containerLimit = 0;
	int[] nodeManagersFree = new int[nodes.size()];

	for (int i = 0; i < nodes.size(); i++) {
		NodeReport rep = nodes.get(i);
		int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0);
		nodeManagersFree[i] = free;
		totalFreeMemory += free;
		if (free > containerLimit) {
			containerLimit = free;
		}
	}
	return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree);
}
 
Example 2
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static AllocateResponse newAllocateResponse(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt) {
  AllocateResponse response = recordFactory
      .newRecordInstance(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);

  return response;
}
 
Example 3
Source Project: hadoop   Source File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
    throws YarnException {
  GetClusterNodesResponse response = 
    recordFactory.newRecordInstance(GetClusterNodesResponse.class);
  EnumSet<NodeState> nodeStates = request.getNodeStates();
  if (nodeStates == null || nodeStates.isEmpty()) {
    nodeStates = EnumSet.allOf(NodeState.class);
  }
  Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext,
      nodeStates);
  
  List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
  for (RMNode nodeInfo : nodes) {
    nodeReports.add(createNodeReports(nodeInfo));
  }
  response.setNodeReports(nodeReports);
  return response;
}
 
Example 4
Source Project: hadoop   Source File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example 5
Source Project: hadoop   Source File: NodeCLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Lists the nodes matching the given node states
 * 
 * @param nodeStates
 * @throws YarnException
 * @throws IOException
 */
private void listClusterNodes(Set<NodeState> nodeStates) 
          throws YarnException, IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));
  List<NodeReport> nodesReport = client.getNodeReports(
                                     nodeStates.toArray(new NodeState[0]));
  writer.println("Total Nodes:" + nodesReport.size());
  writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
      "Number-of-Running-Containers");
  for (NodeReport nodeReport : nodesReport) {
    writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
        .getNodeState(), nodeReport.getHttpAddress(), nodeReport
        .getNumContainers());
  }
  writer.flush();
}
 
Example 6
Source Project: big-c   Source File: AllocateResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens) {
  AllocateResponse response = Records.newRecord(AllocateResponse.class);
  response.setNumClusterNodes(numClusterNodes);
  response.setResponseId(responseId);
  response.setCompletedContainersStatuses(completedContainers);
  response.setAllocatedContainers(allocatedContainers);
  response.setUpdatedNodes(updatedNodes);
  response.setAvailableResources(availResources);
  response.setAMCommand(command);
  response.setPreemptionMessage(preempt);
  response.setNMTokens(nmTokens);
  return response;
}
 
Example 7
Source Project: hadoop   Source File: AllocateResponse.java    License: Apache License 2.0 6 votes vote down vote up
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
    List<ContainerStatus> completedContainers,
    List<Container> allocatedContainers, List<NodeReport> updatedNodes,
    Resource availResources, AMCommand command, int numClusterNodes,
    PreemptionMessage preempt, List<NMToken> nmTokens,
    List<ContainerResourceIncrease> increasedContainers,
    List<ContainerResourceDecrease> decreasedContainers) {
  AllocateResponse response = newInstance(responseId, completedContainers,
      allocatedContainers, updatedNodes, availResources, command,
      numClusterNodes, preempt, nmTokens);
  response.setIncreasedContainers(increasedContainers);
  response.setDecreasedContainers(decreasedContainers);
  return response;
}
 
Example 8
Source Project: attic-apex-core   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Assign host to container given affinity and anti-affinity constraints and resource availibility on node
 * @param host
 * @param antiHosts
 * @param antiPreferredHosts
 * @param grpObj
 * @param nodeLocalSet
 * @param aggrMemory
 * @param vCores
 * @return
 */
public String assignHost(String host, List<String> antiHosts, List<String> antiPreferredHosts, HostOperatorSet grpObj, Set<PTOperator> nodeLocalSet, int aggrMemory, int vCores)
{
  for (Map.Entry<String, NodeReport> nodeEntry : nodeReportMap.entrySet()) {
    if (nodeEntry.getValue().getNodeState() == NodeState.RUNNING) {
      int memAvailable = nodeEntry.getValue().getCapability().getMemory() - nodeEntry.getValue().getUsed().getMemory();
      int vCoresAvailable = nodeEntry.getValue().getCapability().getVirtualCores() - nodeEntry.getValue().getUsed().getVirtualCores();
      if (memAvailable >= aggrMemory && vCoresAvailable >= vCores && !antiHosts.contains(nodeEntry.getKey()) && !antiPreferredHosts.contains(nodeEntry.getKey())) {
        host = nodeEntry.getKey();
        grpObj.setHost(host);
        nodeLocalMapping.put(nodeLocalSet, host);

        return host;
      }
    }
  }
  return null;
}
 
Example 9
Source Project: ignite   Source File: ApplicationMaster.java    License: Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override public synchronized void onNodesUpdated(List<NodeReport> updated) {
    for (NodeReport node : updated) {
        // If node unusable.
        if (node.getNodeState().isUnusable()) {
            for (IgniteContainer cont : containers.values()) {
                if (cont.nodeId().equals(node.getNodeId())) {
                    containers.remove(cont.id());

                    log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                        new Object[]{node.getNodeId().getHost(), node.getNodeState()});
                }
            }

            log.log(Level.WARNING, "Node is unusable. Node: {0}, state: {1}.",
                new Object[]{node.getNodeId().getHost(), node.getNodeState()});
        }
    }
}
 
Example 10
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public AllocateResponse allocate(AllocateRequest request)
    throws YarnException, IOException {
  lastAsk = request.getAskList();
  for (ResourceRequest req : lastAsk) {
    if (ResourceRequest.ANY.equals(req.getResourceName())) {
      Priority priority = req.getPriority();
      if (priority.equals(RMContainerAllocator.PRIORITY_MAP)) {
        lastAnyAskMap = req.getNumContainers();
      } else if (priority.equals(RMContainerAllocator.PRIORITY_REDUCE)){
        lastAnyAskReduce = req.getNumContainers();
      }
    }
  }
  AllocateResponse response =  AllocateResponse.newInstance(
      request.getResponseId(),
      containersToComplete, containersToAllocate,
      Collections.<NodeReport>emptyList(),
      Resource.newInstance(512000, 1024, 1024), null, 10, null,
      Collections.<NMToken>emptyList());
  containersToComplete.clear();
  containersToAllocate.clear();
  return response;
}
 
Example 11
Source Project: big-c   Source File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example 12
Source Project: big-c   Source File: TestYarnCLI.java    License: Apache License 2.0 6 votes vote down vote up
private List<NodeReport> getNodeReports(int noOfNodes, NodeState state,
    boolean emptyNodeLabel) {
  List<NodeReport> nodeReports = new ArrayList<NodeReport>();

  for (int i = 0; i < noOfNodes; i++) {
    Set<String> nodeLabels = null;
    if (!emptyNodeLabel) {
      // node labels is not ordered, but when we output it, it should be
      // ordered
      nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
    }
    NodeReport nodeReport = NodeReport.newInstance(NodeId
      .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
        "rack1", Records.newRecord(Resource.class), Records
            .newRecord(Resource.class), 0, "", 0, nodeLabels);
    nodeReports.add(nodeReport);
  }
  return nodeReports;
}
 
Example 13
Source Project: big-c   Source File: TestAllocateResponse.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Test
public void testAllocateResponseWithoutIncDecContainers() {
  AllocateResponse r =
      AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
          new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
          AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(), null, null);

  // serde
  AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
  r = new AllocateResponsePBImpl(p);

  // check value
  Assert.assertEquals(0, r.getIncreasedContainers().size());
  Assert.assertEquals(0, r.getDecreasedContainers().size());
}
 
Example 14
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
    String httpAddress, String rackName, Resource used, Resource capability,
    int numContainers, String healthReport, long lastHealthReportTime,
    Set<String> nodeLabels) {
  NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class);
  nodeReport.setNodeId(nodeId);
  nodeReport.setNodeState(nodeState);
  nodeReport.setHttpAddress(httpAddress);
  nodeReport.setRackName(rackName);
  nodeReport.setUsed(used);
  nodeReport.setCapability(capability);
  nodeReport.setNumContainers(numContainers);
  nodeReport.setHealthReport(healthReport);
  nodeReport.setLastHealthReportTime(lastHealthReportTime);
  nodeReport.setNodeLabels(nodeLabels);
  return nodeReport;
}
 
Example 15
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
 
Example 16
Source Project: tez   Source File: TaskSchedulerManager.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void nodesUpdated(int schedulerId, List<NodeReport> updatedNodes) {
  for (NodeReport nr : updatedNodes) {
    // Scheduler will find out from the node, if at all.
    // Relying on the RM to not allocate containers on an unhealthy node.
    eventHandler.handle(new AMNodeEventStateChanged(nr, schedulerId));
  }
}
 
Example 17
Source Project: flink   Source File: AbstractYarnClusterDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
 
Example 18
Source Project: hadoop   Source File: GetClusterNodesResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setNodeReports(List<NodeReport> nodeManagers) {
  if (nodeManagers == null) {
    builder.clearNodeReports();
  }
  this.nodeManagerInfoList = nodeManagers;
}
 
Example 19
Source Project: hadoop   Source File: GetClusterNodesResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initLocalNodeManagerInfosList() {
  if (this.nodeManagerInfoList != null) {
    return;
  }
  GetClusterNodesResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<NodeReportProto> list = p.getNodeReportsList();
  nodeManagerInfoList = new ArrayList<NodeReport>();

  for (NodeReportProto a : list) {
    nodeManagerInfoList.add(convertFromProtoFormat(a));
  }
}
 
Example 20
Source Project: hadoop   Source File: GetClusterNodesResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void addLocalNodeManagerInfosToProto() {
  maybeInitBuilder();
  builder.clearNodeReports();
  if (nodeManagerInfoList == null)
    return;
  Iterable<NodeReportProto> iterable = new Iterable<NodeReportProto>() {
    @Override
    public Iterator<NodeReportProto> iterator() {
      return new Iterator<NodeReportProto>() {

        Iterator<NodeReport> iter = nodeManagerInfoList.iterator();

        @Override
        public boolean hasNext() {
          return iter.hasNext();
        }

        @Override
        public NodeReportProto next() {
          return convertToProtoFormat(iter.next());
        }

        @Override
        public void remove() {
          throw new UnsupportedOperationException();

        }
      };

    }
  };
  builder.addAllNodeReports(iterable);
}
 
Example 21
private AMRMClientAsync<AMRMClient.ContainerRequest> initClient(Configuration conf) throws YarnException, IOException {
    AMRMClientAsync.CallbackHandler allocListener = new AMRMClientAsync.CallbackHandler() {
        @Override
        public void onContainersCompleted(List<ContainerStatus> statuses) {
        }

        @Override
        public void onContainersAllocated(List<Container> containers) {
        }

        @Override
        public void onShutdownRequest() {
            LOG.warn("Shutting down");
            end.set(true);
        }

        @Override
        public void onNodesUpdated(List<NodeReport> updatedNodes) {
        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public void onError(Throwable e) {
            LOG.error("Unexpected error", e);
            end.set(true);
        }
    };
    AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
    rmClient.init(conf);
    rmClient.start();

    // Register with ResourceManager
    rmClient.registerApplicationMaster(Utils.localHostName(), 0, "");

    return rmClient;
}
 
Example 22
Source Project: hadoop   Source File: AllocateResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void initLocalNewNodeReportList() {
  if (this.updatedNodes != null) {
    return;
  }
  AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<NodeReportProto> list = p.getUpdatedNodesList();
  updatedNodes = new ArrayList<NodeReport>(list.size());

  for (NodeReportProto n : list) {
    updatedNodes.add(convertFromProtoFormat(n));
  }
}
 
Example 23
Source Project: hadoop   Source File: AllocateResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized Iterable<NodeReportProto>
getNodeReportProtoIterable(
    final List<NodeReport> newNodeReportsList) {
  maybeInitBuilder();
  return new Iterable<NodeReportProto>() {
    @Override
    public synchronized Iterator<NodeReportProto> iterator() {
      return new Iterator<NodeReportProto>() {

        Iterator<NodeReport> iter = newNodeReportsList.iterator();

        @Override
        public synchronized boolean hasNext() {
          return iter.hasNext();
        }

        @Override
        public synchronized NodeReportProto next() {
          return convertToProtoFormat(iter.next());
        }

        @Override
        public synchronized void remove() {
          throw new UnsupportedOperationException();

        }
      };
    }
  };
}
 
Example 24
Source Project: attic-apex-core   Source File: HostLocalTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerLocal()
{
  LogicalPlan dag = new LogicalPlan();
  dag.getAttributes().put(com.datatorrent.api.Context.DAGContext.APPLICATION_PATH, new File("target", HostLocalTest.class.getName()).getAbsolutePath());
  dag.setAttribute(OperatorContext.STORAGE_AGENT, new MemoryStorageAgent());

  GenericTestOperator o1 = dag.addOperator("o1", GenericTestOperator.class);
  dag.getMeta(o1).getAttributes().put(OperatorContext.LOCALITY_HOST, "host2");

  GenericTestOperator partitioned = dag.addOperator("partitioned", GenericTestOperator.class);
  dag.addStream("o1_outport1", o1.outport1, partitioned.inport1).setLocality(Locality.CONTAINER_LOCAL);
  dag.setOperatorAttribute(o1, OperatorContext.MEMORY_MB, 256);
  dag.setOperatorAttribute(partitioned,OperatorContext.MEMORY_MB,256);

  StreamingContainerManager scm = new StreamingContainerManager(dag);

  ResourceRequestHandler rr = new ResourceRequestHandler();

  int containerMem = 1000;
  Map<String, NodeReport> nodeReports = Maps.newHashMap();
  NodeReport nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host1", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);
  nr = BuilderUtils.newNodeReport(BuilderUtils.newNodeId("host2", 0), NodeState.RUNNING, "httpAddress", "rackName", BuilderUtils.newResource(0, 0), BuilderUtils.newResource(containerMem * 2, 2), 0, null, 0);
  nodeReports.put(nr.getNodeId().getHost(), nr);

  // set resources
  rr.updateNodeReports(Lists.newArrayList(nodeReports.values()));
  Assert.assertEquals("number of containers is 1", 1, scm.containerStartRequests.size());
  for (ContainerStartRequest csr : scm.containerStartRequests) {
    String host = rr.getHost(csr, true);
    csr.container.host = host;
    Assert.assertEquals("Hosts set to host2", "host2", host);
  }
}
 
Example 25
Source Project: big-c   Source File: TestYarnCLI.java    License: Apache License 2.0 5 votes vote down vote up
private List<NodeReport> getNodeReports(
    List<NodeReport> nodeReports,
    Set<NodeState> nodeStates) {
  List<NodeReport> reports = new ArrayList<NodeReport>();

  for (NodeReport nodeReport : nodeReports) {
    if (nodeStates.contains(nodeReport.getNodeState())) {
      reports.add(nodeReport);
    }
  }
  return reports;
}
 
Example 26
Source Project: big-c   Source File: RMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void handleUpdatedNodes(AllocateResponse response) {
  // send event to the job about on updated nodes
  List<NodeReport> updatedNodes = response.getUpdatedNodes();
  if (!updatedNodes.isEmpty()) {

    // send event to the job to act upon completed tasks
    eventHandler.handle(new JobUpdatedNodesEvent(getJob().getID(),
        updatedNodes));

    // act upon running tasks
    HashSet<NodeId> unusableNodes = new HashSet<NodeId>();
    for (NodeReport nr : updatedNodes) {
      NodeState nodeState = nr.getNodeState();
      if (nodeState.isUnusable()) {
        unusableNodes.add(nr.getNodeId());
      }
    }
    for (int i = 0; i < 2; ++i) {
      HashMap<TaskAttemptId, Container> taskSet = i == 0 ? assignedRequests.maps
          : assignedRequests.reduces;
      // kill running containers
      for (Map.Entry<TaskAttemptId, Container> entry : taskSet.entrySet()) {
        TaskAttemptId tid = entry.getKey();
        NodeId taskAttemptNodeId = entry.getValue().getNodeId();
        if (unusableNodes.contains(taskAttemptNodeId)) {
          LOG.info("Killing taskAttempt:" + tid
              + " because it is running on unusable node:"
              + taskAttemptNodeId);
          eventHandler.handle(new TaskAttemptKillEvent(tid,
              "TaskAttempt killed because it ran on unusable node"
                  + taskAttemptNodeId));
        }
      }
    }
  }
}
 
Example 27
Source Project: hadoop   Source File: YarnClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<NodeReport> getNodeReports(NodeState... states) throws YarnException,
    IOException {
  EnumSet<NodeState> statesSet = (states.length == 0) ?
      EnumSet.allOf(NodeState.class) : EnumSet.noneOf(NodeState.class);
  for (NodeState state : states) {
    statesSet.add(state);
  }
  GetClusterNodesRequest request = GetClusterNodesRequest
      .newInstance(statesSet);
  GetClusterNodesResponse response = rmClient.getClusterNodes(request);
  return response.getNodeReports();
}
 
Example 28
Source Project: big-c   Source File: TestApplicationClientProtocolOnHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 15000)
public void testGetClusterNodesOnHA() throws Exception {
  List<NodeReport> reports = client.getNodeReports(NodeState.RUNNING);
  Assert.assertTrue(reports != null && !reports.isEmpty());
  Assert.assertEquals(cluster.createFakeNodeReports(),
      reports);
}
 
Example 29
/**
 * Gets the container utilization alert.
 *
 * @param cluster the cluster
 * @param rmCommunicator 
 * @return the container utilization alert
 * @throws IOException Signals that an I/O exception has occurred.
 * @throws YarnException the yarn exception
 */
public List <AlertInfo> getContainerUtilizationAlert(Cluster cluster, RMCommunicator rmCommunicator){

	List<AlertInfo> containerAlert = new ArrayList<AlertInfo>();
	float containerBasedonVcores, containerBasedonMemory, container;

	List<NodeReport> nodeReports;
	try {
		nodeReports = rmCommunicator.getNodeReports();
	
	for(NodeReport nr:nodeReports){

		if (nr.getNumContainers() > 0){
			containerBasedonVcores = nr.getCapability().getVirtualCores()/getMinimumParameterMandatoryForContainer(ExtendedConstants.YARN_CONTAINER_MINIMUM_VCORE,1,cluster);
			containerBasedonMemory = nr.getCapability().getMemory()/getMinimumParameterMandatoryForContainer(ExtendedConstants.YARN_CONTAINER_MINIMUM_MEMORY,1,cluster);
			container = Math.min(containerBasedonVcores, containerBasedonMemory);
			if (((nr.getNumContainers()/container)*100) > 80){
				String host = nr.getNodeId().getHost();
				if(!host.contains(".")){
					host = InetAddress.getByName(host).getHostAddress();
				}
				AlertInfo alertInfo = new AlertInfo(ExtendedConstants.CRITICAL_LEVEL,host,"Container capacity is over utilized", getDate() );
				containerAlert.add(alertInfo);
			}
		}
	}} catch (YarnException | IOException e) {
		LOGGER.error("Unable to get container utilization alert due to : ",e);
	}

	return containerAlert;
}
 
Example 30
Source Project: big-c   Source File: TypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
public static TaskTrackerInfo[] fromYarnNodes(List<NodeReport> nodes) {
  List<TaskTrackerInfo> taskTrackers = new ArrayList<TaskTrackerInfo>();
  for (NodeReport node : nodes) {
    taskTrackers.add(fromYarn(node));
  }
  return taskTrackers.toArray(new TaskTrackerInfo[nodes.size()]);
}