Java Code Examples for org.apache.hadoop.yarn.api.records.NodeReport#getCapability()

The following examples show how to use org.apache.hadoop.yarn.api.records.NodeReport#getCapability() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YarnBinding.java    From mr4c with Apache License 2.0 6 votes vote down vote up 
public static List<ResourceLimit> computeResourceLimits(Iterable<NodeReport> reports) {
	if ( !reports.iterator().hasNext() ) {
		return Collections.emptyList();
	}

	List<ResourceLimit> limits = new ArrayList<ResourceLimit>();
	int cores = 0;
	int memory = 0;
	for ( NodeReport report : reports ) {
		org.apache.hadoop.yarn.api.records.Resource cap = report.getCapability();
		cores = Math.max(cores, cap.getVirtualCores());
		memory = Math.max(memory, cap.getMemory());
	}
	limits.add(new ResourceLimit(Resource.CORES, cores, LimitSource.CLUSTER));
	limits.add(new ResourceLimit(Resource.MEMORY, memory, LimitSource.CLUSTER));
	return limits;
}
Example 2
Source File: AbstractYarnClusterDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
Example 3
Source File: AbstractYarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
				q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
Example 4
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public String getClusterDescription() {

	try {
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		PrintStream ps = new PrintStream(baos);

		YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();

		ps.append("NodeManagers in the ClusterClient " + metrics.getNumNodeManagers());
		List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING);
		final String format = "|%-16s |%-16s %n";
		ps.printf("|Property         |Value          %n");
		ps.println("+---------------------------------------+");
		int totalMemory = 0;
		int totalCores = 0;
		for (NodeReport rep : nodes) {
			final Resource res = rep.getCapability();
			totalMemory += res.getMemory();
			totalCores += res.getVirtualCores();
			ps.format(format, "NodeID", rep.getNodeId());
			ps.format(format, "Memory", res.getMemory() + " MB");
			ps.format(format, "vCores", res.getVirtualCores());
			ps.format(format, "HealthReport", rep.getHealthReport());
			ps.format(format, "Containers", rep.getNumContainers());
			ps.println("+---------------------------------------+");
		}
		ps.println("Summary: totalMemory " + totalMemory + " totalCores " + totalCores);
		List<QueueInfo> qInfo = yarnClient.getAllQueues();
		for (QueueInfo q : qInfo) {
			ps.println("Queue: " + q.getQueueName() + ", Current Capacity: " + q.getCurrentCapacity() + " Max Capacity: " +
					q.getMaximumCapacity() + " Applications: " + q.getApplications().size());
		}
		return baos.toString();
	} catch (Exception e) {
		throw new RuntimeException("Couldn't get cluster description", e);
	}
}
Example 5
Source File: Client.java    From stratosphere with Apache License 2.0 5 votes vote down vote up 
private void showClusterMetrics(YarnClient yarnClient)
		throws YarnException, IOException {
	YarnClusterMetrics metrics = yarnClient.getYarnClusterMetrics();
	System.out.println("NodeManagers in Cluster " + metrics.getNumNodeManagers());
	List<NodeReport> nodes = yarnClient.getNodeReports();
	final String format = "|%-16s |%-16s %n";
	System.out.printf("|Property         |Value          %n");
	System.out.println("+---------------------------------------+");
	int totalMemory = 0;
	int totalCores = 0;
	for(NodeReport rep : nodes) {
		final Resource res = rep.getCapability();
		totalMemory += res.getMemory();
		totalCores += res.getVirtualCores();
		System.out.format(format, "NodeID", rep.getNodeId());
		System.out.format(format, "Memory", res.getMemory()+" MB");
		System.out.format(format, "vCores", res.getVirtualCores());
		System.out.format(format, "HealthReport", rep.getHealthReport());
		System.out.format(format, "Containers", rep.getNumContainers());
		System.out.println("+---------------------------------------+");
	}
	System.out.println("Summary: totalMemory "+totalMemory+" totalCores "+totalCores);
	List<QueueInfo> qInfo = yarnClient.getAllQueues();
	for(QueueInfo q : qInfo) {
		System.out.println("Queue: "+q.getQueueName()+", Current Capacity: "+q.getCurrentCapacity()+" Max Capacity: "+q.getMaximumCapacity()+" Applications: "+q.getApplications().size());
	}
	yarnClient.stop();
	System.exit(0);
}
Example 6
Source File: PlacementPolicyTestRun.java    From twill with Apache License 2.0 4 votes vote down vote up 
/**
 * Verify the cluster configuration (number and capability of node managers) required for the tests.
 */
@BeforeClass
public static void verifyClusterCapability() throws InterruptedException {
  // Ignore verifications if it is running against older Hadoop versions which does not support blacklists.
  Assume.assumeTrue(YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_22));

  // All runnables in this test class use same resource specification for the sake of convenience.
  resource = ResourceSpecification.Builder.with()
    .setVirtualCores(RUNNABLE_CORES)
    .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA)
    .build();
  twoInstancesResource = ResourceSpecification.Builder.with()
    .setVirtualCores(RUNNABLE_CORES)
    .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA)
    .setInstances(2)
    .build();

  // The tests need exactly three NodeManagers in the cluster.
  int trials = 0;
  while (trials++ < 20) {
    try {
      nodeReports = TWILL_TESTER.getNodeReports();
      if (nodeReports != null && nodeReports.size() == 3) {
        break;
      }
    } catch (Exception e) {
      LOG.error("Failed to get node reports", e);
    }
    LOG.warn("NodeManagers != 3. {}", nodeReports);
    TimeUnit.SECONDS.sleep(1);
  }

  // All NodeManagers should have enough capacity available to accommodate at least two runnables.
  for (NodeReport nodeReport : nodeReports) {
    Resource capability = nodeReport.getCapability();
    Resource used = nodeReport.getUsed();
    Assert.assertNotNull(capability);
    if (used != null) {
      Assert.assertTrue(2 * resource.getMemorySize() < capability.getMemory() - used.getMemory());
    } else {
      Assert.assertTrue(2 * resource.getMemorySize() < capability.getMemory());
    }
  }
}