Java Code Examples for org.apache.hadoop.yarn.api.records.NodeReport#getUsed()
The following examples show how to use
org.apache.hadoop.yarn.api.records.NodeReport#getUsed() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractYarnClusterDescriptor.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
Example 2
Source File: AbstractYarnClusterDescriptor.java From flink with Apache License 2.0 | 6 votes |
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
Example 3
Source File: YarnClusterDescriptor.java From flink with Apache License 2.0 | 6 votes |
private ClusterResourceDescription getCurrentFreeClusterResources(YarnClient yarnClient) throws YarnException, IOException { List<NodeReport> nodes = yarnClient.getNodeReports(NodeState.RUNNING); int totalFreeMemory = 0; int containerLimit = 0; int[] nodeManagersFree = new int[nodes.size()]; for (int i = 0; i < nodes.size(); i++) { NodeReport rep = nodes.get(i); int free = rep.getCapability().getMemory() - (rep.getUsed() != null ? rep.getUsed().getMemory() : 0); nodeManagersFree[i] = free; totalFreeMemory += free; if (free > containerLimit) { containerLimit = free; } } return new ClusterResourceDescription(totalFreeMemory, containerLimit, nodeManagersFree); }
Example 4
Source File: ClusterProfilingHelper.java From jumbune with GNU Lesser General Public License v3.0 | 6 votes |
/** * Get the available v cores in cluster. * @param rmCommunicator * * @return the available v cores in cluster * @throws IOException Signals that an I/O exception has occurred. */ private int getAvailableVCoresInCluster(RMCommunicator rmCommunicator) throws IOException { List<NodeReport> nodeReports = null; try { nodeReports = rmCommunicator.getNodeReports(); } catch (YarnException e) { LOGGER.error(JumbuneRuntimeException.throwYarnException(e.getStackTrace())); } Set<String> hostname = new HashSet<String>(); int totalVCores = 0; int usedVCores = 0; for(NodeReport report: nodeReports){ if(!hostname.contains(report.getHttpAddress()) && report.getNodeState().equals(NodeState.RUNNING)){ hostname.add(report.getHttpAddress()); totalVCores += report.getCapability().getVirtualCores(); if(report.getUsed()!=null){ usedVCores += report.getUsed().getVirtualCores(); } } } int availableVCores = totalVCores - usedVCores; return availableVCores ; }
Example 5
Source File: ClusterProfilingHelper.java From jumbune with GNU Lesser General Public License v3.0 | 6 votes |
/** * Get the total memory available in cluster. * @param rmCommunicator * * @return the total memory available in cluster * @throws YarnException the yarn exception * @throws IOException Signals that an I/O exception has occurred. */ private int getTotalMemoryAvailableInCluster(RMCommunicator rmCommunicator) throws YarnException, IOException{ List<NodeReport> reports=rmCommunicator.getNodeReports(); int availableMemory=0; Set<String> hostname = new HashSet<String>(); for (NodeReport nodeReport : reports) { if(!hostname.contains(nodeReport.getHttpAddress()) && nodeReport.getNodeState().equals(NodeState.RUNNING)){ hostname.add(nodeReport.getHttpAddress()); availableMemory+=nodeReport.getCapability().getMemory() - (nodeReport.getUsed()==null?0:nodeReport.getUsed().getMemory()); } } return availableMemory; }
Example 6
Source File: PlacementPolicyTestRun.java From twill with Apache License 2.0 | 5 votes |
/** * Helper function to verify DISTRIBUTED placement policies. * Returns the number of NodeManagers on which runnables got provisioned. * @return number of NodeManagers on which runnables got provisioned. */ private int getProvisionedNodeManagerCount() throws Exception { int provisionedNodeManagerCount = 0; for (NodeReport nodeReport : getNodeReports()) { Resource used = nodeReport.getUsed(); if (used != null && used.getMemory() > 0) { provisionedNodeManagerCount++; } } return provisionedNodeManagerCount; }
Example 7
Source File: PlacementPolicyTestRun.java From twill with Apache License 2.0 | 4 votes |
/** * Verify the cluster configuration (number and capability of node managers) required for the tests. */ @BeforeClass public static void verifyClusterCapability() throws InterruptedException { // Ignore verifications if it is running against older Hadoop versions which does not support blacklists. Assume.assumeTrue(YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_22)); // All runnables in this test class use same resource specification for the sake of convenience. resource = ResourceSpecification.Builder.with() .setVirtualCores(RUNNABLE_CORES) .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA) .build(); twoInstancesResource = ResourceSpecification.Builder.with() .setVirtualCores(RUNNABLE_CORES) .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA) .setInstances(2) .build(); // The tests need exactly three NodeManagers in the cluster. int trials = 0; while (trials++ < 20) { try { nodeReports = TWILL_TESTER.getNodeReports(); if (nodeReports != null && nodeReports.size() == 3) { break; } } catch (Exception e) { LOG.error("Failed to get node reports", e); } LOG.warn("NodeManagers != 3. {}", nodeReports); TimeUnit.SECONDS.sleep(1); } // All NodeManagers should have enough capacity available to accommodate at least two runnables. for (NodeReport nodeReport : nodeReports) { Resource capability = nodeReport.getCapability(); Resource used = nodeReport.getUsed(); Assert.assertNotNull(capability); if (used != null) { Assert.assertTrue(2 * resource.getMemorySize() < capability.getMemory() - used.getMemory()); } else { Assert.assertTrue(2 * resource.getMemorySize() < capability.getMemory()); } } }