Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse#getAllocatedContainers()
The following examples show how to use
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse#getAllocatedContainers() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Hadoop21YarnAMClient.java From twill with Apache License 2.0 | 5 votes |
@Override protected AllocateResult doAllocate(float progress) throws Exception { AllocateResponse allocateResponse = amrmClient.allocate(progress); List<RunnableProcessLauncher> launchers = Lists.newArrayListWithCapacity(allocateResponse.getAllocatedContainers().size()); for (Container container : allocateResponse.getAllocatedContainers()) { launchers.add(new RunnableProcessLauncher(new Hadoop21YarnContainerInfo(container), nmClient)); } List<YarnContainerStatus> completed = ImmutableList.copyOf( Iterables.transform(allocateResponse.getCompletedContainersStatuses(), STATUS_TRANSFORM)); return new AllocateResult(launchers, completed); }
Example 2
Source File: TestFifoScheduler.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void test() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); RMApp app2 = rm.submitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); SchedulerNodeReport report_nm2 = rm.getResourceScheduler().getNodeReport( nm2.getNodeId()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // add request for containers am2.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 0, 1); AllocateResponse alloc2Response = am2.schedule(); // send the request // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0 nm1.nodeHeartbeat(true); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(1000); alloc1Response = am1.schedule(); } while (alloc2Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 2..."); Thread.sleep(1000); alloc2Response = am2.schedule(); } // kick the scheduler, nothing given remaining 2 GB. nm2.nodeHeartbeat(true); List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); List<Container> allocated2 = alloc2Response.getAllocatedContainers(); Assert.assertEquals(1, allocated2.size()); Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemory()); Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); Container c1 = allocated1.get(0); Assert.assertEquals(GB, c1.getResource().getMemory()); ContainerStatus containerStatus = BuilderUtils.newContainerStatus( c1.getId(), ContainerState.COMPLETE, "", 0); nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) { LOG.info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Thread.sleep(1000); } Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemory()); rm.stop(); }
Example 3
Source File: TestFifoScheduler.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testResourceOverCommit() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); // check node report, 2 GB used and 2 GB available Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.nodeHeartbeat(true); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(1000); alloc1Response = am1.schedule(); } List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); Assert.assertEquals(2 * GB, c1.getResource().getMemory()); // update node resource to 2 GB, so resource is over-consumed. Map<NodeId, ResourceOption> nodeResourceMap = new HashMap<NodeId, ResourceOption>(); nodeResourceMap.put(nm1.getNodeId(), ResourceOption.newInstance(Resource.newInstance(2 * GB, 1, 1), -1)); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.newInstance(nodeResourceMap); AdminService as = rm.adminService; as.updateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.newContainerStatus( c1.getId(), ContainerState.COMPLETE, "", 0); nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) { LOG.info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Thread.sleep(100); } Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); // As container return 2 GB back, the available resource becomes 0 again. Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory()); rm.stop(); }
Example 4
Source File: TestApplicationCleanup.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings("resource") @Test(timeout = 60000) public void testAppCleanupWhenRMRestartedBeforeAppFinished() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); // start RM MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.getResourceTrackerService()); nm1.registerNode(); MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.getResourceTrackerService()); nm2.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // alloc another container on nm2 AllocateResponse allocResponse = am0.allocate(Arrays.asList(ResourceRequest.newInstance( Priority.newInstance(1), "*", Resource.newInstance(1024, 0, 0), 1)), null); while (null == allocResponse.getAllocatedContainers() || allocResponse.getAllocatedContainers().isEmpty()) { nm2.nodeHeartbeat(true); allocResponse = am0.allocate(null, null); Thread.sleep(1000); } // start new RM MockRM rm2 = new MockRM(conf, memStore); rm2.start(); // nm1/nm2 register to rm2, and do a heartbeat nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm1.registerNode(Arrays.asList(NMContainerStatus.newInstance( ContainerId.newContainerId(am0.getApplicationAttemptId(), 1), ContainerState.COMPLETE, Resource.newInstance(1024, 1, 1), "", 0, Priority.newInstance(0), 1234)), Arrays.asList(app0.getApplicationId())); nm2.setResourceTrackerService(rm2.getResourceTrackerService()); nm2.registerNode(Arrays.asList(app0.getApplicationId())); // assert app state has been saved. rm2.waitForState(app0.getApplicationId(), RMAppState.FAILED); // wait for application cleanup message received on NM1 waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); // wait for application cleanup message received on NM2 waitForAppCleanupMessageRecved(nm2, app0.getApplicationId()); rm1.stop(); rm2.stop(); }
Example 5
Source File: TestCapacityScheduler.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testResourceOverCommit() throws Exception { Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); // check node report, 2 GB used and 2 GB available Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.nodeHeartbeat(true); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(100); alloc1Response = am1.schedule(); } List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); Assert.assertEquals(2 * GB, c1.getResource().getMemory()); // update node resource to 2 GB, so resource is over-consumed. Map<NodeId, ResourceOption> nodeResourceMap = new HashMap<NodeId, ResourceOption>(); nodeResourceMap.put(nm1.getNodeId(), ResourceOption.newInstance(Resource.newInstance(2 * GB, 1), -1)); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.newInstance(nodeResourceMap); AdminService as = ((MockRM)rm).getAdminService(); as.updateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.newContainerStatus( c1.getId(), ContainerState.COMPLETE, "", 0); nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) { LOG.info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Thread.sleep(100); } Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); // As container return 2 GB back, the available resource becomes 0 again. Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory()); // Verify no NPE is trigger in schedule after resource is updated. am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1); alloc1Response = am1.schedule(); Assert.assertEquals("Shouldn't have enough resource to allocate containers", 0, alloc1Response.getAllocatedContainers().size()); int times = 0; // try 10 times as scheduling is async process. while (alloc1Response.getAllocatedContainers().size() < 1 && times++ < 10) { LOG.info("Waiting for containers to be allocated for app 1... Tried " + times + " times already.."); Thread.sleep(100); } Assert.assertEquals("Shouldn't have enough resource to allocate containers", 0, alloc1Response.getAllocatedContainers().size()); rm.stop(); }
Example 6
Source File: TestNMClient.java From hadoop with Apache License 2.0 | 4 votes |
private Set<Container> allocateContainers( AMRMClientImpl<ContainerRequest> rmClient, int num) throws YarnException, IOException { // setup container request Resource capability = Resource.newInstance(1024, 0); Priority priority = Priority.newInstance(0); String node = nodeReports.get(0).getNodeId().getHost(); String rack = nodeReports.get(0).getRackName(); String[] nodes = new String[] {node}; String[] racks = new String[] {rack}; for (int i = 0; i < num; ++i) { rmClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority)); } int containersRequestedAny = rmClient.remoteRequestsTable.get(priority) .get(ResourceRequest.ANY).get(capability).remoteRequest .getNumContainers(); // RM should allocate container within 2 calls to allocate() int allocatedContainerCount = 0; int iterationsLeft = 2; Set<Container> containers = new TreeSet<Container>(); while (allocatedContainerCount < containersRequestedAny && iterationsLeft > 0) { AllocateResponse allocResponse = rmClient.allocate(0.1f); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); for(Container container : allocResponse.getAllocatedContainers()) { containers.add(container); } if (!allocResponse.getNMTokens().isEmpty()) { for (NMToken token : allocResponse.getNMTokens()) { rmClient.getNMTokenCache().setToken(token.getNodeId().toString(), token.getToken()); } } if(allocatedContainerCount < containersRequestedAny) { // sleep to let NM's heartbeat to RM and trigger allocations sleep(1000); } --iterationsLeft; } return containers; }
Example 7
Source File: TestFifoScheduler.java From big-c with Apache License 2.0 | 4 votes |
@Test public void test() throws Exception { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB); MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); RMApp app2 = rm.submitApp(2048); // kick the scheduling, 2GB given to AM, remaining 2 GB on nm2 nm2.nodeHeartbeat(true); RMAppAttempt attempt2 = app2.getCurrentAppAttempt(); MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId()); am2.registerAppAttempt(); SchedulerNodeReport report_nm2 = rm.getResourceScheduler().getNodeReport( nm2.getNodeId()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // add request for containers am2.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 0, 1); AllocateResponse alloc2Response = am2.schedule(); // send the request // kick the scheduler, 1 GB and 3 GB given to AM1 and AM2, remaining 0 nm1.nodeHeartbeat(true); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(1000); alloc1Response = am1.schedule(); } while (alloc2Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 2..."); Thread.sleep(1000); alloc2Response = am2.schedule(); } // kick the scheduler, nothing given remaining 2 GB. nm2.nodeHeartbeat(true); List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(1 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); List<Container> allocated2 = alloc2Response.getAllocatedContainers(); Assert.assertEquals(1, allocated2.size()); Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId()); Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemory()); Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory()); Container c1 = allocated1.get(0); Assert.assertEquals(GB, c1.getResource().getMemory()); ContainerStatus containerStatus = BuilderUtils.newContainerStatus( c1.getId(), ContainerState.COMPLETE, "", 0); nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) { LOG.info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Thread.sleep(1000); } Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemory()); rm.stop(); }
Example 8
Source File: TestFifoScheduler.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testResourceOverCommit() throws Exception { MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); // check node report, 2 GB used and 2 GB available Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.nodeHeartbeat(true); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(1000); alloc1Response = am1.schedule(); } List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); Assert.assertEquals(2 * GB, c1.getResource().getMemory()); // update node resource to 2 GB, so resource is over-consumed. Map<NodeId, ResourceOption> nodeResourceMap = new HashMap<NodeId, ResourceOption>(); nodeResourceMap.put(nm1.getNodeId(), ResourceOption.newInstance(Resource.newInstance(2 * GB, 1), -1)); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.newInstance(nodeResourceMap); AdminService as = rm.adminService; as.updateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.newContainerStatus( c1.getId(), ContainerState.COMPLETE, "", 0); nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) { LOG.info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Thread.sleep(100); } Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); // As container return 2 GB back, the available resource becomes 0 again. Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory()); rm.stop(); }
Example 9
Source File: TestApplicationCleanup.java From big-c with Apache License 2.0 | 4 votes |
@SuppressWarnings("resource") @Test(timeout = 60000) public void testAppCleanupWhenRMRestartedBeforeAppFinished() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); // start RM MockRM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.getResourceTrackerService()); nm1.registerNode(); MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.getResourceTrackerService()); nm2.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // alloc another container on nm2 AllocateResponse allocResponse = am0.allocate(Arrays.asList(ResourceRequest.newInstance( Priority.newInstance(1), "*", Resource.newInstance(1024, 0), 1)), null); while (null == allocResponse.getAllocatedContainers() || allocResponse.getAllocatedContainers().isEmpty()) { nm2.nodeHeartbeat(true); allocResponse = am0.allocate(null, null); Thread.sleep(1000); } // start new RM MockRM rm2 = new MockRM(conf, memStore); rm2.start(); // nm1/nm2 register to rm2, and do a heartbeat nm1.setResourceTrackerService(rm2.getResourceTrackerService()); nm1.registerNode(Arrays.asList(NMContainerStatus.newInstance( ContainerId.newContainerId(am0.getApplicationAttemptId(), 1), ContainerState.COMPLETE, Resource.newInstance(1024, 1), "", 0, Priority.newInstance(0), 1234)), Arrays.asList(app0.getApplicationId())); nm2.setResourceTrackerService(rm2.getResourceTrackerService()); nm2.registerNode(Arrays.asList(app0.getApplicationId())); // assert app state has been saved. rm2.waitForState(app0.getApplicationId(), RMAppState.FAILED); // wait for application cleanup message received on NM1 waitForAppCleanupMessageRecved(nm1, app0.getApplicationId()); // wait for application cleanup message received on NM2 waitForAppCleanupMessageRecved(nm2, app0.getApplicationId()); rm1.stop(); rm2.stop(); }
Example 10
Source File: TestCapacityScheduler.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testResourceOverCommit() throws Exception { Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); MockRM rm = new MockRM(conf); rm.start(); MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB); RMApp app1 = rm.submitApp(2048); // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1 nm1.nodeHeartbeat(true); RMAppAttempt attempt1 = app1.getCurrentAppAttempt(); MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId()); am1.registerAppAttempt(); SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport( nm1.getNodeId()); // check node report, 2 GB used and 2 GB available Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory()); // add request for containers am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1); AllocateResponse alloc1Response = am1.schedule(); // send the request // kick the scheduler, 2 GB given to AM1, resource remaining 0 nm1.nodeHeartbeat(true); while (alloc1Response.getAllocatedContainers().size() < 1) { LOG.info("Waiting for containers to be created for app 1..."); Thread.sleep(100); alloc1Response = am1.schedule(); } List<Container> allocated1 = alloc1Response.getAllocatedContainers(); Assert.assertEquals(1, allocated1.size()); Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory()); Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); // check node report, 4 GB used and 0 GB available Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); // check container is assigned with 2 GB. Container c1 = allocated1.get(0); Assert.assertEquals(2 * GB, c1.getResource().getMemory()); // update node resource to 2 GB, so resource is over-consumed. Map<NodeId, ResourceOption> nodeResourceMap = new HashMap<NodeId, ResourceOption>(); nodeResourceMap.put(nm1.getNodeId(), ResourceOption.newInstance(Resource.newInstance(2 * GB, 1), -1)); UpdateNodeResourceRequest request = UpdateNodeResourceRequest.newInstance(nodeResourceMap); AdminService as = ((MockRM)rm).getAdminService(); as.updateNodeResource(request); // Now, the used resource is still 4 GB, and available resource is minus value. report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory()); Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory()); // Check container can complete successfully in case of resource over-commitment. ContainerStatus containerStatus = BuilderUtils.newContainerStatus( c1.getId(), ContainerState.COMPLETE, "", 0); nm1.containerStatus(containerStatus); int waitCount = 0; while (attempt1.getJustFinishedContainers().size() < 1 && waitCount++ != 20) { LOG.info("Waiting for containers to be finished for app 1... Tried " + waitCount + " times already.."); Thread.sleep(100); } Assert.assertEquals(1, attempt1.getJustFinishedContainers().size()); Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size()); report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId()); Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory()); // As container return 2 GB back, the available resource becomes 0 again. Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory()); // Verify no NPE is trigger in schedule after resource is updated. am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1); alloc1Response = am1.schedule(); Assert.assertEquals("Shouldn't have enough resource to allocate containers", 0, alloc1Response.getAllocatedContainers().size()); int times = 0; // try 10 times as scheduling is async process. while (alloc1Response.getAllocatedContainers().size() < 1 && times++ < 10) { LOG.info("Waiting for containers to be allocated for app 1... Tried " + times + " times already.."); Thread.sleep(100); } Assert.assertEquals("Shouldn't have enough resource to allocate containers", 0, alloc1Response.getAllocatedContainers().size()); rm.stop(); }
Example 11
Source File: TestNMClient.java From big-c with Apache License 2.0 | 4 votes |
private Set<Container> allocateContainers( AMRMClientImpl<ContainerRequest> rmClient, int num) throws YarnException, IOException { // setup container request Resource capability = Resource.newInstance(1024, 0); Priority priority = Priority.newInstance(0); String node = nodeReports.get(0).getNodeId().getHost(); String rack = nodeReports.get(0).getRackName(); String[] nodes = new String[] {node}; String[] racks = new String[] {rack}; for (int i = 0; i < num; ++i) { rmClient.addContainerRequest(new ContainerRequest(capability, nodes, racks, priority)); } int containersRequestedAny = rmClient.remoteRequestsTable.get(priority) .get(ResourceRequest.ANY).get(capability).remoteRequest .getNumContainers(); // RM should allocate container within 2 calls to allocate() int allocatedContainerCount = 0; int iterationsLeft = 2; Set<Container> containers = new TreeSet<Container>(); while (allocatedContainerCount < containersRequestedAny && iterationsLeft > 0) { AllocateResponse allocResponse = rmClient.allocate(0.1f); allocatedContainerCount += allocResponse.getAllocatedContainers().size(); for(Container container : allocResponse.getAllocatedContainers()) { containers.add(container); } if (!allocResponse.getNMTokens().isEmpty()) { for (NMToken token : allocResponse.getNMTokens()) { rmClient.getNMTokenCache().setToken(token.getNodeId().toString(), token.getToken()); } } if(allocatedContainerCount < containersRequestedAny) { // sleep to let NM's heartbeat to RM and trigger allocations sleep(1000); } --iterationsLeft; } return containers; }
Example 12
Source File: YarnManager.java From Scribengin with GNU Affero General Public License v3.0 | 4 votes |
public List<Container> getAllocatedContainers() throws YarnException, IOException { AllocateResponse response = amrmClient.allocate(0); return response.getAllocatedContainers() ; }