org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestMoveApplication.java From hadoop with Apache License 2.0 | 6 votes |
@Test (timeout = 10000) public void testMoveSuccessful() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); RMApp app = rm1.submitApp(1024); ClientRMService clientRMService = rm1.getClientRMService(); // FIFO scheduler does not support moves clientRMService .moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest .newInstance(app.getApplicationId(), "newqueue")); RMApp rmApp = rm1.getRMContext().getRMApps().get(app.getApplicationId()); assertEquals("newqueue", rmApp.getQueue()); rm1.stop(); }
Example #2
Source File: RMStateStoreTestBase.java From hadoop with Apache License 2.0 | 6 votes |
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); store.storeNewApplication(mockApp); return mockApp; }
Example #3
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 6 votes |
private RMApp getRMAppForAppId(String appId) { if (appId == null || appId.isEmpty()) { throw new NotFoundException("appId, " + appId + ", is empty or null"); } ApplicationId id; try { id = ConverterUtils.toApplicationId(recordFactory, appId); } catch (NumberFormatException e) { throw new NotFoundException("appId is invalid"); } if (id == null) { throw new NotFoundException("appId is invalid"); } RMApp app = rm.getRMContext().getRMApps().get(id); if (app == null) { throw new NotFoundException("app with id: " + appId + " not found"); } return app; }
Example #4
Source File: TestRMWebServicesApps.java From hadoop with Apache License 2.0 | 6 votes |
public void testAppsHelper(String path, RMApp app, String media) throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path(path).accept(media).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject apps = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); verifyAppInfo(array.getJSONObject(0), app); }
Example #5
Source File: AbstractYarnScheduler.java From big-c with Apache License 2.0 | 6 votes |
public synchronized List<Container> getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); SchedulerApplication<T> app = applications.get(appId); List<Container> containerList = new ArrayList<Container>(); RMApp appImpl = this.rmContext.getRMApps().get(appId); if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) { return containerList; } Collection<RMContainer> liveContainers = app.getCurrentAppAttempt().getLiveContainers(); ContainerId amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt() .getMasterContainer().getId(); for (RMContainer rmContainer : liveContainers) { if (!rmContainer.getContainerId().equals(amContainerId)) { containerList.add(rmContainer.getContainer()); } } return containerList; }
Example #6
Source File: TestRMWebServicesApps.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testAppsQueryFinalStatus() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString()) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); System.out.println(json.toString()); JSONObject apps = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 1, array.length()); verifyAppInfo(array.getJSONObject(0), app1); rm.stop(); }
Example #7
Source File: TestFairScheduler.java From big-c with Apache License 2.0 | 6 votes |
@Test public void testAssignToQueue() throws Exception { conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true"); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW); RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW); FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix"); FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix"); // assert FSLeafQueue's name is the correct name is the one set in the RMApp assertEquals(rmApp1.getQueue(), queue1.getName()); assertEquals("root.asterix", rmApp1.getQueue()); assertEquals(rmApp2.getQueue(), queue2.getName()); assertEquals("root.notdefault", rmApp2.getQueue()); }
Example #8
Source File: ResourceManager.java From hadoop with Apache License 2.0 | 6 votes |
@Override public void handle(RMAppAttemptEvent event) { ApplicationAttemptId appAttemptID = event.getApplicationAttemptId(); ApplicationId appAttemptId = appAttemptID.getApplicationId(); RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId); if (rmApp != null) { RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID); if (rmAppAttempt != null) { try { rmAppAttempt.handle(event); } catch (Throwable t) { LOG.error("Error in handling event type " + event.getType() + " for applicationAttempt " + appAttemptId, t); } } } }
Example #9
Source File: SystemMetricsPublisher.java From big-c with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") public void appAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) { if (publishSystemMetrics) { dispatcher.getEventHandler().handle( new AppAttemptFinishedEvent( appAttempt.getAppAttemptId(), appAttempt.getTrackingUrl(), appAttempt.getOriginalTrackingUrl(), appAttempt.getDiagnostics(), // app will get the final status from app attempt, or create one // based on app state if it doesn't exist app.getFinalApplicationStatus(), RMServerUtils.createApplicationAttemptState(appAttemtpState), finishedTime)); } }
Example #10
Source File: TestAppManager.java From hadoop with Apache License 2.0 | 6 votes |
@Test (timeout = 30000) public void testRMAppSubmitDuplicateApplicationId() throws Exception { ApplicationId appId = MockApps.newAppID(0); asContext.setApplicationId(appId); RMApp appOrig = rmContext.getRMApps().get(appId); Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName()); // our testApp1 should be rejected and original app with same id should be left in place try { appMonitor.submitApplication(asContext, "test"); Assert.fail("Exception is expected when applicationId is duplicate."); } catch (YarnException e) { Assert.assertTrue("The thrown exception is not the expectd one.", e.getMessage().contains("Cannot add a duplicate!")); } // make sure original app didn't get removed RMApp app = rmContext.getRMApps().get(appId); Assert.assertNotNull("app is null", app); Assert.assertEquals("app id doesn't match", appId, app.getApplicationId()); Assert.assertEquals("app state doesn't match", RMAppState.FINISHED, app.getState()); }
Example #11
Source File: AbstractYarnScheduler.java From hadoop with Apache License 2.0 | 6 votes |
public synchronized List<Container> getTransferredContainers( ApplicationAttemptId currentAttempt) { ApplicationId appId = currentAttempt.getApplicationId(); SchedulerApplication<T> app = applications.get(appId); List<Container> containerList = new ArrayList<Container>(); RMApp appImpl = this.rmContext.getRMApps().get(appId); if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) { return containerList; } Collection<RMContainer> liveContainers = app.getCurrentAppAttempt().getLiveContainers(); ContainerId amContainerId = rmContext.getRMApps().get(appId).getCurrentAppAttempt() .getMasterContainer().getId(); for (RMContainer rmContainer : liveContainers) { if (!rmContainer.getContainerId().equals(amContainerId)) { containerList.add(rmContainer.getContainer()); } } return containerList; }
Example #12
Source File: RMStateStoreTestBase.java From big-c with Apache License 2.0 | 6 votes |
protected RMApp storeApp(RMStateStore store, ApplicationId appId, long submitTime, long startTime) throws Exception { ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.setApplicationId(appId); RMApp mockApp = mock(RMApp.class); when(mockApp.getApplicationId()).thenReturn(appId); when(mockApp.getSubmitTime()).thenReturn(submitTime); when(mockApp.getStartTime()).thenReturn(startTime); when(mockApp.getApplicationSubmissionContext()).thenReturn(context); when(mockApp.getUser()).thenReturn("test"); store.storeNewApplication(mockApp); return mockApp; }
Example #13
Source File: TestKillApplicationWithRMHA.java From hadoop with Apache License 2.0 | 6 votes |
@Test (timeout = 20000) public void testKillAppWhenFailoverHappensAtRunningState() throws Exception { startRMs(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = launchAM(app0, rm1, nm1); // failover and kill application // The application is at RUNNING State when failOver happens. // Since RMStateStore has already saved ApplicationState, the active RM // will load the ApplicationState. After that, the application will be at // ACCEPTED State. Because the application is not at Final State, // KillApplicationResponse.getIsKillCompleted is expected to return false. failOverAndKillApp(app0.getApplicationId(), am0.getApplicationAttemptId(), RMAppState.RUNNING, RMAppAttemptState.RUNNING, RMAppState.ACCEPTED); }
Example #14
Source File: TestSystemMetricsPublisher.java From hadoop with Apache License 2.0 | 6 votes |
private static RMApp createRMApp(ApplicationId appId) { RMApp app = mock(RMApp.class); when(app.getApplicationId()).thenReturn(appId); when(app.getName()).thenReturn("test app"); when(app.getApplicationType()).thenReturn("test app type"); when(app.getUser()).thenReturn("test user"); when(app.getQueue()).thenReturn("test queue"); when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L); when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L); when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L); when(app.getDiagnostics()).thenReturn( new StringBuilder("test diagnostics info")); RMAppAttempt appAttempt = mock(RMAppAttempt.class); when(appAttempt.getAppAttemptId()).thenReturn( ApplicationAttemptId.newInstance(appId, 1)); when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()).thenReturn( FinalApplicationStatus.UNDEFINED); when(app.getRMAppMetrics()).thenReturn( new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE)); return app; }
Example #15
Source File: FairScheduler.java From hadoop with Apache License 2.0 | 5 votes |
/** * Helper method that attempts to assign the app to a queue. The method is * responsible to call the appropriate event-handler if the app is rejected. */ @VisibleForTesting FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) { FSLeafQueue queue = null; String appRejectMsg = null; try { QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); queueName = placementPolicy.assignAppToQueue(queueName, user); if (queueName == null) { appRejectMsg = "Application rejected by queue placement policy"; } else { queue = queueMgr.getLeafQueue(queueName, true); if (queue == null) { appRejectMsg = queueName + " is not a leaf queue"; } } } catch (IOException ioe) { appRejectMsg = "Error assigning app to queue " + queueName; } if (appRejectMsg != null && rmApp != null) { LOG.error(appRejectMsg); rmContext.getDispatcher().getEventHandler().handle( new RMAppRejectedEvent(rmApp.getApplicationId(), appRejectMsg)); return null; } if (rmApp != null) { rmApp.setQueue(queue.getName()); } else { LOG.error("Couldn't find RM app to set queue name on"); } return queue; }
Example #16
Source File: TestContainerAllocation.java From big-c with Apache License 2.0 | 5 votes |
@Test(timeout = 30000) public void testAMContainerAllocationWhenDNSUnavailable() throws Exception { MockRM rm1 = new MockRM(conf) { @Override protected RMSecretManagerService createRMSecretManagerService() { return new TestRMSecretManagerService(conf, rmContext); } }; rm1.start(); MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000); SecurityUtilTestHelper.setTokenServiceUseIp(true); RMApp app1 = rm1.submitApp(200); RMAppAttempt attempt = app1.getCurrentAppAttempt(); nm1.nodeHeartbeat(true); // fetching am container will fail, keep retrying 5 times. while (numRetries <= 5) { nm1.nodeHeartbeat(true); Thread.sleep(1000); Assert.assertEquals(RMAppAttemptState.SCHEDULED, attempt.getAppAttemptState()); System.out.println("Waiting for am container to be allocated."); } SecurityUtilTestHelper.setTokenServiceUseIp(false); rm1.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.ALLOCATED); MockRM.launchAndRegisterAM(app1, rm1, nm1); }
Example #17
Source File: TestRMRestart.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout = 60000) public void testAppSubmissionWithOldDelegationTokenAfterRMRestart() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2); conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032"); UserGroupInformation.setConfiguration(conf); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); MockRM rm1 = new TestSecurityMockRM(conf, memStore); rm1.start(); GetDelegationTokenRequest request1 = GetDelegationTokenRequest.newInstance("renewer1"); UserGroupInformation.getCurrentUser().setAuthenticationMethod( AuthMethod.KERBEROS); GetDelegationTokenResponse response1 = rm1.getClientRMService().getDelegationToken(request1); Token<RMDelegationTokenIdentifier> token1 = ConverterUtils.convertFromYarn(response1.getRMDelegationToken(), rmAddr); // start new RM MockRM rm2 = new TestSecurityMockRM(conf, memStore); rm2.start(); // submit an app with the old delegation token got from previous RM. Credentials ts = new Credentials(); ts.addToken(token1.getService(), token1); RMApp app = rm2.submitApp(200, "name", "user", new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts); rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED); }
Example #18
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 5 votes |
protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) { // Check for the authorization. UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true); if (callerUGI != null && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI, ApplicationAccessType.VIEW_APP, app.getUser(), app.getApplicationId()) || this.rm.getQueueACLsManager().checkAccess(callerUGI, QueueACL.ADMINISTER_QUEUE, app.getQueue()))) { return false; } return true; }
Example #19
Source File: ResourceTrackerService.java From hadoop with Apache License 2.0 | 5 votes |
/** * Helper method to handle received ContainerStatus. If this corresponds to * the completion of a master-container of a managed AM, * we call the handler for RMAppAttemptContainerFinishedEvent. */ @SuppressWarnings("unchecked") @VisibleForTesting void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) { ApplicationAttemptId appAttemptId = containerStatus.getContainerId().getApplicationAttemptId(); RMApp rmApp = rmContext.getRMApps().get(appAttemptId.getApplicationId()); if (rmApp == null) { LOG.error("Received finished container : " + containerStatus.getContainerId() + " for unknown application " + appAttemptId.getApplicationId() + " Skipping."); return; } if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) { if (LOG.isDebugEnabled()) { LOG.debug("Ignoring container completion status for unmanaged AM " + rmApp.getApplicationId()); } return; } RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId); Container masterContainer = rmAppAttempt.getMasterContainer(); if (masterContainer.getId().equals(containerStatus.getContainerId()) && containerStatus.getContainerState() == ContainerState.COMPLETE) { ContainerStatus status = ContainerStatus.newInstance(containerStatus.getContainerId(), containerStatus.getContainerState(), containerStatus.getDiagnostics(), containerStatus.getContainerExitStatus()); // sending master container finished event. RMAppAttemptContainerFinishedEvent evt = new RMAppAttemptContainerFinishedEvent(appAttemptId, status, nodeId); rmContext.getDispatcher().getEventHandler().handle(evt); } }
Example #20
Source File: TestRMRestart.java From big-c with Apache License 2.0 | 5 votes |
private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm, MockAM am) throws Exception { final FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance( FinalApplicationStatus.SUCCEEDED, "", ""); finishApplicationMaster(rmApp, rm, nm, am, req); }
Example #21
Source File: FiCaSchedulerApp.java From hadoop with Apache License 2.0 | 5 votes |
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, String user, Queue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { super(applicationAttemptId, user, queue, activeUsersManager, rmContext); RMApp rmApp = rmContext.getRMApps().get(getApplicationId()); Resource amResource; String partition; if (rmApp == null || rmApp.getAMResourceRequest() == null) { // the rmApp may be undefined (the resource manager checks for this too) // and unmanaged applications do not provide an amResource request // in these cases, provide a default using the scheduler amResource = rmContext.getScheduler().getMinimumResourceCapability(); partition = CommonNodeLabelsManager.NO_LABEL; } else { amResource = rmApp.getAMResourceRequest().getCapability(); partition = (rmApp.getAMResourceRequest().getNodeLabelExpression() == null) ? CommonNodeLabelsManager.NO_LABEL : rmApp.getAMResourceRequest().getNodeLabelExpression(); } setAppAMNodePartitionName(partition); setAMResource(partition, amResource); }
Example #22
Source File: RMAppManager.java From hadoop with Apache License 2.0 | 5 votes |
/** * create a summary of the application's runtime. * * @param app {@link RMApp} whose summary is to be created, cannot * be <code>null</code>. */ public static SummaryBuilder createAppSummary(RMApp app) { String trackingUrl = "N/A"; String host = "N/A"; RMAppAttempt attempt = app.getCurrentAppAttempt(); if (attempt != null) { trackingUrl = attempt.getTrackingUrl(); host = attempt.getHost(); } RMAppMetrics metrics = app.getRMAppMetrics(); SummaryBuilder summary = new SummaryBuilder() .add("appId", app.getApplicationId()) .add("name", app.getName()) .add("user", app.getUser()) .add("queue", app.getQueue()) .add("state", app.getState()) .add("trackingUrl", trackingUrl) .add("appMasterHost", host) .add("startTime", app.getStartTime()) .add("finishTime", app.getFinishTime()) .add("finalStatus", app.getFinalApplicationStatus()) .add("memorySeconds", metrics.getMemorySeconds()) .add("vcoreSeconds", metrics.getVcoreSeconds()) .add("gcoreSeconds", metrics.getGcoreSeconds()) .add("preemptedAMContainers", metrics.getNumAMContainersPreempted()) .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted()) .add("preemptedResources", metrics.getResourcePreempted()) .add("applicationType", app.getApplicationType()); return summary; }
Example #23
Source File: TestRMWebServicesApps.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testAppAttemtpsDefault() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, ""); rm.stop(); }
Example #24
Source File: TestRMWebServicesApps.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testAppsQueryFinishEnd() throws JSONException, Exception { rm.start(); MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048); RMApp app1 = rm.submitApp(CONTAINER_MB); amNodeManager.nodeHeartbeat(true); // finish App MockAM am = rm .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId()); am.registerAppAttempt(); am.unregisterAppAttempt(); amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(), 1, ContainerState.COMPLETE); rm.submitApp(CONTAINER_MB); rm.submitApp(CONTAINER_MB); long end = System.currentTimeMillis(); WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").queryParam("finishedTimeEnd", String.valueOf(end)) .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); JSONObject apps = json.getJSONObject("apps"); assertEquals("incorrect number of elements", 1, apps.length()); JSONArray array = apps.getJSONArray("app"); assertEquals("incorrect number of elements", 3, array.length()); rm.stop(); }
Example #25
Source File: MockRM.java From big-c with Apache License 2.0 | 5 votes |
public RMApp submitApp(int masterMemory, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, ApplicationId applicationId) throws Exception { return submitApp(masterMemory, name, user, acls, unmanaged, queue, maxAppAttempts, ts, appType, waitForAccepted, keepContainers, isAppIdProvided, applicationId, 0, null, true); }
Example #26
Source File: TestRMApplicationHistoryWriter.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testWriteApplication() throws Exception { RMApp app = createRMApp(ApplicationId.newInstance(0, 1)); writer.applicationStarted(app); ApplicationHistoryData appHD = null; for (int i = 0; i < MAX_RETRIES; ++i) { appHD = store.getApplication(ApplicationId.newInstance(0, 1)); if (appHD != null) { break; } else { Thread.sleep(100); } } Assert.assertNotNull(appHD); Assert.assertEquals("test app", appHD.getApplicationName()); Assert.assertEquals("test app type", appHD.getApplicationType()); Assert.assertEquals("test user", appHD.getUser()); Assert.assertEquals("test queue", appHD.getQueue()); Assert.assertEquals(0L, appHD.getSubmitTime()); Assert.assertEquals(1L, appHD.getStartTime()); writer.applicationFinished(app, RMAppState.FINISHED); for (int i = 0; i < MAX_RETRIES; ++i) { appHD = store.getApplication(ApplicationId.newInstance(0, 1)); if (appHD.getYarnApplicationState() != null) { break; } else { Thread.sleep(100); } } Assert.assertEquals(2L, appHD.getFinishTime()); Assert.assertEquals("test diagnostics info", appHD.getDiagnosticsInfo()); Assert.assertEquals(FinalApplicationStatus.UNDEFINED, appHD.getFinalApplicationStatus()); Assert.assertEquals(YarnApplicationState.FINISHED, appHD.getYarnApplicationState()); }
Example #27
Source File: TestRMWebServicesApps.java From big-c with Apache License 2.0 | 5 votes |
public void testSingleAppsHelper(String path, RMApp app, String media) throws JSONException, Exception { WebResource r = resource(); ClientResponse response = r.path("ws").path("v1").path("cluster") .path("apps").path(path).accept(media).get(ClientResponse.class); assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); JSONObject json = response.getEntity(JSONObject.class); assertEquals("incorrect number of elements", 1, json.length()); verifyAppInfo(json.getJSONObject("app"), app); }
Example #28
Source File: TestRMRestart.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 60000) public void testClientRetryOnKillingApplication() throws Exception { MemoryRMStateStore memStore = new TestMemoryRMStateStore(); memStore.init(conf); // start RM MockRM rm1 = createMockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); RMApp app1 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType"); MockAM am1 = launchAM(app1, rm1, nm1); KillApplicationResponse response; int count = 0; while (true) { response = rm1.killApp(app1.getApplicationId()); if (response.getIsKillCompleted()) { break; } Thread.sleep(100); count++; } // we expect at least 2 calls for killApp as the first killApp always return // false. Assert.assertTrue(count >= 1); rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED); rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED); Assert.assertEquals(1, ((TestMemoryRMStateStore) memStore).updateAttempt); Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateApp); }
Example #29
Source File: TestWorkPreservingRMRestart.java From hadoop with Apache License 2.0 | 5 votes |
/** * Testing to confirm that retried finishApplicationMaster() doesn't throw * InvalidApplicationMasterRequest before and after RM restart. */ @Test (timeout = 20000) public void testRetriedFinishApplicationMasterRequest() throws Exception { conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); // start RM rm1 = new MockRM(conf, memStore); rm1.start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService()); nm1.registerNode(); // create app and launch the AM RMApp app0 = rm1.submitApp(200); MockAM am0 = MockRM.launchAM(app0, rm1, nm1); am0.registerAppAttempt(); // Emulating following a scenario: // RM1 saves the app in RMStateStore and then crashes, // FinishApplicationMasterResponse#isRegistered still return false, // so AM still retry the 2nd RM MockRM.finishAMAndVerifyAppState(app0, rm1, nm1, am0); // start new RM rm2 = new MockRM(conf, memStore); rm2.start(); am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext()); am0.unregisterAppAttempt(false); }
Example #30
Source File: TestContainerAllocation.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testContainerTokenGeneratedOnPullRequest() throws Exception { MockRM rm1 = new MockRM(conf); rm1.start(); MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000); RMApp app1 = rm1.submitApp(200); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); // request a container. am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>()); ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED); RMContainer container = rm1.getResourceScheduler().getRMContainer(containerId2); // no container token is generated. Assert.assertEquals(containerId2, container.getContainerId()); Assert.assertNull(container.getContainer().getContainerToken()); // acquire the container. List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers(); Assert.assertEquals(containerId2, containers.get(0).getId()); // container token is generated. Assert.assertNotNull(containers.get(0).getContainerToken()); rm1.stop(); }