Java Code Examples for org.apache.hadoop.yarn.event.DrainDispatcher#register()
The following examples show how to use
org.apache.hadoop.yarn.event.DrainDispatcher#register() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestRMContainerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testReleaseWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); ConcurrentMap<ApplicationId, RMApp> rmApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>()); RMApp rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null); Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any()); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getRMApps()).thenReturn(rmApps); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.RELEASED)); drainDispatcher.await(); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); assertEquals(SchedulerUtils.RELEASED_CONTAINER, rmContainer.getDiagnosticsInfo()); assertEquals(ContainerExitStatus.ABORTED, rmContainer.getContainerExitStatus()); assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState()); verify(writer).containerFinished(any(RMContainer.class)); verify(publisher).containerFinished(any(RMContainer.class), anyLong()); ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor .forClass(RMAppAttemptContainerFinishedEvent.class); verify(appAttemptEventHandler).handle(captor.capture()); RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue(); assertEquals(appAttemptId, cfEvent.getApplicationAttemptId()); assertEquals(containerStatus, cfEvent.getContainerStatus()); assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType()); // In RELEASED state. A FINIHSED event may come in. rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); }
Example 2
Source File: TestResourceLocalizationService.java From big-c with Apache License 2.0 | 4 votes |
@Test( timeout = 10000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizerRunnerException() throws Exception { DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler); dirsHandlerSpy.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getAppId()).thenReturn(appId); spyService.handle(new ApplicationLocalizationEvent( LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0"); final LocalResource resource1 = getPrivateMockedResource(r); System.out.println("Here 4"); final LocalResourceRequest req1 = new LocalResourceRequest(resource1); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors(); constructors[0].setAccessible(true); FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error")); Mockito .doThrow(fsError) .when(dirsHandlerSpy) .getLocalPathForWrite(isA(String.class)); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); Thread.sleep(1000); dispatcher.await(); // Verify if ContainerResourceFailedEvent is invoked on FSError verify(containerBus).handle(isA(ContainerResourceFailedEvent.class)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
Example 3
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResourceWithDirCacheMgr() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDirRoot = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, true, conf, stateStore); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDirRoot + "/4/2", Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); LocalCacheDirectoryManager dirMgrRoot = tracker.getDirectoryManager(localDirRoot); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr2)); final long localizedId2 = localizedId1 + 1; Path hierarchicalPath2 = new Path(localDirRoot + "/4/2", Long.toString(localizedId2)); Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr2)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr3)); final long localizedId3 = 128; Path hierarchicalPath3 = new Path(localDirRoot + "/4/3", Long.toString(localizedId3)); Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr3)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr4)); final long localizedId4 = 256; Path hierarchicalPath4 = new Path(localDirRoot + "/4", Long.toString(localizedId4)); Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr4)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 4
Source File: TestRMContainerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testReleaseWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); ConcurrentMap<ApplicationId, RMApp> rmApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>()); RMApp rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null); Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any()); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getRMApps()).thenReturn(rmApps); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify RELEASED and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.RELEASED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.RELEASED)); drainDispatcher.await(); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); assertEquals(SchedulerUtils.RELEASED_CONTAINER, rmContainer.getDiagnosticsInfo()); assertEquals(ContainerExitStatus.ABORTED, rmContainer.getContainerExitStatus()); assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState()); verify(writer).containerFinished(any(RMContainer.class)); verify(publisher).containerFinished(any(RMContainer.class), anyLong()); ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor .forClass(RMAppAttemptContainerFinishedEvent.class); verify(appAttemptEventHandler).handle(captor.capture()); RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue(); assertEquals(appAttemptId, cfEvent.getApplicationAttemptId()); assertEquals(containerStatus, cfEvent.getContainerStatus()); assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType()); // In RELEASED state. A FINIHSED event may come in. rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils .createAbnormalContainerStatus(containerId, "FinishedContainer"), RMContainerEventType.FINISHED)); assertEquals(RMContainerState.RELEASED, rmContainer.getState()); }
Example 5
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreFailedLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent( lr1, new Exception("Test").toString()); tracker.handle(rfe1); dispatcher.await(); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 6
Source File: TestRMContainerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testExpireWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.EXPIRE)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); verify(writer, never()).containerFinished(any(RMContainer.class)); verify(publisher, never()).containerFinished(any(RMContainer.class), anyLong()); }
Example 7
Source File: TestApplication.java From big-c with Apache License 2.0 | 4 votes |
WrappedApplication(int id, long timestamp, String user, int numContainers) { Configuration conf = new Configuration(); dispatcher = new DrainDispatcher(); containerTokenIdentifierMap = new HashMap<ContainerId, ContainerTokenIdentifier>(); dispatcher.init(conf); localizerBus = mock(EventHandler.class); launcherBus = mock(EventHandler.class); monitorBus = mock(EventHandler.class); auxBus = mock(EventHandler.class); containerBus = mock(EventHandler.class); logAggregationBus = mock(EventHandler.class); dispatcher.register(LocalizationEventType.class, localizerBus); dispatcher.register(ContainersLauncherEventType.class, launcherBus); dispatcher.register(ContainersMonitorEventType.class, monitorBus); dispatcher.register(AuxServicesEventType.class, auxBus); dispatcher.register(ContainerEventType.class, containerBus); dispatcher.register(LogHandlerEventType.class, logAggregationBus); nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class); context = mock(Context.class); when(context.getContainerTokenSecretManager()).thenReturn( new NMContainerTokenSecretManager(conf)); when(context.getApplicationACLsManager()).thenReturn( new ApplicationACLsManager(conf)); when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr); // Setting master key MasterKey masterKey = new MasterKeyPBImpl(); masterKey.setKeyId(123); masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123) .byteValue()) })); context.getContainerTokenSecretManager().setMasterKey(masterKey); this.user = user; this.appId = BuilderUtils.newApplicationId(timestamp, id); app = new ApplicationImpl(dispatcher, this.user, appId, null, context); containers = new ArrayList<Container>(); for (int i = 0; i < numContainers; i++) { Container container = createMockedContainer(this.appId, i); containers.add(container); long currentTime = System.currentTimeMillis(); ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(container.getContainerId(), "", "", null, currentTime + 2000, masterKey.getKeyId(), currentTime, Priority.newInstance(0), 0); containerTokenIdentifierMap .put(identifier.getContainerID(), identifier); context.getContainerTokenSecretManager().startContainerSuccessful( identifier); Assert.assertFalse(context.getContainerTokenSecretManager() .isValidStartContainerRequest(identifier)); } dispatcher.start(); }
Example 8
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void test() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2); LocalizerContext lc2 = new LocalizerContext(user, cId2, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); LocalizedResource lr2 = createLocalizedResource(req2, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); localrsrc.put(req2, lr2); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService()); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2); ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2); ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1); // Localize R1 for C1 tracker.handle(req11Event); // Localize R1 for C2 tracker.handle(req12Event); // Localize R2 for C1 tracker.handle(req21Event); dispatcher.await(); verify(localizerEventHandler, times(3)).handle( any(LocalizerResourceRequestEvent.class)); // Verify refCount for R1 is 2 Assert.assertEquals(2, lr1.getRefCount()); // Verify refCount for R2 is 1 Assert.assertEquals(1, lr2.getRefCount()); // Release R2 for C1 tracker.handle(rel21Event); dispatcher.await(); verifyTrackedResourceCount(tracker, 2); // Verify resource with non zero ref count is not removed. Assert.assertEquals(2, lr1.getRefCount()); Assert.assertFalse(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 2); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); // Release resource1 tracker.handle(rel11Event); tracker.handle(rel12Event); Assert.assertEquals(0, lr1.getRefCount()); // Verify resources in state LOCALIZED with ref-count=0 is removed. Assert.assertTrue(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 1); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 9
Source File: TestRMAppTransitions.java From hadoop with Apache License 2.0 | 4 votes |
@Before public void setUp() throws Exception { conf = new YarnConfiguration(); AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; if (isSecurityEnabled) { authMethod = AuthenticationMethod.KERBEROS; } SecurityUtil.setAuthenticationMethod(authMethod, conf); UserGroupInformation.setConfiguration(conf); rmDispatcher = new DrainDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class); AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class); store = mock(RMStateStore.class); writer = mock(RMApplicationHistoryWriter.class); DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class); RMContext realRMContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, renewer, new AMRMTokenSecretManager(conf, this.rmContext), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), writer); ((RMContextImpl)realRMContext).setStateStore(store); publisher = mock(SystemMetricsPublisher.class); ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher); this.rmContext = spy(realRMContext); ResourceScheduler resourceScheduler = mock(ResourceScheduler.class); doReturn(null).when(resourceScheduler) .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()); doReturn(resourceScheduler).when(rmContext).getScheduler(); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher(this.rmContext)); rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher(rmContext)); rmDispatcher.register(RMAppManagerEventType.class, new TestApplicationManagerEventDispatcher()); schedulerDispatcher = new TestSchedulerEventDispatcher(); rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher); rmDispatcher.init(conf); rmDispatcher.start(); }
Example 10
Source File: TestRMContainerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testExpireWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class); EventHandler generic = mock(EventHandler.class); drainDispatcher.register(RMAppAttemptEventType.class, appAttemptEventHandler); drainDispatcher.register(RMNodeEventType.class, generic); drainDispatcher.init(new YarnConfiguration()); drainDispatcher.start(); NodeId nodeId = BuilderUtils.newNodeId("host", 3425); ApplicationId appId = BuilderUtils.newApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class); Resource resource = BuilderUtils.newResource(512, 1, 1); Priority priority = BuilderUtils.newPriority(5); Container container = BuilderUtils.newContainer(containerId, nodeId, "host:3465", resource, priority, null); RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class); SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class); RMContext rmContext = mock(RMContext.class); when(rmContext.getDispatcher()).thenReturn(drainDispatcher); when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer); when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer); when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher); when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user", rmContext); assertEquals(RMContainerState.NEW, rmContainer.getState()); assertEquals(resource, rmContainer.getAllocatedResource()); assertEquals(nodeId, rmContainer.getAllocatedNode()); assertEquals(priority, rmContainer.getAllocatedPriority()); verify(writer).containerStarted(any(RMContainer.class)); verify(publisher).containerCreated(any(RMContainer.class), anyLong()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.START)); drainDispatcher.await(); assertEquals(RMContainerState.ALLOCATED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.ACQUIRED)); drainDispatcher.await(); assertEquals(RMContainerState.ACQUIRED, rmContainer.getState()); rmContainer.handle(new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user", rmContainer.getLogURL()); // In RUNNING state. Verify EXPIRE and associated actions. reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils .createAbnormalContainerStatus(containerId, SchedulerUtils.EXPIRED_CONTAINER); rmContainer.handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType.EXPIRE)); drainDispatcher.await(); assertEquals(RMContainerState.RUNNING, rmContainer.getState()); verify(writer, never()).containerFinished(any(RMContainer.class)); verify(publisher, never()).containerFinished(any(RMContainer.class), anyLong()); }
Example 11
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void test() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2); LocalizerContext lc2 = new LocalizerContext(user, cId2, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); LocalizedResource lr2 = createLocalizedResource(req2, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); localrsrc.put(req2, lr2); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(),null); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2); ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2); ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1); // Localize R1 for C1 tracker.handle(req11Event); // Localize R1 for C2 tracker.handle(req12Event); // Localize R2 for C1 tracker.handle(req21Event); dispatcher.await(); verify(localizerEventHandler, times(3)).handle( any(LocalizerResourceRequestEvent.class)); // Verify refCount for R1 is 2 Assert.assertEquals(2, lr1.getRefCount()); // Verify refCount for R2 is 1 Assert.assertEquals(1, lr2.getRefCount()); // Release R2 for C1 tracker.handle(rel21Event); dispatcher.await(); verifyTrackedResourceCount(tracker, 2); // Verify resource with non zero ref count is not removed. Assert.assertEquals(2, lr1.getRefCount()); Assert.assertFalse(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 2); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); // Release resource1 tracker.handle(rel11Event); tracker.handle(rel12Event); Assert.assertEquals(0, lr1.getRefCount()); // Verify resources in state LOCALIZED with ref-count=0 is removed. Assert.assertTrue(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 1); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 12
Source File: TestRMAppTransitions.java From big-c with Apache License 2.0 | 4 votes |
@Before public void setUp() throws Exception { conf = new YarnConfiguration(); AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; if (isSecurityEnabled) { authMethod = AuthenticationMethod.KERBEROS; } SecurityUtil.setAuthenticationMethod(authMethod, conf); UserGroupInformation.setConfiguration(conf); rmDispatcher = new DrainDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class); AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class); store = mock(RMStateStore.class); writer = mock(RMApplicationHistoryWriter.class); DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class); RMContext realRMContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, renewer, new AMRMTokenSecretManager(conf, this.rmContext), new RMContainerTokenSecretManager(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM(), writer); ((RMContextImpl)realRMContext).setStateStore(store); publisher = mock(SystemMetricsPublisher.class); ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher); this.rmContext = spy(realRMContext); ResourceScheduler resourceScheduler = mock(ResourceScheduler.class); doReturn(null).when(resourceScheduler) .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()); doReturn(resourceScheduler).when(rmContext).getScheduler(); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher(this.rmContext)); rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher(rmContext)); rmDispatcher.register(RMAppManagerEventType.class, new TestApplicationManagerEventDispatcher()); schedulerDispatcher = new TestSchedulerEventDispatcher(); rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher); rmDispatcher.init(conf); rmDispatcher.start(); }
Example 13
Source File: TestResourceLocalizationService.java From hadoop with Apache License 2.0 | 4 votes |
@Test( timeout = 10000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizerRunnerException() throws Exception { DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler); dirsHandlerSpy.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getAppId()).thenReturn(appId); spyService.handle(new ApplicationLocalizationEvent( LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0"); final LocalResource resource1 = getPrivateMockedResource(r); System.out.println("Here 4"); final LocalResourceRequest req1 = new LocalResourceRequest(resource1); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors(); constructors[0].setAccessible(true); FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error")); Mockito .doThrow(fsError) .when(dirsHandlerSpy) .getLocalPathForWrite(isA(String.class)); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); Thread.sleep(1000); dispatcher.await(); // Verify if ContainerResourceFailedEvent is invoked on FSError verify(containerBus).handle(isA(ContainerResourceFailedEvent.class)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
Example 14
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") @Test public void testResourcePresentInGoodDir() throws IOException { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); LocalizedResource lr2 = createLocalizedResource(req2, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); localrsrc.put(req2, lr2); LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class); List<String> goodDirs = new ArrayList<String>(); // /tmp/somedir2 is bad goodDirs.add("/tmp/somedir1/"); goodDirs.add("/tmp/somedir2"); Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs); Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs); LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true , conf, new NMNullStateStoreService(), dirsHandler); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1); // Localize R1 for C1 tracker.handle(req11Event); // Localize R2 for C1 tracker.handle(req21Event); dispatcher.await(); // Localize resource1 Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null); Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null); ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1); tracker.handle(rle1); ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1); tracker.handle(rle2); dispatcher.await(); // Remove somedir2 from gooddirs Assert.assertTrue(tracker.checkLocalResource(lr2)); goodDirs.remove(1); Assert.assertFalse(tracker.checkLocalResource(lr2)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 15
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testGetPathForLocalization() throws Exception { FileContext lfs = FileContext.getLocalFSFileContext(); Path base_path = new Path("target", TestLocalResourcesTrackerImpl.class.getSimpleName()); final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); DeletionService delService = mock(DeletionService.class); try { LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, localrsrc, true, conf, stateStore, null); Path conflictPath = new Path(base_path, "10"); Path qualifiedConflictPath = lfs.makeQualified(conflictPath); lfs.mkdir(qualifiedConflictPath, null, true); Path rPath = tracker.getPathForLocalization(req1, base_path, delService); Assert.assertFalse(lfs.util().exists(rPath)); verify(delService, times(1)).delete(eq(user), eq(conflictPath)); } finally { lfs.delete(base_path, true); if (dispatcher != null) { dispatcher.stop(); } } }
Example 16
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResourceWithDirCacheMgr() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDirRoot = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, true, conf, stateStore); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDirRoot + "/4/2", Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); LocalCacheDirectoryManager dirMgrRoot = tracker.getDirectoryManager(localDirRoot); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr2)); final long localizedId2 = localizedId1 + 1; Path hierarchicalPath2 = new Path(localDirRoot + "/4/2", Long.toString(localizedId2)); Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr2)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr3)); final long localizedId3 = 128; Path hierarchicalPath3 = new Path(localDirRoot + "/4/3", Long.toString(localizedId3)); Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr3)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr4)); final long localizedId4 = 256; Path hierarchicalPath4 = new Path(localDirRoot + "/4", Long.toString(localizedId4)); Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr4)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 17
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResource() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDir, Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); // verify new paths reflect recovery of previous resources LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.APPLICATION); LocalizerContext lc2 = new LocalizerContext(user, cId1, null); ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.APPLICATION, lc2); tracker.handle(reqEvent2); dispatcher.await(); Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null); long localizedId2 = Long.parseLong(hierarchicalPath2.getName()); Assert.assertEquals(localizedId1 + 1, localizedId2); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 18
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreFailedLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent( lr1, new Exception("Test").toString()); tracker.handle(rfe1); dispatcher.await(); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 19
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreSuccessfulLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); // Simulate lr1 getting localized ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120); tracker.handle(rle1); dispatcher.await(); ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class); verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture()); LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest( new LocalResourcePBImpl(localizedProto.getResource()))); Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath()); LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1); Assert.assertNotNull(localizedRsrc1); // simulate release and retention processing tracker.handle(new ResourceReleaseEvent(lr1, cId1)); dispatcher.await(); boolean removeResult = tracker.remove(localizedRsrc1, mockDelService); Assert.assertTrue(removeResult); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 20
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void testConsistency() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); // Localize R1 for C1 tracker.handle(req11Event); dispatcher.await(); // Verify refCount for R1 is 1 Assert.assertEquals(1, lr1.getRefCount()); dispatcher.await(); verifyTrackedResourceCount(tracker, 1); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path( "file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1"))); LocalizedResource rsrcbefore = tracker.iterator().next(); File resFile = new File(lr1.getLocalPath().toUri().getRawPath() .toString()); Assert.assertTrue(resFile.exists()); Assert.assertTrue(resFile.delete()); // Localize R1 for C1 tracker.handle(req11Event); dispatcher.await(); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); LocalizedResource rsrcafter = tracker.iterator().next(); if (rsrcbefore == rsrcafter) { Assert.fail("Localized resource should not be equal"); } // Release resource1 tracker.handle(rel11Event); } finally { if (dispatcher != null) { dispatcher.stop(); } } }