Java Code Examples for org.apache.hadoop.yarn.event.DrainDispatcher#stop()
The following examples show how to use
org.apache.hadoop.yarn.event.DrainDispatcher#stop() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestJobImpl.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout=10000) public void testFailAbortDoesntHang() throws IOException { Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000"); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); OutputCommitter committer = Mockito.mock(OutputCommitter.class); CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer); commitHandler.init(conf); commitHandler.start(); //Job has only 1 mapper task. No reducers conf.setInt(MRJobConfig.NUM_REDUCES, 0); conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1); JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null); //Fail / finish all the tasks. This should land the JobImpl directly in the //FAIL_ABORT state for(Task t: job.tasks.values()) { TaskImpl task = (TaskImpl) t; task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE)); for(TaskAttempt ta: task.getAttempts().values()) { task.handle(new TaskTAttemptEvent(ta.getID(), TaskEventType.T_ATTEMPT_FAILED)); } } dispatcher.await(); //Verify abortJob is called once and the job failed Mockito.verify(committer, Mockito.timeout(2000).times(1)) .abortJob((JobContext) Mockito.any(), (State) Mockito.any()); assertJobState(job, JobStateInternal.FAILED); dispatcher.stop(); }
Example 2
Source File: TestJobImpl.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout=10000) public void testFailAbortDoesntHang() throws IOException { Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000"); DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); OutputCommitter committer = Mockito.mock(OutputCommitter.class); CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer); commitHandler.init(conf); commitHandler.start(); //Job has only 1 mapper task. No reducers conf.setInt(MRJobConfig.NUM_REDUCES, 0); conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1); JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null); //Fail / finish all the tasks. This should land the JobImpl directly in the //FAIL_ABORT state for(Task t: job.tasks.values()) { TaskImpl task = (TaskImpl) t; task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE)); for(TaskAttempt ta: task.getAttempts().values()) { task.handle(new TaskTAttemptEvent(ta.getID(), TaskEventType.T_ATTEMPT_FAILED)); } } dispatcher.await(); //Verify abortJob is called once and the job failed Mockito.verify(committer, Mockito.timeout(2000).times(1)) .abortJob((JobContext) Mockito.any(), (State) Mockito.any()); assertJobState(job, JobStateInternal.FAILED); dispatcher.stop(); }
Example 3
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreSuccessfulLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); // Simulate lr1 getting localized ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120); tracker.handle(rle1); dispatcher.await(); ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class); verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture()); LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest( new LocalResourcePBImpl(localizedProto.getResource()))); Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath()); LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1); Assert.assertNotNull(localizedRsrc1); // simulate release and retention processing tracker.handle(new ResourceReleaseEvent(lr1, cId1)); dispatcher.await(); boolean removeResult = tracker.remove(localizedRsrc1, mockDelService); Assert.assertTrue(removeResult); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 4
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreFailedLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent( lr1, new Exception("Test").toString()); tracker.handle(rfe1); dispatcher.await(); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 5
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResource() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDir, Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); // verify new paths reflect recovery of previous resources LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.APPLICATION); LocalizerContext lc2 = new LocalizerContext(user, cId1, null); ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.APPLICATION, lc2); tracker.handle(reqEvent2); dispatcher.await(); Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir, null); long localizedId2 = Long.parseLong(hierarchicalPath2.getName()); Assert.assertEquals(localizedId1 + 1, localizedId2); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 6
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResourceWithDirCacheMgr() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDirRoot = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, true, conf, stateStore); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDirRoot + "/4/2", Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); LocalCacheDirectoryManager dirMgrRoot = tracker.getDirectoryManager(localDirRoot); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr2)); final long localizedId2 = localizedId1 + 1; Path hierarchicalPath2 = new Path(localDirRoot + "/4/2", Long.toString(localizedId2)); Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr2)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr3)); final long localizedId3 = 128; Path hierarchicalPath3 = new Path(localDirRoot + "/4/3", Long.toString(localizedId3)); Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr3)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr4)); final long localizedId4 = 256; Path hierarchicalPath4 = new Path(localDirRoot + "/4", Long.toString(localizedId4)); Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr4)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 7
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testGetPathForLocalization() throws Exception { FileContext lfs = FileContext.getLocalFSFileContext(); Path base_path = new Path("target", TestLocalResourcesTrackerImpl.class.getSimpleName()); final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); DeletionService delService = mock(DeletionService.class); try { LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, localrsrc, true, conf, stateStore, null); Path conflictPath = new Path(base_path, "10"); Path qualifiedConflictPath = lfs.makeQualified(conflictPath); lfs.mkdir(qualifiedConflictPath, null, true); Path rPath = tracker.getPathForLocalization(req1, base_path, delService); Assert.assertFalse(lfs.util().exists(rPath)); verify(delService, times(1)).delete(eq(user), eq(conflictPath)); } finally { lfs.delete(base_path, true); if (dispatcher != null) { dispatcher.stop(); } } }
Example 8
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") @Test public void testResourcePresentInGoodDir() throws IOException { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); LocalizedResource lr2 = createLocalizedResource(req2, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); localrsrc.put(req2, lr2); LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class); List<String> goodDirs = new ArrayList<String>(); // /tmp/somedir2 is bad goodDirs.add("/tmp/somedir1/"); goodDirs.add("/tmp/somedir2"); Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs); Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs); LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true , conf, new NMNullStateStoreService(), dirsHandler); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1); // Localize R1 for C1 tracker.handle(req11Event); // Localize R2 for C1 tracker.handle(req21Event); dispatcher.await(); // Localize resource1 Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null); Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null); ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1); tracker.handle(rle1); ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1); tracker.handle(rle2); dispatcher.await(); // Remove somedir2 from gooddirs Assert.assertTrue(tracker.checkLocalResource(lr2)); goodDirs.remove(1); Assert.assertFalse(tracker.checkLocalResource(lr2)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 9
Source File: TestResourceLocalizationService.java From hadoop with Apache License 2.0 | 4 votes |
@Test( timeout = 10000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizerRunnerException() throws Exception { DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler); dirsHandlerSpy.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getAppId()).thenReturn(appId); spyService.handle(new ApplicationLocalizationEvent( LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0"); final LocalResource resource1 = getPrivateMockedResource(r); System.out.println("Here 4"); final LocalResourceRequest req1 = new LocalResourceRequest(resource1); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors(); constructors[0].setAccessible(true); FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error")); Mockito .doThrow(fsError) .when(dirsHandlerSpy) .getLocalPathForWrite(isA(String.class)); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); Thread.sleep(1000); dispatcher.await(); // Verify if ContainerResourceFailedEvent is invoked on FSError verify(containerBus).handle(isA(ContainerResourceFailedEvent.class)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
Example 10
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void test() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2); LocalizerContext lc2 = new LocalizerContext(user, cId2, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); LocalizedResource lr2 = createLocalizedResource(req2, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); localrsrc.put(req2, lr2); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(),null); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2); ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2); ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1); // Localize R1 for C1 tracker.handle(req11Event); // Localize R1 for C2 tracker.handle(req12Event); // Localize R2 for C1 tracker.handle(req21Event); dispatcher.await(); verify(localizerEventHandler, times(3)).handle( any(LocalizerResourceRequestEvent.class)); // Verify refCount for R1 is 2 Assert.assertEquals(2, lr1.getRefCount()); // Verify refCount for R2 is 1 Assert.assertEquals(1, lr2.getRefCount()); // Release R2 for C1 tracker.handle(rel21Event); dispatcher.await(); verifyTrackedResourceCount(tracker, 2); // Verify resource with non zero ref count is not removed. Assert.assertEquals(2, lr1.getRefCount()); Assert.assertFalse(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 2); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); // Release resource1 tracker.handle(rel11Event); tracker.handle(rel12Event); Assert.assertEquals(0, lr1.getRefCount()); // Verify resources in state LOCALIZED with ref-count=0 is removed. Assert.assertTrue(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 1); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 11
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void test() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2); LocalizerContext lc2 = new LocalizerContext(user, cId2, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); LocalizedResource lr2 = createLocalizedResource(req2, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); localrsrc.put(req2, lr2); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService()); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2); ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2); ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1); // Localize R1 for C1 tracker.handle(req11Event); // Localize R1 for C2 tracker.handle(req12Event); // Localize R2 for C1 tracker.handle(req21Event); dispatcher.await(); verify(localizerEventHandler, times(3)).handle( any(LocalizerResourceRequestEvent.class)); // Verify refCount for R1 is 2 Assert.assertEquals(2, lr1.getRefCount()); // Verify refCount for R2 is 1 Assert.assertEquals(1, lr2.getRefCount()); // Release R2 for C1 tracker.handle(rel21Event); dispatcher.await(); verifyTrackedResourceCount(tracker, 2); // Verify resource with non zero ref count is not removed. Assert.assertEquals(2, lr1.getRefCount()); Assert.assertFalse(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 2); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); // Release resource1 tracker.handle(rel11Event); tracker.handle(rel12Event); Assert.assertEquals(0, lr1.getRefCount()); // Verify resources in state LOCALIZED with ref-count=0 is removed. Assert.assertTrue(tracker.remove(lr1, mockDelService)); verifyTrackedResourceCount(tracker, 1); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 12
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void testConsistency() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService()); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); // Localize R1 for C1 tracker.handle(req11Event); dispatcher.await(); // Verify refCount for R1 is 1 Assert.assertEquals(1, lr1.getRefCount()); dispatcher.await(); verifyTrackedResourceCount(tracker, 1); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path( "file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1"))); LocalizedResource rsrcbefore = tracker.iterator().next(); File resFile = new File(lr1.getLocalPath().toUri().getRawPath() .toString()); Assert.assertTrue(resFile.exists()); Assert.assertTrue(resFile.delete()); // Localize R1 for C1 tracker.handle(req11Event); dispatcher.await(); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); LocalizedResource rsrcafter = tracker.iterator().next(); if (rsrcbefore == rsrcafter) { Assert.fail("Localized resource should not be equal"); } // Release resource1 tracker.handle(rel11Event); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 13
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreSuccessfulLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); DeletionService mockDelService = mock(DeletionService.class); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); // Simulate lr1 getting localized ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120); tracker.handle(rle1); dispatcher.await(); ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class); verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture()); LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest( new LocalResourcePBImpl(localizedProto.getResource()))); Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath()); LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1); Assert.assertNotNull(localizedRsrc1); // simulate release and retention processing tracker.handle(new ResourceReleaseEvent(lr1, cId1)); dispatcher.await(); boolean removeResult = tracker.remove(localizedRsrc1, mockDelService); Assert.assertTrue(removeResult); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 14
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testStateStoreFailedLocalization() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); // Container 1 requests lr1 to be localized ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1); tracker.handle(reqEvent1); dispatcher.await(); // Simulate the process of localization of lr1 Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir); ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class); ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture()); LocalResourceProto lrProto = localResourceCaptor.getValue(); Path localizedPath1 = pathCaptor.getValue(); Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto))); Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent()); ResourceFailedLocalizationEvent rfe1 = new ResourceFailedLocalizationEvent( lr1, new Exception("Test").toString()); tracker.handle(rfe1); dispatcher.await(); verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1)); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 15
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResource() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDir = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDir, Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); // verify new paths reflect recovery of previous resources LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.APPLICATION); LocalizerContext lc2 = new LocalizerContext(user, cId1, null); ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2, LocalResourceVisibility.APPLICATION, lc2); tracker.handle(reqEvent2); dispatcher.await(); Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir); long localizedId2 = Long.parseLong(hierarchicalPath2.getName()); Assert.assertEquals(localizedId1 + 1, localizedId2); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 16
Source File: TestLocalResourcesTrackerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Test @SuppressWarnings("unchecked") public void testRecoveredResourceWithDirCacheMgr() throws Exception { final String user = "someuser"; final ApplicationId appId = ApplicationId.newInstance(1, 1); // This is a random path. NO File creation will take place at this place. final Path localDirRoot = new Path("/tmp/localdir"); Configuration conf = new YarnConfiguration(); DrainDispatcher dispatcher = null; dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); NMStateStoreService stateStore = mock(NMStateStoreService.class); try { LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, true, conf, stateStore); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr1)); final long localizedId1 = 52; Path hierarchicalPath1 = new Path(localDirRoot + "/4/2", Long.toString(localizedId1)); Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr1)); LocalCacheDirectoryManager dirMgrRoot = tracker.getDirectoryManager(localDirRoot); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr2)); final long localizedId2 = localizedId1 + 1; Path hierarchicalPath2 = new Path(localDirRoot + "/4/2", Long.toString(localizedId2)); Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr2)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr3)); final long localizedId3 = 128; Path hierarchicalPath3 = new Path(localDirRoot + "/4/3", Long.toString(localizedId3)); Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr3)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr4)); final long localizedId4 = 256; Path hierarchicalPath4 = new Path(localDirRoot + "/4", Long.toString(localizedId4)); Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar"); tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120)); dispatcher.await(); Assert.assertNotNull(tracker.getLocalizedResource(lr4)); Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount()); Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount()); Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount()); } finally { if (dispatcher != null) { dispatcher.stop(); } } }
Example 17
Source File: TestResourceLocalizationService.java From big-c with Apache License 2.0 | 4 votes |
@Test( timeout = 10000) @SuppressWarnings("unchecked") // mocked generics public void testLocalizerRunnerException() throws Exception { DrainDispatcher dispatcher = new DrainDispatcher(); dispatcher.init(conf); dispatcher.start(); EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class); dispatcher.register(ApplicationEventType.class, applicationBus); EventHandler<ContainerEvent> containerBus = mock(EventHandler.class); dispatcher.register(ContainerEventType.class, containerBus); ContainerExecutor exec = mock(ContainerExecutor.class); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler); dirsHandlerSpy.init(conf); DeletionService delServiceReal = new DeletionService(exec); DeletionService delService = spy(delServiceReal); delService.init(new Configuration()); delService.start(); ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext); ResourceLocalizationService spyService = spy(rawService); doReturn(mockServer).when(spyService).createServer(); try { spyService.init(conf); spyService.start(); // init application final Application app = mock(Application.class); final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3); when(app.getUser()).thenReturn("user0"); when(app.getAppId()).thenReturn(appId); spyService.handle(new ApplicationLocalizationEvent( LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); dispatcher.await(); Random r = new Random(); long seed = r.nextLong(); System.out.println("SEED: " + seed); r.setSeed(seed); final Container c = getMockContainer(appId, 42, "user0"); final LocalResource resource1 = getPrivateMockedResource(r); System.out.println("Here 4"); final LocalResourceRequest req1 = new LocalResourceRequest(resource1); Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>(); List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>(); privateResourceList.add(req1); rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList); final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors(); constructors[0].setAccessible(true); FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error")); Mockito .doThrow(fsError) .when(dirsHandlerSpy) .getLocalPathForWrite(isA(String.class)); spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs)); Thread.sleep(1000); dispatcher.await(); // Verify if ContainerResourceFailedEvent is invoked on FSError verify(containerBus).handle(isA(ContainerResourceFailedEvent.class)); } finally { spyService.stop(); dispatcher.stop(); delService.stop(); } }
Example 18
Source File: TestLocalResourcesTrackerImpl.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout=10000) @SuppressWarnings("unchecked") public void testConsistency() { String user = "testuser"; DrainDispatcher dispatcher = null; try { Configuration conf = new Configuration(); dispatcher = createDispatcher(conf); EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class); EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class); dispatcher.register(LocalizerEventType.class, localizerEventHandler); dispatcher.register(ContainerEventType.class, containerEventHandler); ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalizerContext lc1 = new LocalizerContext(user, cId1, null); LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); LocalizedResource lr1 = createLocalizedResource(req1, dispatcher); ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>(); localrsrc.put(req1, lr1); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null); ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1); ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1); // Localize R1 for C1 tracker.handle(req11Event); dispatcher.await(); // Verify refCount for R1 is 1 Assert.assertEquals(1, lr1.getRefCount()); dispatcher.await(); verifyTrackedResourceCount(tracker, 1); // Localize resource1 ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path( "file:///tmp/r1"), 1); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1"))); LocalizedResource rsrcbefore = tracker.iterator().next(); File resFile = new File(lr1.getLocalPath().toUri().getRawPath() .toString()); Assert.assertTrue(resFile.exists()); Assert.assertTrue(resFile.delete()); // Localize R1 for C1 tracker.handle(req11Event); dispatcher.await(); lr1.handle(rle); Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED)); LocalizedResource rsrcafter = tracker.iterator().next(); if (rsrcbefore == rsrcafter) { Assert.fail("Localized resource should not be equal"); } // Release resource1 tracker.handle(rel11Event); } finally { if (dispatcher != null) { dispatcher.stop(); } } }