Java Code Examples for org.apache.hadoop.yarn.event.DrainDispatcher

The following examples show how to use org.apache.hadoop.yarn.event.DrainDispatcher. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 
Example 2
Source Project: hadoop   Source File: TestApplicationCleanup.java    License: Apache License 2.0 6 votes vote down vote up
protected void waitForContainerCleanup(DrainDispatcher dispatcher, MockNM nm,
    NodeHeartbeatResponse resp) throws Exception {
  int waitCount = 0, cleanedConts = 0;
  List<ContainerId> contsToClean;
  do {
    dispatcher.await();
    contsToClean = resp.getContainersToCleanup();
    cleanedConts += contsToClean.size();
    if (cleanedConts >= 1) {
      break;
    }
    Thread.sleep(100);
    resp = nm.nodeHeartbeat(true);
  } while(waitCount++ < 200);

  if (contsToClean.isEmpty()) {
    LOG.error("Failed to get any containers to cleanup");
  } else {
    LOG.info("Got cleanup for " + contsToClean.get(0));
  }
  Assert.assertEquals(1, cleanedConts);
}
 
Example 3
Source Project: big-c   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore,null);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 
Example 4
Source Project: big-c   Source File: TestApplicationCleanup.java    License: Apache License 2.0 6 votes vote down vote up
protected void waitForContainerCleanup(DrainDispatcher dispatcher, MockNM nm,
    NodeHeartbeatResponse resp) throws Exception {
  int waitCount = 0, cleanedConts = 0;
  List<ContainerId> contsToClean;
  do {
    dispatcher.await();
    contsToClean = resp.getContainersToCleanup();
    cleanedConts += contsToClean.size();
    if (cleanedConts >= 1) {
      break;
    }
    Thread.sleep(100);
    resp = nm.nodeHeartbeat(true);
  } while(waitCount++ < 200);

  if (contsToClean.isEmpty()) {
    LOG.error("Failed to get any containers to cleanup");
  } else {
    LOG.info("Got cleanup for " + contsToClean.get(0));
  }
  Assert.assertEquals(1, cleanedConts);
}
 
Example 5
Source Project: incubator-tez   Source File: TestVertexImpl.java    License: Apache License 2.0 5 votes vote down vote up
public RootInputInitializerManagerControlled(Vertex vertex, AppContext appContext,
                                             EventHandler eventHandler,
                                             DrainDispatcher dispatcher
) throws IOException {
  super(vertex, appContext, UserGroupInformation.getCurrentUser());
  this.eventHandler = eventHandler;
  this.dispatcher = dispatcher;
  this.vertexID = vertex.getVertexId();
}
 
Example 6
Source Project: hadoop   Source File: TestAMRMRPCNodeUpdates.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  dispatcher = new DrainDispatcher();
  this.rm = new MockRM() {
    @Override
    public void init(Configuration conf) {
      conf.set(
        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
        "1.0");
      super.init(conf);
    }
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  amService = rm.getApplicationMasterService();
}
 
Example 7
Source Project: hadoop   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=10000)
public void testFailAbortDoesntHang() throws IOException {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000");
  
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = Mockito.mock(OutputCommitter.class);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();
  //Job has only 1 mapper task. No reducers
  conf.setInt(MRJobConfig.NUM_REDUCES, 0);
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null);

  //Fail / finish all the tasks. This should land the JobImpl directly in the
  //FAIL_ABORT state
  for(Task t: job.tasks.values()) {
    TaskImpl task = (TaskImpl) t;
    task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE));
    for(TaskAttempt ta: task.getAttempts().values()) {
      task.handle(new TaskTAttemptEvent(ta.getID(),
        TaskEventType.T_ATTEMPT_FAILED));
    }
  }

  dispatcher.await();
  //Verify abortJob is called once and the job failed
  Mockito.verify(committer, Mockito.timeout(2000).times(1))
    .abortJob((JobContext) Mockito.any(), (State) Mockito.any());
  assertJobState(job, JobStateInternal.FAILED);

  dispatcher.stop();
}
 
Example 8
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private void finishNextNTasks(DrainDispatcher rmDispatcher, MockNM node,
    MRApp mrApp, Iterator<Task> it, int nextN) throws Exception {
  Task task;
  for (int i=0; i<nextN; i++) {
    task = it.next();
    finishTask(rmDispatcher, node, mrApp, task);
  }
}
 
Example 9
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private void finishTask(DrainDispatcher rmDispatcher, MockNM node,
    MRApp mrApp, Task task) throws Exception {
  TaskAttempt attempt = task.getAttempts().values().iterator().next();
  List<ContainerStatus> contStatus = new ArrayList<ContainerStatus>(1);
  contStatus.add(ContainerStatus.newInstance(attempt.getAssignedContainerID(),
      ContainerState.COMPLETE, "", 0));
  Map<ApplicationId,List<ContainerStatus>> statusUpdate =
      new HashMap<ApplicationId,List<ContainerStatus>>(1);
  statusUpdate.put(mrApp.getAppID(), contStatus);
  node.nodeHeartbeat(statusUpdate, true);
  rmDispatcher.await();
  mrApp.getContext().getEventHandler().handle(
        new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE));
  mrApp.waitForState(task, TaskState.SUCCEEDED);
}
 
Example 10
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private
    List<TaskAttemptContainerAssignedEvent> getContainerOnHost(JobId jobId,
        int taskAttemptId, int memory, String[] hosts, MockNM mockNM,
        DrainDispatcher dispatcher, MyContainerAllocator allocator,
        int expectedAdditions1, int expectedRemovals1,
        int expectedAdditions2, int expectedRemovals2, MyResourceManager rm)
        throws Exception {
  ContainerRequestEvent reqEvent =
      createReq(jobId, taskAttemptId, memory, hosts);
  allocator.sendRequest(reqEvent);

  // Send the request to the RM
  List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
  dispatcher.await();
  assertBlacklistAdditionsAndRemovals(
      expectedAdditions1, expectedRemovals1, rm);
  Assert.assertEquals("No of assignments must be 0", 0, assigned.size());

  // Heartbeat from the required nodeManager
  mockNM.nodeHeartbeat(true);
  dispatcher.await();

  assigned = allocator.schedule();
  dispatcher.await();
  assertBlacklistAdditionsAndRemovals(
      expectedAdditions2, expectedRemovals2, rm);
  return assigned;
}
 
Example 11
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private void finishTask(DrainDispatcher rmDispatcher, MockNM node,
    MRApp mrApp, Task task) throws Exception {
  TaskAttempt attempt = task.getAttempts().values().iterator().next();
  List<ContainerStatus> contStatus = new ArrayList<ContainerStatus>(1);
  contStatus.add(ContainerStatus.newInstance(attempt.getAssignedContainerID(),
      ContainerState.COMPLETE, "", 0));
  Map<ApplicationId,List<ContainerStatus>> statusUpdate =
      new HashMap<ApplicationId,List<ContainerStatus>>(1);
  statusUpdate.put(mrApp.getAppID(), contStatus);
  node.nodeHeartbeat(statusUpdate, true);
  rmDispatcher.await();
  mrApp.getContext().getEventHandler().handle(
        new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE));
  mrApp.waitForState(task, TaskState.SUCCEEDED);
}
 
Example 12
Source Project: incubator-tez   Source File: TestVertexImpl.java    License: Apache License 2.0 5 votes vote down vote up
public VertexImplWithRunningInputInitializer(TezVertexID vertexId,
                                             VertexPlan vertexPlan, String vertexName,
                                             Configuration conf,
                                             EventHandler eventHandler,
                                             TaskAttemptListener taskAttemptListener,
                                             Clock clock, TaskHeartbeatHandler thh,
                                             AppContext appContext,
                                             VertexLocationHint vertexLocationHint,
                                             DrainDispatcher dispatcher,
                                             TezRootInputInitializer presetInitializer) {
  super(vertexId, vertexPlan, vertexName, conf, eventHandler,
      taskAttemptListener, clock, thh, true,
      appContext, vertexLocationHint, null, javaProfilerOptions);
  this.presetInitializer = presetInitializer;
}
 
Example 13
Source Project: big-c   Source File: TestAMRMRPCNodeUpdates.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  dispatcher = new DrainDispatcher();
  this.rm = new MockRM() {
    @Override
    public void init(Configuration conf) {
      conf.set(
        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
        "1.0");
      super.init(conf);
    }
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  amService = rm.getApplicationMasterService();
}
 
Example 14
Source Project: incubator-tez   Source File: TestVertexImpl.java    License: Apache License 2.0 5 votes vote down vote up
public VertexImplWithControlledInitializerManager(TezVertexID vertexId,
                                                  VertexPlan vertexPlan, String vertexName,
                                                  Configuration conf,
                                                  EventHandler eventHandler,
                                                  TaskAttemptListener taskAttemptListener,
                                                  Clock clock, TaskHeartbeatHandler thh,
                                                  AppContext appContext,
                                                  VertexLocationHint vertexLocationHint,
                                                  DrainDispatcher dispatcher) {
  super(vertexId, vertexPlan, vertexName, conf, eventHandler,
      taskAttemptListener, clock, thh, true,
      appContext, vertexLocationHint, null, javaProfilerOptions);
  this.dispatcher = dispatcher;
}
 
Example 15
Source Project: big-c   Source File: TestJobImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=10000)
public void testFailAbortDoesntHang() throws IOException {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000");
  
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = Mockito.mock(OutputCommitter.class);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();
  //Job has only 1 mapper task. No reducers
  conf.setInt(MRJobConfig.NUM_REDUCES, 0);
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null);

  //Fail / finish all the tasks. This should land the JobImpl directly in the
  //FAIL_ABORT state
  for(Task t: job.tasks.values()) {
    TaskImpl task = (TaskImpl) t;
    task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE));
    for(TaskAttempt ta: task.getAttempts().values()) {
      task.handle(new TaskTAttemptEvent(ta.getID(),
        TaskEventType.T_ATTEMPT_FAILED));
    }
  }

  dispatcher.await();
  //Verify abortJob is called once and the job failed
  Mockito.verify(committer, Mockito.timeout(2000).times(1))
    .abortJob((JobContext) Mockito.any(), (State) Mockito.any());
  assertJobState(job, JobStateInternal.FAILED);

  dispatcher.stop();
}
 
Example 16
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private
    List<TaskAttemptContainerAssignedEvent> getContainerOnHost(JobId jobId,
        int taskAttemptId, int memory, String[] hosts, MockNM mockNM,
        DrainDispatcher dispatcher, MyContainerAllocator allocator,
        int expectedAdditions1, int expectedRemovals1,
        int expectedAdditions2, int expectedRemovals2, MyResourceManager rm)
        throws Exception {
  ContainerRequestEvent reqEvent =
      createReq(jobId, taskAttemptId, memory, hosts);
  allocator.sendRequest(reqEvent);

  // Send the request to the RM
  List<TaskAttemptContainerAssignedEvent> assigned = allocator.schedule();
  dispatcher.await();
  assertBlacklistAdditionsAndRemovals(
      expectedAdditions1, expectedRemovals1, rm);
  Assert.assertEquals("No of assignments must be 0", 0, assigned.size());

  // Heartbeat from the required nodeManager
  mockNM.nodeHeartbeat(true);
  dispatcher.await();

  assigned = allocator.schedule();
  dispatcher.await();
  assertBlacklistAdditionsAndRemovals(
      expectedAdditions2, expectedRemovals2, rm);
  return assigned;
}
 
Example 17
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
private void finishNextNTasks(DrainDispatcher rmDispatcher, MockNM node,
    MRApp mrApp, Iterator<Task> it, int nextN) throws Exception {
  Task task;
  for (int i=0; i<nextN; i++) {
    task = it.next();
    finishTask(rmDispatcher, node, mrApp, task);
  }
}
 
Example 18
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void test() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    DeletionService mockDelService = mock(DeletionService.class);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
    LocalizerContext lc2 = new LocalizerContext(user, cId2, null);

    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalResourcesTracker tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            false, conf, new NMNullStateStoreService(),null);

    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req12Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
    ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
    ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    // Localize R1 for C2
    tracker.handle(req12Event);

    // Localize R2 for C1
    tracker.handle(req21Event);

    dispatcher.await();
    verify(localizerEventHandler, times(3)).handle(
        any(LocalizerResourceRequestEvent.class));
    // Verify refCount for R1 is 2
    Assert.assertEquals(2, lr1.getRefCount());
    // Verify refCount for R2 is 1
    Assert.assertEquals(1, lr2.getRefCount());

    // Release R2 for C1
    tracker.handle(rel21Event);

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 2);

    // Verify resource with non zero ref count is not removed.
    Assert.assertEquals(2, lr1.getRefCount());
    Assert.assertFalse(tracker.remove(lr1, mockDelService));
    verifyTrackedResourceCount(tracker, 2);

    // Localize resource1
    ResourceLocalizedEvent rle =
        new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));

    // Release resource1
    tracker.handle(rel11Event);
    tracker.handle(rel12Event);
    Assert.assertEquals(0, lr1.getRefCount());

    // Verify resources in state LOCALIZED with ref-count=0 is removed.
    Assert.assertTrue(tracker.remove(lr1, mockDelService));
    verifyTrackedResourceCount(tracker, 1);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 19
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService(), null);

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 20
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  DeletionService mockDelService = mock(DeletionService.class);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
        null);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    // Simulate lr1 getting localized
    ResourceLocalizedEvent rle1 =
        new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
    tracker.handle(rle1);
    dispatcher.await();

    ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor =
        ArgumentCaptor.forClass(LocalizedResourceProto.class);
    verify(stateStore).finishResourceLocalization(eq(user), eq(appId),
        localizedProtoCaptor.capture());
    LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
    Assert.assertEquals(lr1, new LocalResourceRequest(
        new LocalResourcePBImpl(localizedProto.getResource())));
    Assert.assertEquals(localizedPath1.toString(),
        localizedProto.getLocalPath());
    LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
    Assert.assertNotNull(localizedRsrc1);

    // simulate release and retention processing
    tracker.handle(new ResourceReleaseEvent(lr1, cId1));
    dispatcher.await();
    boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);

    Assert.assertTrue(removeResult);
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 21
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testStateStoreFailedLocalization() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);

    // Container 1 requests lr1 to be localized
    ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
        LocalResourceVisibility.APPLICATION, lc1);
    tracker.handle(reqEvent1);
    dispatcher.await();

    // Simulate the process of localization of lr1
    Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir,
        null);

    ArgumentCaptor<LocalResourceProto> localResourceCaptor =
        ArgumentCaptor.forClass(LocalResourceProto.class);
    ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
    verify(stateStore).startResourceLocalization(eq(user), eq(appId),
        localResourceCaptor.capture(), pathCaptor.capture());
    LocalResourceProto lrProto = localResourceCaptor.getValue();
    Path localizedPath1 = pathCaptor.getValue();
    Assert.assertEquals(lr1,
        new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
    Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());

    ResourceFailedLocalizationEvent rfe1 =
        new ResourceFailedLocalizationEvent(
            lr1, new Exception("Test").toString());
    tracker.handle(rfe1);
    dispatcher.await();
    verify(stateStore).removeLocalizedResource(eq(user), eq(appId),
        eq(localizedPath1));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 22
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testRecoveredResource() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDir = new Path("/tmp/localdir");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, false, conf, stateStore);
    // Container 1 needs lr1 resource
    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.APPLICATION);
    Assert.assertNull(tracker.getLocalizedResource(lr1));
    final long localizedId1 = 52;
    Path hierarchicalPath1 = new Path(localDir,
        Long.toString(localizedId1));
    Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr1));

    // verify new paths reflect recovery of previous resources
    LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2,
        LocalResourceVisibility.APPLICATION);
    LocalizerContext lc2 = new LocalizerContext(user, cId1, null);
    ResourceEvent reqEvent2 = new ResourceRequestEvent(lr2,
        LocalResourceVisibility.APPLICATION, lc2);
    tracker.handle(reqEvent2);
    dispatcher.await();
    Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir,
        null);
    long localizedId2 = Long.parseLong(hierarchicalPath2.getName());
    Assert.assertEquals(localizedId1 + 1, localizedId2);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 23
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testRecoveredResourceWithDirCacheMgr() throws Exception {
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  // This is a random path. NO File creation will take place at this place.
  final Path localDirRoot = new Path("/tmp/localdir");
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);

  try {
    LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, true, conf, stateStore);
    LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    Assert.assertNull(tracker.getLocalizedResource(lr1));
    final long localizedId1 = 52;
    Path hierarchicalPath1 = new Path(localDirRoot + "/4/2",
        Long.toString(localizedId1));
    Path localizedPath1 = new Path(hierarchicalPath1, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr1, localizedPath1, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr1));
    LocalCacheDirectoryManager dirMgrRoot =
        tracker.getDirectoryManager(localDirRoot);
    Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
    Assert.assertEquals(1, dirMgrRoot.getDirectory("4/2").getCount());

    LocalResourceRequest lr2 = createLocalResourceRequest(user, 2, 2,
        LocalResourceVisibility.PUBLIC);
    Assert.assertNull(tracker.getLocalizedResource(lr2));
    final long localizedId2 = localizedId1 + 1;
    Path hierarchicalPath2 = new Path(localDirRoot + "/4/2",
        Long.toString(localizedId2));
    Path localizedPath2 = new Path(hierarchicalPath2, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr2, localizedPath2, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr2));
    Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
    Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());

    LocalResourceRequest lr3 = createLocalResourceRequest(user, 3, 3,
        LocalResourceVisibility.PUBLIC);
    Assert.assertNull(tracker.getLocalizedResource(lr3));
    final long localizedId3 = 128;
    Path hierarchicalPath3 = new Path(localDirRoot + "/4/3",
        Long.toString(localizedId3));
    Path localizedPath3 = new Path(hierarchicalPath3, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr3, localizedPath3, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr3));
    Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
    Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
    Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount());

    LocalResourceRequest lr4 = createLocalResourceRequest(user, 4, 4,
        LocalResourceVisibility.PUBLIC);
    Assert.assertNull(tracker.getLocalizedResource(lr4));
    final long localizedId4 = 256;
    Path hierarchicalPath4 = new Path(localDirRoot + "/4",
        Long.toString(localizedId4));
    Path localizedPath4 = new Path(hierarchicalPath4, "resource.jar");
    tracker.handle(new ResourceRecoveredEvent(lr4, localizedPath4, 120));
    dispatcher.await();
    Assert.assertNotNull(tracker.getLocalizedResource(lr4));
    Assert.assertEquals(0, dirMgrRoot.getDirectory("").getCount());
    Assert.assertEquals(1, dirMgrRoot.getDirectory("4").getCount());
    Assert.assertEquals(2, dirMgrRoot.getDirectory("4/2").getCount());
    Assert.assertEquals(1, dirMgrRoot.getDirectory("4/3").getCount());
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 24
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testGetPathForLocalization() throws Exception {
  FileContext lfs = FileContext.getLocalFSFileContext();
  Path base_path = new Path("target",
      TestLocalResourcesTrackerImpl.class.getSimpleName());
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);
  DeletionService delService = mock(DeletionService.class);
  try {
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, localrsrc, true, conf, stateStore, null);
    Path conflictPath = new Path(base_path, "10");
    Path qualifiedConflictPath = lfs.makeQualified(conflictPath);
    lfs.mkdir(qualifiedConflictPath, null, true);
    Path rPath = tracker.getPathForLocalization(req1, base_path,
        delService);
    Assert.assertFalse(lfs.util().exists(rPath));
    verify(delService, times(1)).delete(eq(user), eq(conflictPath));
  } finally {
    lfs.delete(base_path, true);
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 25
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
    List<String> goodDirs = new ArrayList<String>();
    // /tmp/somedir2 is bad
    goodDirs.add("/tmp/somedir1/");
    goodDirs.add("/tmp/somedir2");
    Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
    Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
    LocalResourcesTrackerImpl tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            true , conf, new NMNullStateStoreService(), dirsHandler);
    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
    // Localize R1 for C1
    tracker.handle(req11Event);
    // Localize R2 for C1
    tracker.handle(req21Event);
    dispatcher.await();
    // Localize resource1
    Path p1 = tracker.getPathForLocalization(req1,
        new Path("/tmp/somedir1"), null);
    Path p2 = tracker.getPathForLocalization(req2,
        new Path("/tmp/somedir2"), null);
    ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
    tracker.handle(rle1);
    ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
    tracker.handle(rle2);
    dispatcher.await();
    // Remove somedir2 from gooddirs
    Assert.assertTrue(tracker.checkLocalResource(lr2));
    goodDirs.remove(1);
    Assert.assertFalse(tracker.checkLocalResource(lr2));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
Example 26
Source Project: hadoop   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 4 votes vote down vote up
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}
 
Example 27
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testRMUnavailable()
    throws Exception {
  Configuration conf = new Configuration();
  conf.setInt(
    MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS, 0);
  MyResourceManager rm1 = new MyResourceManager(conf);
  rm1.start();
  DrainDispatcher dispatcher =
      (DrainDispatcher) rm1.getRMContext().getDispatcher();
  RMApp app = rm1.submitApp(1024);
  dispatcher.await();

  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  nm1.nodeHeartbeat(true);
  dispatcher.await();

  ApplicationAttemptId appAttemptId =
      app.getCurrentAppAttempt().getAppAttemptId();
  rm1.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
      0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
  MyContainerAllocator2 allocator =
      new MyContainerAllocator2(rm1, conf, appAttemptId, mockJob);
  allocator.jobEvents.clear();
  try {
    allocator.schedule();
    Assert.fail("Should Have Exception");
  } catch (YarnRuntimeException e) {
    Assert.assertTrue(e.getMessage().contains("Could not contact RM after"));
  }
  dispatcher.await();
  Assert.assertEquals("Should Have 1 Job Event", 1,
      allocator.jobEvents.size());
  JobEvent event = allocator.jobEvents.get(0); 
  Assert.assertTrue("Should Reboot", event.getType().equals(JobEventType.JOB_AM_REBOOT));
}
 
Example 28
Source Project: hadoop   Source File: TestNonAggregatingLogHandler.java    License: Apache License 2.0 4 votes vote down vote up
private DrainDispatcher createDispatcher(Configuration conf) {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  return dispatcher;
}
 
Example 29
Source Project: hadoop   Source File: TestLogAggregationService.java    License: Apache License 2.0 4 votes vote down vote up
private DrainDispatcher createDispatcher() {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(this.conf);
  dispatcher.start();
  return dispatcher;
}
 
Example 30
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testCompletedTasksRecalculateSchedule() throws Exception {
  LOG.info("Running testCompletedTasksRecalculateSchedule");

  Configuration conf = new Configuration();
  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  // Make a node to register so as to launch the AM.
  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  Job job = mock(Job.class);
  when(job.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
  doReturn(10).when(job).getTotalMaps();
  doReturn(10).when(job).getTotalReduces();
  doReturn(0).when(job).getCompletedMaps();
  RecalculateContainerAllocator allocator =
      new RecalculateContainerAllocator(rm, conf, appAttemptId, job);
  allocator.schedule();

  allocator.recalculatedReduceSchedule = false;
  allocator.schedule();
  Assert.assertFalse("Unexpected recalculate of reduce schedule",
      allocator.recalculatedReduceSchedule);

  doReturn(1).when(job).getCompletedMaps();
  allocator.schedule();
  Assert.assertTrue("Expected recalculate of reduce schedule",
      allocator.recalculatedReduceSchedule);
}