org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
Example #2
Source File: MyriadOperationsTest.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up 
private RMContext generateRMContext(AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler) throws Exception {
  Configuration conf = new Configuration();
  MockRMContext context = null;
  Dispatcher dispatcher = new MockDispatcher();

  RMApplicationHistoryWriter rmApplicationHistoryWriter = new RMApplicationHistoryWriter(); 
  AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(dispatcher);
  AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(dispatcher);    
  RMDelegationTokenSecretManager delegationTokenSecretManager = new RMDelegationTokenSecretManager(1, 1, 1, 1, context);

  context = new MockRMContext();
  context.setStateStore(TestObjectFactory.getStateStore(conf, "tmp/myriad-operations-test"));
  context.setAmLivelinessMonitor(amLivelinessMonitor);
  context.setAmFinishingMonitor(amFinishingMonitor);
  context.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  context.setRMDelegationTokenSecretManager(delegationTokenSecretManager);
  return context;
}
Example #3
Source File: NMHeartBeatHandler.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up 
@Inject
public NMHeartBeatHandler(InterceptorRegistry registry, AbstractYarnScheduler yarnScheduler, MyriadDriver myriadDriver,
                          YarnNodeCapacityManager yarnNodeCapacityMgr, OfferLifecycleManager offerLifecycleMgr,
                          NodeStore nodeStore, SchedulerState state, NodeManagerConfiguration conf) {

  if (registry != null) {
    registry.register(this);
  }

  this.yarnScheduler = yarnScheduler;
  this.myriadDriver = myriadDriver;
  this.yarnNodeCapacityMgr = yarnNodeCapacityMgr;
  this.offerLifecycleMgr = offerLifecycleMgr;
  this.nodeStore = nodeStore;
  this.state = state;
  this.conf = conf;
}
Example #4
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static void waitForNumContainersToRecover(int num, MockRM rm,
    ApplicationAttemptId attemptId) throws Exception {
  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm.getResourceScheduler();
  SchedulerApplicationAttempt attempt =
      scheduler.getApplicationAttempt(attemptId);
  while (attempt == null) {
    System.out.println("Wait for scheduler attempt " + attemptId
        + " to be created");
    Thread.sleep(200);
    attempt = scheduler.getApplicationAttempt(attemptId);
  }
  while (attempt.getLiveContainers().size() < num) {
    System.out.println("Wait for " + num
        + " containers to recover. currently: "
        + attempt.getLiveContainers().size());
    Thread.sleep(200);
  }
}
Example #5
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static void waitForNumContainersToRecover(int num, MockRM rm,
    ApplicationAttemptId attemptId) throws Exception {
  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm.getResourceScheduler();
  SchedulerApplicationAttempt attempt =
      scheduler.getApplicationAttempt(attemptId);
  while (attempt == null) {
    System.out.println("Wait for scheduler attempt " + attemptId
        + " to be created");
    Thread.sleep(200);
    attempt = scheduler.getApplicationAttempt(attemptId);
  }
  while (attempt.getLiveContainers().size() < num) {
    System.out.println("Wait for " + num
        + " containers to recover. currently: "
        + attempt.getLiveContainers().size());
    Thread.sleep(200);
  }
}
Example #6
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
Example #7
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStop() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).stop();
  super.serviceStop();
}
Example #8
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 20000)
public void testContainersNotRecoveredForCompletedApps() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);

  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  NMContainerStatus runningContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2,
        ContainerState.RUNNING);
  NMContainerStatus completedContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 3,
        ContainerState.COMPLETE);
  nm1.registerNode(Arrays.asList(runningContainer, completedContainer), null);
  RMApp recoveredApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  assertEquals(RMAppState.FINISHED, recoveredApp1.getState());

  // Wait for RM to settle down on recovering containers;
  Thread.sleep(3000);

  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();

  // scheduler should not recover containers for finished apps.
  assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
  assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
}
Example #9
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 30000)
public void testAMContainerStatusWithRMRestart() throws Exception {  
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1_1 = rm1.submitApp(1024);
  MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
  
  RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
  AbstractYarnScheduler scheduler =
      ((AbstractYarnScheduler) rm1.getResourceScheduler());
  
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());

  // Re-start RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());

  List<NMContainerStatus> am1_1Containers =
      createNMContainerStatusForApp(am1_1);
  nm1.registerNode(am1_1Containers, null);

  // Wait for RM to settle down on recovering containers;
  waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());

  scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());
}
Example #10
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 20000)
public void testRecoverSchedulerAppAndAttemptSynchronously() throws Exception {
  // start RM
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  // scheduler app/attempt is immediately available after RM is re-started.
  Assert.assertNotNull(rm2.getResourceScheduler().getSchedulerAppInfo(
    am0.getApplicationAttemptId()));

  // getTransferredContainers should not throw NPE.
  ((AbstractYarnScheduler) rm2.getResourceScheduler())
    .getTransferredContainers(am0.getApplicationAttemptId());

  List<NMContainerStatus> containers = createNMContainerStatusForApp(am0);
  nm1.registerNode(containers, null);
  waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId());
}
Example #11
Source File: ReservationSystemTestUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static void initializeRMContext(int numContainers,
    AbstractYarnScheduler scheduler, RMContext mockRMContext) {

  when(mockRMContext.getScheduler()).thenReturn(scheduler);
  Resource r = calculateClusterResource(numContainers);
  doReturn(r).when(scheduler).getClusterResource();
}
Example #12
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromFairScheduler() throws Exception {
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> scheduler =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) resourceManager
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    scheduler.getSchedulerApplications(), scheduler, "default");
}
Example #13
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceInit(Configuration conf) throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).init(conf);
  super.serviceInit(conf);
}
Example #14
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).start();
  super.serviceStart();
}
Example #15
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStop() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).stop();
  super.serviceStop();
}
Example #16
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromFairScheduler() throws Exception {
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> scheduler =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) resourceManager
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    scheduler.getSchedulerApplications(), scheduler, "default");
}
Example #17
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
Example #18
Source File: ReservationSystemTestUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static void initializeRMContext(int numContainers,
    AbstractYarnScheduler scheduler, RMContext mockRMContext) {

  when(mockRMContext.getScheduler()).thenReturn(scheduler);
  Resource r = calculateClusterResource(numContainers);
  doReturn(r).when(scheduler).getClusterResource();
}
Example #19
Source File: MyriadModule.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
public MyriadModule(String configFile, Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext,
                    InterceptorRegistry interceptorRegistry) {
  this.cfg = this.generateMyriadConfiguration(configFile);
  this.hadoopConf = hadoopConf;
  this.yarnScheduler = yarnScheduler;
  this.rmContext = rmContext;
  this.interceptorRegistry = interceptorRegistry;
}
Example #20
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
Example #21
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).start();
  super.serviceStart();
}
Example #22
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
Example #23
Source File: MyriadOperationsTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
private MyriadOperations initialize() throws Exception {
  resetStoreState();
  SchedulerState sState = TestObjectFactory.getSchedulerState(cfg, "tmp/myriad-operations-test");
  sState.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build());
  AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler = TestObjectFactory.getYarnScheduler();
  MyriadDriverManager manager = TestObjectFactory.getMyriadDriverManager();
  MyriadWebServer webServer = TestObjectFactory.getMyriadWebServer(cfg);
  CompositeInterceptor registry = new CompositeInterceptor();
  LeastAMNodesFirstPolicy policy = new LeastAMNodesFirstPolicy(registry, scheduler, sState);

  manager.startDriver();

  return new MyriadOperations(cfg, sState, policy, manager, webServer, generateRMContext(scheduler));
}
Example #24
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
Example #25
Source File: Main.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
public static void initialize(Configuration hadoopConf, AbstractYarnScheduler yarnScheduler, RMContext rmContext,
                              InterceptorRegistry registry) throws Exception {
  MyriadModule myriadModule = new MyriadModule("myriad-config-default.yml", hadoopConf, yarnScheduler, rmContext, registry);
  MesosModule mesosModule = new MesosModule();
  injector = Guice.createInjector(myriadModule, mesosModule, new WebAppGuiceModule());

  new Main().run(injector.getInstance(MyriadConfiguration.class));
}
Example #26
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 20000)
public void testRecoverSchedulerAppAndAttemptSynchronously() throws Exception {
  // start RM
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  // scheduler app/attempt is immediately available after RM is re-started.
  Assert.assertNotNull(rm2.getResourceScheduler().getSchedulerAppInfo(
    am0.getApplicationAttemptId()));

  // getTransferredContainers should not throw NPE.
  ((AbstractYarnScheduler) rm2.getResourceScheduler())
    .getTransferredContainers(am0.getApplicationAttemptId());

  List<NMContainerStatus> containers = createNMContainerStatusForApp(am0);
  nm1.registerNode(containers, null);
  waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId());
}
Example #27
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 30000)
public void testAMContainerStatusWithRMRestart() throws Exception {  
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1_1 = rm1.submitApp(1024);
  MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
  
  RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
  AbstractYarnScheduler scheduler =
      ((AbstractYarnScheduler) rm1.getResourceScheduler());
  
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());

  // Re-start RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());

  List<NMContainerStatus> am1_1Containers =
      createNMContainerStatusForApp(am1_1);
  nm1.registerNode(am1_1Containers, null);

  // Wait for RM to settle down on recovering containers;
  waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());

  scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());
}
Example #28
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test(timeout = 20000)
public void testContainersNotRecoveredForCompletedApps() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);

  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  NMContainerStatus runningContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2,
        ContainerState.RUNNING);
  NMContainerStatus completedContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 3,
        ContainerState.COMPLETE);
  nm1.registerNode(Arrays.asList(runningContainer, completedContainer), null);
  RMApp recoveredApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  assertEquals(RMAppState.FINISHED, recoveredApp1.getState());

  // Wait for RM to settle down on recovering containers;
  Thread.sleep(3000);

  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();

  // scheduler should not recover containers for finished apps.
  assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
  assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
}
Example #29
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceInit(Configuration conf) throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).init(conf);
  super.serviceInit(conf);
}
Example #30
Source File: YarnNodeCapacityManager.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
@Inject
public YarnNodeCapacityManager(InterceptorRegistry registry, AbstractYarnScheduler yarnScheduler, RMContext rmContext,
                               MyriadDriver myriadDriver, OfferLifecycleManager offerLifecycleMgr, NodeStore nodeStore,
                               SchedulerState state, TaskUtils taskUtils) {
  if (registry != null) {
    registry.register(this);
  }
  this.yarnScheduler = yarnScheduler;
  this.rmContext = rmContext;
  this.myriadDriver = myriadDriver;
  this.offerLifecycleMgr = offerLifecycleMgr;
  this.nodeStore = nodeStore;
  this.state = state;
  this.taskUtils = taskUtils;
}