org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
 
Example #2
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #3
Source File: TestRMWebServicesHttpStaticUserPermissions.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal);
  rmconf.set("yarn.resourcemanager.keytab",
      spnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #4
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 
Example #5
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAsyncScheduling() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  final int NODES = 100;
  
  // Register nodes
  for (int i=0; i < NODES; ++i) {
    String host = "192.168.1." + i;
    RMNode node =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
  }
  
  // Now directly exercise the scheduling loop
  for (int i=0; i < NODES; ++i) {
    CapacityScheduler.schedule(cs);
  }
}
 
Example #6
Source File: RMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #7
Source File: RMContextImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example #8
Source File: TestAppManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
@Before
public void setUp() {
  long now = System.currentTimeMillis();

  rmContext = mockRMContext(1, now - 10);
  ResourceScheduler scheduler = mockResourceScheduler();
  Configuration conf = new Configuration();
  ApplicationMasterService masterService =
      new ApplicationMasterService(rmContext, scheduler);
  appMonitor = new TestRMAppManager(rmContext,
      new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
      new ApplicationACLsManager(conf), conf);

  appId = MockApps.newAppID(1);
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  asContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  asContext.setApplicationId(appId);
  asContext.setAMContainerSpec(mockContainerLaunchContext(recordFactory));
  asContext.setResource(mockResource());
  setupDispatcher(rmContext, conf);
}
 
Example #9
Source File: RmController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #10
Source File: RmController.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #11
Source File: RMContextImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
// helper constructor for tests
public RMContextImpl(Dispatcher rmDispatcher,
    ContainerAllocationExpirer containerAllocationExpirer,
    AMLivelinessMonitor amLivelinessMonitor,
    AMLivelinessMonitor amFinishingMonitor,
    DelegationTokenRenewer delegationTokenRenewer,
    AMRMTokenSecretManager appTokenSecretManager,
    RMContainerTokenSecretManager containerTokenSecretManager,
    NMTokenSecretManagerInRM nmTokenSecretManager,
    ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
    RMApplicationHistoryWriter rmApplicationHistoryWriter,
    ResourceScheduler scheduler) {
  this();
  this.setDispatcher(rmDispatcher);
  setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      delegationTokenRenewer, appTokenSecretManager,
      containerTokenSecretManager, nmTokenSecretManager,
      clientToAMTokenSecretManager, rmApplicationHistoryWriter,
      scheduler));

  ConfigurationProvider provider = new LocalConfigurationProvider();
  setConfigurationProvider(provider);
}
 
Example #12
Source File: UnitTestCluster.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception {
  yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
  yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
  MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1);
  miniYarnCluster.init(yarnConf);
  yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ;
  miniYarnCluster.start();
  return miniYarnCluster ;
}
 
Example #13
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #14
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private MockRM setUpMove(Configuration config) {
  CapacitySchedulerConfiguration conf =
      new CapacitySchedulerConfiguration(config);
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  return rm;
}
 
Example #15
Source File: TestCapacitySchedulerNodeLabelUpdate.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  mgr = new NullRMNodeLabelsManager();
  mgr.init(conf);
}
 
Example #16
Source File: FairReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is fair scheduler
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof FairScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + FairScheduler.class.getCanonicalName());
  }
  fairScheduler = (FairScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}
 
Example #17
Source File: YARNSessionCapacitySchedulerITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team");
	YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40);
	YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler");
	startYARNWithConfig(YARN_CONFIGURATION);

	restClientExecutor = Executors.newSingleThreadExecutor();
	restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor);
}
 
Example #18
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
 
Example #19
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example #20
Source File: TestRMWebServicesCapacitySched.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  csConf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
  ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #21
Source File: TestClientRMTokens.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ClientRMServiceForTest(Configuration conf,
    ResourceScheduler scheduler,
    RMDelegationTokenSecretManager rmDTSecretManager) {
  super(mock(RMContext.class), scheduler, mock(RMAppManager.class),
      new ApplicationACLsManager(conf), new QueueACLsManager(scheduler,
          conf), rmDTSecretManager);
}
 
Example #22
Source File: TestRMNMInfo.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testRMNMInfo() throws Exception {
  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
         + " not found. Not running test.");
    return;
  }
  
  RMContext rmc = mrCluster.getResourceManager().getRMContext();
  ResourceScheduler rms = mrCluster.getResourceManager()
                                                 .getResourceScheduler();
  RMNMInfo rmInfo = new RMNMInfo(rmc,rms);
  String liveNMs = rmInfo.getLiveNodeManagers();
  ObjectMapper mapper = new ObjectMapper();
  JsonNode jn = mapper.readTree(liveNMs);
  Assert.assertEquals("Unexpected number of live nodes:",
                                             NUMNODEMANAGERS, jn.size());
  Iterator<JsonNode> it = jn.iterator();
  while (it.hasNext()) {
    JsonNode n = it.next();
    Assert.assertNotNull(n.get("HostName"));
    Assert.assertNotNull(n.get("Rack"));
    Assert.assertTrue("Node " + n.get("NodeId") + " should be RUNNING",
            n.get("State").asText().contains("RUNNING"));
    Assert.assertNotNull(n.get("NodeHTTPAddress"));
    Assert.assertNotNull(n.get("LastHealthUpdate"));
    Assert.assertNotNull(n.get("HealthReport"));
    Assert.assertNotNull(n.get("NodeManagerVersion"));
    Assert.assertNotNull(n.get("NumContainers"));
    Assert.assertEquals(
            n.get("NodeId") + ": Unexpected number of used containers",
            0, n.get("NumContainers").asInt());
    Assert.assertEquals(
            n.get("NodeId") + ": Unexpected amount of used memory",
            0, n.get("UsedMemoryMB").asInt());
    Assert.assertNotNull(n.get("AvailableMemoryMB"));
  }
}
 
Example #23
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static ResourceManager mockRmWithApps(RMContext rmContext) throws
    IOException {
  ResourceManager rm = mock(ResourceManager.class);
  ResourceScheduler rs =  mockFairSchedulerWithoutApps(rmContext);
  ClientRMService clientRMService = mockClientRMService(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  when(rm.getClientRMService()).thenReturn(clientRMService);
  return rm;
}
 
Example #24
Source File: YARNHighAvailabilityITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	storageDir = FOLDER.newFolder().getAbsolutePath();

	// startYARNWithConfig should be implemented by subclass
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #25
Source File: TestFairReservationSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example #26
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAppsInQueue() throws Exception {
  Application application_0 = new Application("user_0", "a1", resourceManager);
  application_0.submit();
  
  Application application_1 = new Application("user_0", "a2", resourceManager);
  application_1.submit();
  
  Application application_2 = new Application("user_0", "b2", resourceManager);
  application_2.submit();
  
  ResourceScheduler scheduler = resourceManager.getResourceScheduler();
  
  List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
  assertEquals(1, appsInA1.size());
  
  List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
  assertTrue(appsInA.contains(application_0.getApplicationAttemptId()));
  assertTrue(appsInA.contains(application_1.getApplicationAttemptId()));
  assertEquals(2, appsInA.size());

  List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
  assertTrue(appsInRoot.contains(application_0.getApplicationAttemptId()));
  assertTrue(appsInRoot.contains(application_1.getApplicationAttemptId()));
  assertTrue(appsInRoot.contains(application_2.getApplicationAttemptId()));
  assertEquals(3, appsInRoot.size());
  
  Assert.assertNull(scheduler.getAppsInQueue("nonexistentqueue"));
}
 
Example #27
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 
Example #28
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #29
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
                                     int mbsPerNode)
throws Exception {
  ResourceManager rm = mock(ResourceManager.class);
  RMContext rmContext = mockRMContext(apps, racks, nodes,
      mbsPerNode);
  ResourceScheduler rs = mockFifoScheduler(rmContext);
  when(rm.getResourceScheduler()).thenReturn(rs);
  when(rm.getRMContext()).thenReturn(rmContext);
  return rm;
}
 
Example #30
Source File: TestRMWebServicesFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}