org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReservationSystemTestUtil.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public CapacityScheduler mockCapacityScheduler(int numContainers)
    throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
  cs.setConf(new YarnConfiguration());

  RMContext mockRmContext = createRMContext(conf);

  cs.setRMContext(mockRmContext);
  try {
    cs.serviceInit(conf);
  } catch (Exception e) {
    Assert.fail(e.getMessage());
  }

  initializeRMContext(numContainers, cs, mockRmContext);
  return cs;
}
 
Example #2
Source File: RmController.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #3
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 
Example #4
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 
Example #5
Source File: TestRMAdminService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  configuration = new YarnConfiguration();
  configuration.set(YarnConfiguration.RM_SCHEDULER,
      CapacityScheduler.class.getCanonicalName());
  fs = FileSystem.get(configuration);
  workingPath =
      new Path(new File("target", this.getClass().getSimpleName()
          + "-remoteDir").getAbsolutePath());
  configuration.set(YarnConfiguration.FS_BASED_RM_CONF_STORE,
      workingPath.toString());
  tmpDir = new Path(new File("target", this.getClass().getSimpleName()
      + "-tmpDir").getAbsolutePath());
  fs.delete(workingPath, true);
  fs.delete(tmpDir, true);
  fs.mkdirs(workingPath);
  fs.mkdirs(tmpDir);

  // reset the groups to what it default test settings
  MockUnixGroupsMapping.resetGroups();
}
 
Example #6
Source File: TestRMAdminService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testAdminRefreshQueuesWithLocalConfigurationProvider()
    throws IOException, YarnException {
  rm = new MockRM(configuration);
  rm.init(configuration);
  rm.start();

  CapacityScheduler cs =
      (CapacityScheduler) rm.getRMContext().getScheduler();
  int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();

  try {
    rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
    Assert.assertEquals(maxAppsBefore, cs.getConfiguration()
        .getMaximumSystemApplications());
  } catch (Exception ex) {
    fail("Using localConfigurationProvider. Should not get any exception.");
  }
}
 
Example #7
Source File: TestRMAdminService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  configuration = new YarnConfiguration();
  configuration.set(YarnConfiguration.RM_SCHEDULER,
      CapacityScheduler.class.getCanonicalName());
  fs = FileSystem.get(configuration);
  workingPath =
      new Path(new File("target", this.getClass().getSimpleName()
          + "-remoteDir").getAbsolutePath());
  configuration.set(YarnConfiguration.FS_BASED_RM_CONF_STORE,
      workingPath.toString());
  tmpDir = new Path(new File("target", this.getClass().getSimpleName()
      + "-tmpDir").getAbsolutePath());
  fs.delete(workingPath, true);
  fs.delete(tmpDir, true);
  fs.mkdirs(workingPath);
  fs.mkdirs(tmpDir);

  // reset the groups to what it default test settings
  MockUnixGroupsMapping.resetGroups();
}
 
Example #8
Source File: TestRMAdminService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAdminRefreshQueuesWithLocalConfigurationProvider()
    throws IOException, YarnException {
  rm = new MockRM(configuration);
  rm.init(configuration);
  rm.start();

  CapacityScheduler cs =
      (CapacityScheduler) rm.getRMContext().getScheduler();
  int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();

  try {
    rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
    Assert.assertEquals(maxAppsBefore, cs.getConfiguration()
        .getMaximumSystemApplications());
  } catch (Exception ex) {
    fail("Using localConfigurationProvider. Should not get any exception.");
  }
}
 
Example #9
Source File: RMWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo = new CapacitySchedulerInfo(root);
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #10
Source File: RmController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #11
Source File: ReservationSystemTestUtil.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public CapacityScheduler mockCapacityScheduler(int numContainers)
    throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
  cs.setConf(new YarnConfiguration());

  RMContext mockRmContext = createRMContext(conf);

  cs.setRMContext(mockRmContext);
  try {
    cs.serviceInit(conf);
  } catch (Exception e) {
    Assert.fail(e.getMessage());
  }

  initializeRMContext(numContainers, cs, mockRmContext);
  return cs;
}
 
Example #12
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
 
Example #13
Source File: RMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #14
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
 
Example #15
Source File: CapacityReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is capacity based
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof CapacityScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + CapacityScheduler.class.getCanonicalName());
  }
  capScheduler = (CapacityScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}
 
Example #16
Source File: ParameterizedSchedulerTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void configureScheduler() throws IOException {
  conf = new YarnConfiguration();
  switch (schedulerType) {
    case CAPACITY:
      conf.set(YarnConfiguration.RM_SCHEDULER,
          CapacityScheduler.class.getName());
      break;
    case FAIR:
      configureFairScheduler(conf);
      break;
  }
}
 
Example #17
Source File: AbstractReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 
Example #18
Source File: TestRMWebServicesCapacitySched.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  csConf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
  ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #19
Source File: AbstractReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
private String getDefaultPlanFollower() {
  // currently only capacity scheduler is supported
  if (scheduler instanceof CapacityScheduler) {
    return CapacitySchedulerPlanFollower.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairSchedulerPlanFollower.class.getName();
  }
  return null;
}
 
Example #20
Source File: TestProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 
Example #21
Source File: TestRMAdminService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAdminRefreshQueuesWithFileSystemBasedConfigurationProvider()
    throws IOException, YarnException {
  configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
      "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");

  //upload default configurations
  uploadDefaultConfiguration();

  try {
    rm = new MockRM(configuration);
    rm.init(configuration);
    rm.start();
  } catch(Exception ex) {
    fail("Should not get any exceptions");
  }

  CapacityScheduler cs =
      (CapacityScheduler) rm.getRMContext().getScheduler();
  int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();

  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  csConf.set("yarn.scheduler.capacity.maximum-applications", "5000");
  uploadConfiguration(csConf, "capacity-scheduler.xml");

  rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());

  int maxAppsAfter = cs.getConfiguration().getMaximumSystemApplications();
  Assert.assertEquals(maxAppsAfter, 5000);
  Assert.assertTrue(maxAppsAfter != maxAppsBefore);
}
 
Example #22
Source File: TestProportionalCapacityPreemptionPolicy.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
@SuppressWarnings("unchecked")
public void setup() {
  conf = new Configuration(false);
  conf.setLong(WAIT_TIME_BEFORE_KILL, 10000);
  conf.setLong(MONITORING_INTERVAL, 3000);
  // report "ideal" preempt
  conf.setFloat(TOTAL_PREEMPTION_PER_ROUND, (float) 1.0);
  conf.setFloat(NATURAL_TERMINATION_FACTOR, (float) 1.0);
  conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
      ProportionalCapacityPreemptionPolicy.class.getCanonicalName());
  conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
  // FairScheduler doesn't support this test,
  // Set CapacityScheduler as the scheduler for this test.
  conf.set("yarn.resourcemanager.scheduler.class",
      CapacityScheduler.class.getName());

  mClock = mock(Clock.class);
  mCS = mock(CapacityScheduler.class);
  when(mCS.getResourceCalculator()).thenReturn(rc);
  lm = mock(RMNodeLabelsManager.class);
  schedConf = new CapacitySchedulerConfiguration();
  when(mCS.getConfiguration()).thenReturn(schedConf);
  rmContext = mock(RMContext.class);
  when(mCS.getRMContext()).thenReturn(rmContext);
  when(rmContext.getNodeLabelManager()).thenReturn(lm);
  mDisp = mock(EventHandler.class);
  rand = new Random();
  long seed = rand.nextLong();
  System.out.println(name.getMethodName() + " SEED: " + seed);
  rand.setSeed(seed);
  appAlloc = 0;
}
 
Example #23
Source File: TestNetworkedJob.java    From big-c with Apache License 2.0 5 votes vote down vote up
private MiniMRClientCluster createMiniClusterWithCapacityScheduler()
    throws IOException {
  Configuration conf = new Configuration();
  // Expected queue names depending on Capacity Scheduler queue naming
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      CapacityScheduler.class);
  return MiniMRClientClusterFactory.create(this.getClass(), 2, conf);
}
 
Example #24
Source File: YARNHighAvailabilityITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	storageDir = FOLDER.newFolder().getAbsolutePath();

	// startYARNWithConfig should be implemented by subclass
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #25
Source File: YARNSessionCapacitySchedulerITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team");
	YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40);
	YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler");
	startYARNWithConfig(YARN_CONFIGURATION);

	restClientExecutor = Executors.newSingleThreadExecutor();
	restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor);
}
 
Example #26
Source File: TestRMWebServicesAppsModification.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAppQueue() throws Exception {
  client().addFilter(new LoggingFilter(System.out));
  boolean isCapacityScheduler =
      rm.getResourceScheduler() instanceof CapacityScheduler;
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  String[] contentTypes =
      { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
  for (String contentType : contentTypes) {
    RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
    amNodeManager.nodeHeartbeat(true);
    ClientResponse response =
        this
          .constructWebResource("apps", app.getApplicationId().toString(),
            "queue").accept(contentType).get(ClientResponse.class);
    assertEquals(Status.OK, response.getClientResponseStatus());
    String expectedQueue = "default";
    if(!isCapacityScheduler) {
      expectedQueue = "root." + webserviceUserName;
    }
    if (contentType.equals(MediaType.APPLICATION_JSON)) {
      verifyAppQueueJson(response, expectedQueue);
    } else {
      verifyAppQueueXML(response, expectedQueue);
    }
  }
  rm.stop();
}
 
Example #27
Source File: CapacitySchedulerPlanFollower.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched, Collection<Plan> plans) {
  super.init(clock, sched, plans);
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
  if (!(sched instanceof CapacityScheduler)) {
    throw new YarnRuntimeException(
        "CapacitySchedulerPlanFollower can only work with CapacityScheduler");
  }
  this.cs = (CapacityScheduler) sched;
}
 
Example #28
Source File: SchedulerInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example #29
Source File: TestNetworkedJob.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private MiniMRClientCluster createMiniClusterWithCapacityScheduler()
    throws IOException {
  Configuration conf = new Configuration();
  // Expected queue names depending on Capacity Scheduler queue naming
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      CapacityScheduler.class);
  return MiniMRClientClusterFactory.create(this.getClass(), 2, conf);
}
 
Example #30
Source File: YARNHighAvailabilityITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	storageDir = FOLDER.newFolder().getAbsolutePath();

	// startYARNWithConfig should be implemented by subclass
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096);
	startYARNWithConfig(YARN_CONFIGURATION);
}