org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #2
Source File: ReservationSystemTestUtil.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static FairScheduler setupFairScheduler(
    ReservationSystemTestUtil testUtil,
    RMContext rmContext, Configuration conf, int numContainers) throws
    IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource = testUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 
Example #3
Source File: RMWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo = new CapacitySchedulerInfo(root);
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #4
Source File: ReservationSystemTestUtil.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static FairScheduler setupFairScheduler(
    ReservationSystemTestUtil testUtil,
    RMContext rmContext, Configuration conf, int numContainers) throws
    IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource = testUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 
Example #5
Source File: TestRMWebServicesAppsModification.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"test\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch(IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 
Example #6
Source File: TestRMWebServicesAppsModification.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"test\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch(IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 
Example #7
Source File: FairSchedulerAppsBlock.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 
Example #8
Source File: ParameterizedSchedulerTestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void configureFairScheduler(YarnConfiguration conf) throws IOException {
  // Disable queueMaxAMShare limitation for fair scheduler
  PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
  out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
  out.println("<queue name=\"root\">");
  out.println("  <schedulingPolicy>drf</schedulingPolicy>");
  out.println("  <weight>1.0</weight>");
  out.println("  <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
  out.println("  <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
  out.println("  <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
}
 
Example #9
Source File: ParameterizedSchedulerTestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void configureFairScheduler(YarnConfiguration conf) throws IOException {
  // Disable queueMaxAMShare limitation for fair scheduler
  PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
  out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
  out.println("<queue name=\"root\">");
  out.println("  <schedulingPolicy>drf</schedulingPolicy>");
  out.println("  <weight>1.0</weight>");
  out.println("  <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
  out.println("  <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
  out.println("  <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
}
 
Example #10
Source File: RmController.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #11
Source File: RmController.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 
Example #12
Source File: FairSchedulerAppsBlock.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 
Example #13
Source File: SchedulerInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example #14
Source File: TestRMWebServicesFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #15
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
 
Example #16
Source File: FairReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is fair scheduler
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof FairScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + FairScheduler.class.getCanonicalName());
  }
  fairScheduler = (FairScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}
 
Example #17
Source File: AbstractReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
private String getDefaultPlanFollower() {
  // currently only capacity scheduler is supported
  if (scheduler instanceof CapacityScheduler) {
    return CapacitySchedulerPlanFollower.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairSchedulerPlanFollower.class.getName();
  }
  return null;
}
 
Example #18
Source File: AbstractReservationSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 
Example #19
Source File: FairSchedulerPlanFollower.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched,
    Collection<Plan> plans) {
  super.init(clock, sched, plans);
  fs = (FairScheduler)sched;
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
}
 
Example #20
Source File: TestRMDelegationTokens.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  ExitUtil.disableSystemExit();
  conf = new YarnConfiguration();
  UserGroupInformation.setConfiguration(conf);
  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
 
Example #21
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void checkFSQueue(ResourceManager rm,
    SchedulerApplication  schedulerApp, Resource usedResources,
    Resource availableResources) throws Exception {
  // waiting for RM's scheduling apps
  int retry = 0;
  Resource assumedFairShare = Resource.newInstance(8192, 8);
  while (true) {
    Thread.sleep(100);
    if (assumedFairShare.equals(((FairScheduler)rm.getResourceScheduler())
        .getQueueManager().getRootQueue().getFairShare())) {
      break;
    }
    retry++;
    if (retry > 30) {
      Assert.fail("Apps are not scheduled within assumed timeout");
    }
  }

  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  FSParentQueue root = scheduler.getQueueManager().getRootQueue();
  // ************ check cluster used Resources ********
  assertTrue(root.getPolicy() instanceof DominantResourceFairnessPolicy);
  assertEquals(usedResources,root.getResourceUsage());

  // ************ check app headroom ****************
  FSAppAttempt schedulerAttempt =
      (FSAppAttempt) schedulerApp.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************ check queue metrics ****************
  QueueMetrics queueMetrics = scheduler.getRootQueueMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResources.getMemory(),
      usedResources.getVirtualCores());
}
 
Example #22
Source File: TestFairReservationSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example #23
Source File: TestFairSchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected Configuration createConfiguration() {
  Configuration conf = testHelper.createConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  return conf;
}
 
Example #24
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static FairScheduler mockFairScheduler() throws IOException {
  FairScheduler fs = new FairScheduler();
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  fs.setRMContext(new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null));
  fs.init(conf);
  return fs;
}
 
Example #25
Source File: TestRMWebServicesFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #26
Source File: TestRMDelegationTokens.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  ExitUtil.disableSystemExit();
  conf = new YarnConfiguration();
  UserGroupInformation.setConfiguration(conf);
  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
 
Example #27
Source File: FairSchedulerPlanFollower.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void init(Clock clock, ResourceScheduler sched,
    Collection<Plan> plans) {
  super.init(clock, sched, plans);
  fs = (FairScheduler)sched;
  LOG.info("Initializing Plan Follower Policy:"
      + this.getClass().getCanonicalName());
}
 
Example #28
Source File: AbstractReservationSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Get the default reservation system corresponding to the scheduler
 * 
 * @param scheduler the scheduler for which the reservation system is required
 */
public static String getDefaultReservationSystem(ResourceScheduler scheduler) {
  if (scheduler instanceof CapacityScheduler) {
    return CapacityReservationSystem.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairReservationSystem.class.getName();
  }
  return null;
}
 
Example #29
Source File: AbstractReservationSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private String getDefaultPlanFollower() {
  // currently only capacity scheduler is supported
  if (scheduler instanceof CapacityScheduler) {
    return CapacitySchedulerPlanFollower.class.getName();
  } else if (scheduler instanceof FairScheduler) {
    return FairSchedulerPlanFollower.class.getName();
  }
  return null;
}
 
Example #30
Source File: FairReservationSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void reinitialize(Configuration conf, RMContext rmContext)
    throws YarnException {
  // Validate if the scheduler is fair scheduler
  ResourceScheduler scheduler = rmContext.getScheduler();
  if (!(scheduler instanceof FairScheduler)) {
    throw new YarnRuntimeException("Class "
        + scheduler.getClass().getCanonicalName() + " not instance of "
        + FairScheduler.class.getCanonicalName());
  }
  fairScheduler = (FairScheduler) scheduler;
  this.conf = conf;
  super.reinitialize(conf, rmContext);
}