org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMWebServices.java    From big-c with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo = new CapacitySchedulerInfo(root);
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #2
Source File: RMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@GET
@Path("/scheduler")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public SchedulerTypeInfo getSchedulerInfo() {
  init();
  ResourceScheduler rs = rm.getResourceScheduler();
  SchedulerInfo sinfo;
  if (rs instanceof CapacityScheduler) {
    CapacityScheduler cs = (CapacityScheduler) rs;
    CSQueue root = cs.getRootQueue();
    sinfo =
        new CapacitySchedulerInfo(root, cs, new NodeLabel(
            RMNodeLabelsManager.NO_LABEL));
  } else if (rs instanceof FairScheduler) {
    FairScheduler fs = (FairScheduler) rs;
    sinfo = new FairSchedulerInfo(fs);
  } else if (rs instanceof FifoScheduler) {
    sinfo = new FifoSchedulerInfo(this.rm);
  } else {
    throw new NotFoundException("Unknown scheduler configured");
  }
  return new SchedulerTypeInfo(sinfo);
}
 
Example #3
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
  try {
    scheduler.serviceInit(conf);
    fail("Exception is expected because the min memory allocation is" +
      " larger than the max memory allocation.");
  } catch (YarnRuntimeException e) {
    // Exception is expected.
    assertTrue("The thrown exception is not the expected one.",
      e.getMessage().startsWith(
        "Invalid resource scheduler memory"));
  }
}
 
Example #4
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testConfValidation() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
  try {
    scheduler.serviceInit(conf);
    fail("Exception is expected because the min memory allocation is" +
      " larger than the max memory allocation.");
  } catch (YarnRuntimeException e) {
    // Exception is expected.
    assertTrue("The thrown exception is not the expected one.",
      e.getMessage().startsWith(
        "Invalid resource scheduler memory"));
  }
}
 
Example #5
Source File: TestRMWebServicesHttpStaticUserPermissions.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal);
  rmconf.set("yarn.resourcemanager.keytab",
      spnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #6
Source File: TestRMWebServicesDelegationTokens.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rm = new MockRM(rmconf);
  bind(ResourceManager.class).toInstance(rm);
  if (isKerberosAuth == true) {
    filter("/*").through(TestKerberosAuthFilter.class);
  } else {
    filter("/*").through(TestSimpleAuthFilter.class);
  }
  serve("/*").with(GuiceContainer.class);
}
 
Example #7
Source File: TestRMWebServicesDelegationTokens.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rm = new MockRM(rmconf);
  bind(ResourceManager.class).toInstance(rm);
  if (isKerberosAuth == true) {
    filter("/*").through(TestKerberosAuthFilter.class);
  } else {
    filter("/*").through(TestSimpleAuthFilter.class);
  }
  serve("/*").with(GuiceContainer.class);
}
 
Example #8
Source File: TestRMWebServicesHttpStaticUserPermissions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal);
  rmconf.set("yarn.resourcemanager.keytab",
      spnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #9
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #10
Source File: FifoSchedulerInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public FifoSchedulerInfo(final ResourceManager rm) {

    RMContext rmContext = rm.getRMContext();

    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    qName = fs.getQueueInfo("", false, false).getQueueName();
    QueueInfo qInfo = fs.getQueueInfo(qName, true, true);

    this.usedCapacity = qInfo.getCurrentCapacity();
    this.capacity = qInfo.getCapacity();
    this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory();
    this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory();
    this.qstate = qInfo.getQueueState();

    this.numNodes = rmContext.getRMNodes().size();
    this.usedNodeCapacity = 0;
    this.availNodeCapacity = 0;
    this.totalNodeCapacity = 0;
    this.numContainers = 0;

    for (RMNode ni : rmContext.getRMNodes().values()) {
      SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
      this.usedNodeCapacity += report.getUsedResource().getMemory();
      this.availNodeCapacity += report.getAvailableResource().getMemory();
      this.totalNodeCapacity += ni.getTotalCapability().getMemory();
      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
    }
  }
 
Example #11
Source File: SchedulerInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example #12
Source File: TestRMWebServicesDelegationTokenAuthentication.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  String httpPrefix = "hadoop.http.authentication.";
  rmconf.setStrings(httpPrefix + "type", "kerberos");
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL,
    httpSpnegoPrincipal);
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  // use any file for signature secret
  rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file",
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
    true);
  rmconf.set("hadoop.http.filter.initializers",
    AuthenticationFilterInitializer.class.getName());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.RM_KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
  rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #13
Source File: TestRMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #14
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static FifoScheduler mockFifoScheduler(RMContext rmContext)
    throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupFifoQueueConfiguration(conf);

  FifoScheduler fs = new FifoScheduler();
  fs.setConf(new YarnConfiguration());
  fs.setRMContext(rmContext);
  fs.init(conf);
  return fs;
}
 
Example #15
Source File: YARNSessionFIFOITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo");
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #16
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  MockRM rm = new MockRM(conf);
  scheduler.setRMContext(rm.getRMContext());
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rm.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1,
          Resources.createResource(1024, 4), 1, "127.0.0.1");
  scheduler.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = ApplicationId.newInstance(0, 1);
  scheduler.addApplication(appId, "queue1", "user1", false);

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  try {
    scheduler.handle(updateEvent);
  } catch (NullPointerException e) {
      Assert.fail();
  }

  ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
  scheduler.addApplicationAttempt(attId, false, false);

  rm.stop();
}
 
Example #17
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 50000)
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  conf.setQueues("default", new String[] {"default"});
  conf.setCapacity("default", 100);
  FifoScheduler fs = new FifoScheduler();
  fs.init(conf);
  fs.start();
  // mock rmContext to avoid NPE.
  RMContext context = mock(RMContext.class);
  fs.reinitialize(conf, null);
  fs.setRMContext(context);

  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  RMNode n2 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3");

  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n2));
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB());

  // reconnect n1 with downgraded memory
  n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2");
  fs.handle(new NodeRemovedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeUpdateSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB());
  fs.stop();
}
 
Example #18
Source File: TestQueueMetrics.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMetricsInitializedOnRMInit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER,
    FifoScheduler.class, ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
  checkApps(metrics, 0, 0, 0, 0, 0, 0, true);
  MetricsAsserts.assertGauge("ReservedContainers", 0, metrics);
}
 
Example #19
Source File: YARNSessionFIFOITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo");
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #20
Source File: UnitTestCluster.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception {
  yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
  yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
  MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1);
  miniYarnCluster.init(yarnConf);
  yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ;
  miniYarnCluster.start();
  return miniYarnCluster ;
}
 
Example #21
Source File: SpliceTestYarnPlatform.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
private void configForTesting(String classPathRoot) throws URISyntaxException {
    yarnSiteConfigURL = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
    if (yarnSiteConfigURL == null) {
        throw new RuntimeException("Could not find 'yarn-site.xml' file in classpath");
    } else {
        LOG.info("Found 'yarn-site.xml' at "+ yarnSiteConfigURL.toURI().toString());
    }

    conf = new YarnConfiguration();
    conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///");

    keytab = classPathRoot.substring(0, classPathRoot.lastIndexOf('/'))+"/splice.keytab";
    if (secure) {
        conf.set("hadoop.security.authentication", "kerberos");
        conf.set("yarn.resourcemanager.principal", "yarn/[email protected]");
        conf.set("yarn.resourcemanager.keytab", keytab);
        conf.set("yarn.nodemanager.principal", "yarn/[email protected]");
        conf.set("yarn.nodemanager.keytab", keytab);
    }
    conf.setDouble("yarn.nodemanager.resource.io-spindles",2.0);
    conf.set("fs.default.name", "file:///");
    conf.set("yarn.nodemanager.container-executor.class","org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor");
    System.setProperty("zookeeper.sasl.client", "false");
    System.setProperty("zookeeper.sasl.serverconfig", "fake");

    conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, DEFAULT_HEARTBEAT_INTERVAL);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
    conf.set("yarn.application.classpath", new File(yarnSiteConfigURL.getPath()).getParent());
}
 
Example #22
Source File: TestQueueMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMetricsInitializedOnRMInit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER,
    FifoScheduler.class, ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
  checkApps(metrics, 0, 0, 0, 0, 0, 0, true);
  MetricsAsserts.assertGauge("ReservedContainers", 0, metrics);
}
 
Example #23
Source File: YARNSessionFIFOITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() {
	YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
	YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768);
	YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
	YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo");
	startYARNWithConfig(YARN_CONFIGURATION);
}
 
Example #24
Source File: FifoSchedulerInfo.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public FifoSchedulerInfo(final ResourceManager rm) {

    RMContext rmContext = rm.getRMContext();

    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    qName = fs.getQueueInfo("", false, false).getQueueName();
    QueueInfo qInfo = fs.getQueueInfo(qName, true, true);

    this.usedCapacity = qInfo.getCurrentCapacity();
    this.capacity = qInfo.getCapacity();
    this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory();
    this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory();
    this.qstate = qInfo.getQueueState();

    this.numNodes = rmContext.getRMNodes().size();
    this.usedNodeCapacity = 0;
    this.availNodeCapacity = 0;
    this.totalNodeCapacity = 0;
    this.numContainers = 0;

    for (RMNode ni : rmContext.getRMNodes().values()) {
      SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
      this.usedNodeCapacity += report.getUsedResource().getMemory();
      this.availNodeCapacity += report.getAvailableResource().getMemory();
      this.totalNodeCapacity += ni.getTotalCapability().getMemory();
      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
    }
  }
 
Example #25
Source File: SchedulerInfo.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 
Example #26
Source File: TestRMWebServicesDelegationTokenAuthentication.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  String httpPrefix = "hadoop.http.authentication.";
  rmconf.setStrings(httpPrefix + "type", "kerberos");
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL,
    httpSpnegoPrincipal);
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  // use any file for signature secret
  rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file",
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
    true);
  rmconf.set("hadoop.http.filter.initializers",
    AuthenticationFilterInitializer.class.getName());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.RM_KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
  rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 
Example #27
Source File: TestRMWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #28
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static FifoScheduler mockFifoScheduler(RMContext rmContext)
    throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupFifoQueueConfiguration(conf);

  FifoScheduler fs = new FifoScheduler();
  fs.setConf(new YarnConfiguration());
  fs.setRMContext(rmContext);
  fs.init(conf);
  return fs;
}
 
Example #29
Source File: TestRMWebServicesApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 
Example #30
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  MockRM rm = new MockRM(conf);
  scheduler.setRMContext(rm.getRMContext());
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rm.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1,
          Resources.createResource(1024, 4, 4), 1, "127.0.0.1");
  scheduler.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = ApplicationId.newInstance(0, 1);
  scheduler.addApplication(appId, "queue1", "user1", false);

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  try {
    scheduler.handle(updateEvent);
  } catch (NullPointerException e) {
      Assert.fail();
  }

  ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
  scheduler.addApplicationAttempt(attId, false, false);

  rm.stop();
}