org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RMWebServices.java From big-c with Apache License 2.0 | 6 votes |
@GET @Path("/scheduler") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public SchedulerTypeInfo getSchedulerInfo() { init(); ResourceScheduler rs = rm.getResourceScheduler(); SchedulerInfo sinfo; if (rs instanceof CapacityScheduler) { CapacityScheduler cs = (CapacityScheduler) rs; CSQueue root = cs.getRootQueue(); sinfo = new CapacitySchedulerInfo(root); } else if (rs instanceof FairScheduler) { FairScheduler fs = (FairScheduler) rs; sinfo = new FairSchedulerInfo(fs); } else if (rs instanceof FifoScheduler) { sinfo = new FifoSchedulerInfo(this.rm); } else { throw new NotFoundException("Unknown scheduler configured"); } return new SchedulerTypeInfo(sinfo); }
Example #2
Source File: RMWebServices.java From hadoop with Apache License 2.0 | 6 votes |
@GET @Path("/scheduler") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public SchedulerTypeInfo getSchedulerInfo() { init(); ResourceScheduler rs = rm.getResourceScheduler(); SchedulerInfo sinfo; if (rs instanceof CapacityScheduler) { CapacityScheduler cs = (CapacityScheduler) rs; CSQueue root = cs.getRootQueue(); sinfo = new CapacitySchedulerInfo(root, cs, new NodeLabel( RMNodeLabelsManager.NO_LABEL)); } else if (rs instanceof FairScheduler) { FairScheduler fs = (FairScheduler) rs; sinfo = new FairSchedulerInfo(fs); } else if (rs instanceof FifoScheduler) { sinfo = new FifoSchedulerInfo(this.rm); } else { throw new NotFoundException("Unknown scheduler configured"); } return new SchedulerTypeInfo(sinfo); }
Example #3
Source File: TestFifoScheduler.java From hadoop with Apache License 2.0 | 6 votes |
@Test (timeout = 30000) public void testConfValidation() throws Exception { FifoScheduler scheduler = new FifoScheduler(); Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { scheduler.serviceInit(conf); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { // Exception is expected. assertTrue("The thrown exception is not the expected one.", e.getMessage().startsWith( "Invalid resource scheduler memory")); } }
Example #4
Source File: TestFifoScheduler.java From big-c with Apache License 2.0 | 6 votes |
@Test (timeout = 30000) public void testConfValidation() throws Exception { FifoScheduler scheduler = new FifoScheduler(); Configuration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 2048); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024); try { scheduler.serviceInit(conf); fail("Exception is expected because the min memory allocation is" + " larger than the max memory allocation."); } catch (YarnRuntimeException e) { // Exception is expected. assertTrue("The thrown exception is not the expected one.", e.getMessage().startsWith( "Invalid resource scheduler memory")); } }
Example #5
Source File: TestRMWebServicesHttpStaticUserPermissions.java From big-c with Apache License 2.0 | 6 votes |
private static void setupAndStartRM() throws Exception { Configuration rmconf = new Configuration(); rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal); rmconf.set("yarn.resourcemanager.keytab", spnegoKeytabFile.getAbsolutePath()); rmconf.setBoolean("mockrm.webapp.enabled", true); UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start(); }
Example #6
Source File: TestRMWebServicesDelegationTokens.java From hadoop with Apache License 2.0 | 6 votes |
@Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration rmconf = new Configuration(); rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); rm = new MockRM(rmconf); bind(ResourceManager.class).toInstance(rm); if (isKerberosAuth == true) { filter("/*").through(TestKerberosAuthFilter.class); } else { filter("/*").through(TestSimpleAuthFilter.class); } serve("/*").with(GuiceContainer.class); }
Example #7
Source File: TestRMWebServicesDelegationTokens.java From big-c with Apache License 2.0 | 6 votes |
@Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration rmconf = new Configuration(); rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); rm = new MockRM(rmconf); bind(ResourceManager.class).toInstance(rm); if (isKerberosAuth == true) { filter("/*").through(TestKerberosAuthFilter.class); } else { filter("/*").through(TestSimpleAuthFilter.class); } serve("/*").with(GuiceContainer.class); }
Example #8
Source File: TestRMWebServicesHttpStaticUserPermissions.java From hadoop with Apache License 2.0 | 6 votes |
private static void setupAndStartRM() throws Exception { Configuration rmconf = new Configuration(); rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); rmconf.set("yarn.resourcemanager.principal", spnegoPrincipal); rmconf.set("yarn.resourcemanager.keytab", spnegoKeytabFile.getAbsolutePath()); rmconf.setBoolean("mockrm.webapp.enabled", true); UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start(); }
Example #9
Source File: TestRMWebServicesApps.java From big-c with Apache License 2.0 | 5 votes |
@Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration conf = new Configuration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); }
Example #10
Source File: FifoSchedulerInfo.java From big-c with Apache License 2.0 | 5 votes |
public FifoSchedulerInfo(final ResourceManager rm) { RMContext rmContext = rm.getRMContext(); FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler(); qName = fs.getQueueInfo("", false, false).getQueueName(); QueueInfo qInfo = fs.getQueueInfo(qName, true, true); this.usedCapacity = qInfo.getCurrentCapacity(); this.capacity = qInfo.getCapacity(); this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory(); this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory(); this.qstate = qInfo.getQueueState(); this.numNodes = rmContext.getRMNodes().size(); this.usedNodeCapacity = 0; this.availNodeCapacity = 0; this.totalNodeCapacity = 0; this.numContainers = 0; for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); this.usedNodeCapacity += report.getUsedResource().getMemory(); this.availNodeCapacity += report.getAvailableResource().getMemory(); this.totalNodeCapacity += ni.getTotalCapability().getMemory(); this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); } }
Example #11
Source File: SchedulerInfo.java From big-c with Apache License 2.0 | 5 votes |
public SchedulerInfo(final ResourceManager rm) { ResourceScheduler rs = rm.getResourceScheduler(); if (rs instanceof CapacityScheduler) { this.schedulerName = "Capacity Scheduler"; } else if (rs instanceof FairScheduler) { this.schedulerName = "Fair Scheduler"; } else if (rs instanceof FifoScheduler) { this.schedulerName = "Fifo Scheduler"; } this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability()); this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability()); this.schedulingResourceTypes = rs.getSchedulingResourceTypes(); }
Example #12
Source File: TestRMWebServicesDelegationTokenAuthentication.java From big-c with Apache License 2.0 | 5 votes |
private static void setupAndStartRM() throws Exception { Configuration rmconf = new Configuration(); rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); String httpPrefix = "hadoop.http.authentication."; rmconf.setStrings(httpPrefix + "type", "kerberos"); rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); // use any file for signature secret rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file", httpSpnegoKeytabFile.getAbsolutePath()); rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, true); rmconf.set("hadoop.http.filter.initializers", AuthenticationFilterInitializer.class.getName()); rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal); rmconf.set(YarnConfiguration.RM_KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal); rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.setBoolean("mockrm.webapp.enabled", true); rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*"); rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*"); UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start(); }
Example #13
Source File: TestRMWebServices.java From big-c with Apache License 2.0 | 5 votes |
@Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); }
Example #14
Source File: TestRMWebApp.java From big-c with Apache License 2.0 | 5 votes |
public static FifoScheduler mockFifoScheduler(RMContext rmContext) throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupFifoQueueConfiguration(conf); FifoScheduler fs = new FifoScheduler(); fs.setConf(new YarnConfiguration()); fs.setRMContext(rmContext); fs.init(conf); return fs; }
Example #15
Source File: YARNSessionFIFOITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
Example #16
Source File: TestFifoScheduler.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testNodeUpdateBeforeAppAttemptInit() throws Exception { FifoScheduler scheduler = new FifoScheduler(); MockRM rm = new MockRM(conf); scheduler.setRMContext(rm.getRMContext()); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, rm.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); scheduler.handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.newInstance(0, 1); scheduler.addApplication(appId, "queue1", "user1", false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try { scheduler.handle(updateEvent); } catch (NullPointerException e) { Assert.fail(); } ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1); scheduler.addApplicationAttempt(attId, false, false); rm.stop(); }
Example #17
Source File: TestFifoScheduler.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout = 50000) public void testReconnectedNode() throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); conf.setQueues("default", new String[] {"default"}); conf.setCapacity("default", 100); FifoScheduler fs = new FifoScheduler(); fs.init(conf); fs.start(); // mock rmContext to avoid NPE. RMContext context = mock(RMContext.class); fs.reinitialize(conf, null); fs.setRMContext(context); RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2"); RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3"); fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n2)); fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB()); // reconnect n1 with downgraded memory n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2"); fs.handle(new NodeRemovedSchedulerEvent(n1)); fs.handle(new NodeAddedSchedulerEvent(n1)); fs.handle(new NodeUpdateSchedulerEvent(n1)); Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB()); fs.stop(); }
Example #18
Source File: TestQueueMetrics.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testMetricsInitializedOnRMInit() { YarnConfiguration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MockRM rm = new MockRM(conf); QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics(); checkApps(metrics, 0, 0, 0, 0, 0, 0, true); MetricsAsserts.assertGauge("ReservedContainers", 0, metrics); }
Example #19
Source File: YARNSessionFIFOITCase.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
Example #20
Source File: UnitTestCluster.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception { yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1); miniYarnCluster.init(yarnConf); yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ; miniYarnCluster.start(); return miniYarnCluster ; }
Example #21
Source File: SpliceTestYarnPlatform.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private void configForTesting(String classPathRoot) throws URISyntaxException { yarnSiteConfigURL = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml"); if (yarnSiteConfigURL == null) { throw new RuntimeException("Could not find 'yarn-site.xml' file in classpath"); } else { LOG.info("Found 'yarn-site.xml' at "+ yarnSiteConfigURL.toURI().toString()); } conf = new YarnConfiguration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); keytab = classPathRoot.substring(0, classPathRoot.lastIndexOf('/'))+"/splice.keytab"; if (secure) { conf.set("hadoop.security.authentication", "kerberos"); conf.set("yarn.resourcemanager.principal", "yarn/[email protected]"); conf.set("yarn.resourcemanager.keytab", keytab); conf.set("yarn.nodemanager.principal", "yarn/[email protected]"); conf.set("yarn.nodemanager.keytab", keytab); } conf.setDouble("yarn.nodemanager.resource.io-spindles",2.0); conf.set("fs.default.name", "file:///"); conf.set("yarn.nodemanager.container-executor.class","org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor"); System.setProperty("zookeeper.sasl.client", "false"); System.setProperty("zookeeper.sasl.serverconfig", "fake"); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, DEFAULT_HEARTBEAT_INTERVAL); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); conf.set("yarn.application.classpath", new File(yarnSiteConfigURL.getPath()).getParent()); }
Example #22
Source File: TestQueueMetrics.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testMetricsInitializedOnRMInit() { YarnConfiguration conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MockRM rm = new MockRM(conf); QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics(); checkApps(metrics, 0, 0, 0, 0, 0, 0, true); MetricsAsserts.assertGauge("ReservedContainers", 0, metrics); }
Example #23
Source File: YARNSessionFIFOITCase.java From flink with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
Example #24
Source File: FifoSchedulerInfo.java From hadoop with Apache License 2.0 | 5 votes |
public FifoSchedulerInfo(final ResourceManager rm) { RMContext rmContext = rm.getRMContext(); FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler(); qName = fs.getQueueInfo("", false, false).getQueueName(); QueueInfo qInfo = fs.getQueueInfo(qName, true, true); this.usedCapacity = qInfo.getCurrentCapacity(); this.capacity = qInfo.getCapacity(); this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory(); this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory(); this.qstate = qInfo.getQueueState(); this.numNodes = rmContext.getRMNodes().size(); this.usedNodeCapacity = 0; this.availNodeCapacity = 0; this.totalNodeCapacity = 0; this.numContainers = 0; for (RMNode ni : rmContext.getRMNodes().values()) { SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID()); this.usedNodeCapacity += report.getUsedResource().getMemory(); this.availNodeCapacity += report.getAvailableResource().getMemory(); this.totalNodeCapacity += ni.getTotalCapability().getMemory(); this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers(); } }
Example #25
Source File: SchedulerInfo.java From hadoop with Apache License 2.0 | 5 votes |
public SchedulerInfo(final ResourceManager rm) { ResourceScheduler rs = rm.getResourceScheduler(); if (rs instanceof CapacityScheduler) { this.schedulerName = "Capacity Scheduler"; } else if (rs instanceof FairScheduler) { this.schedulerName = "Fair Scheduler"; } else if (rs instanceof FifoScheduler) { this.schedulerName = "Fifo Scheduler"; } this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability()); this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability()); this.schedulingResourceTypes = rs.getSchedulingResourceTypes(); }
Example #26
Source File: TestRMWebServicesDelegationTokenAuthentication.java From hadoop with Apache License 2.0 | 5 votes |
private static void setupAndStartRM() throws Exception { Configuration rmconf = new Configuration(); rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); String httpPrefix = "hadoop.http.authentication."; rmconf.setStrings(httpPrefix + "type", "kerberos"); rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL, httpSpnegoPrincipal); rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); // use any file for signature secret rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file", httpSpnegoKeytabFile.getAbsolutePath()); rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos"); rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER, true); rmconf.set("hadoop.http.filter.initializers", AuthenticationFilterInitializer.class.getName()); rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal); rmconf.set(YarnConfiguration.RM_KEYTAB, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY, httpSpnegoPrincipal); rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY, httpSpnegoKeytabFile.getAbsolutePath()); rmconf.setBoolean("mockrm.webapp.enabled", true); rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*"); rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*"); UserGroupInformation.setConfiguration(rmconf); rm = new MockRM(rmconf); rm.start(); }
Example #27
Source File: TestRMWebServices.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration conf = new Configuration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); }
Example #28
Source File: TestRMWebApp.java From hadoop with Apache License 2.0 | 5 votes |
public static FifoScheduler mockFifoScheduler(RMContext rmContext) throws Exception { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); setupFifoQueueConfiguration(conf); FifoScheduler fs = new FifoScheduler(); fs.setConf(new YarnConfiguration()); fs.setRMContext(rmContext); fs.init(conf); return fs; }
Example #29
Source File: TestRMWebServicesApps.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected void configureServlets() { bind(JAXBContextResolver.class); bind(RMWebServices.class); bind(GenericExceptionHandler.class); Configuration conf = new Configuration(); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); rm = new MockRM(conf); bind(ResourceManager.class).toInstance(rm); serve("/*").with(GuiceContainer.class); }
Example #30
Source File: TestFifoScheduler.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testNodeUpdateBeforeAppAttemptInit() throws Exception { FifoScheduler scheduler = new FifoScheduler(); MockRM rm = new MockRM(conf); scheduler.setRMContext(rm.getRMContext()); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, rm.getRMContext()); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4, 4), 1, "127.0.0.1"); scheduler.handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.newInstance(0, 1); scheduler.addApplication(appId, "queue1", "user1", false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try { scheduler.handle(updateEvent); } catch (NullPointerException e) { Assert.fail(); } ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1); scheduler.addApplicationAttempt(attId, false, false); rm.stop(); }