org.apache.hadoop.yarn.server.MiniYARNCluster Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.MiniYARNCluster.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRMFailover.java From hadoop with Apache License 2.0 | 6 votes |
@Before public void setup() throws IOException { fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0); conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE); setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE); conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1); }
Example #2
Source File: HadoopTestCluster.java From yarn-proto with Apache License 2.0 | 6 votes |
public void start() throws IOException { log.info("Checking if cluster=" + clusterName + " needs to be started"); synchronized (this.startupShutdownMonitor) { if (started) { return; } log.info("Starting cluster=" + clusterName); configuration = new YarnConfiguration(); //configuration.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true); configuration.setBoolean("yarn.is.minicluster", true); configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs"); dfsCluster = new MiniDFSCluster.Builder(configuration). numDataNodes(nodes). build(); yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1); yarnCluster.init(configuration); yarnCluster.start(); log.info("Started cluster=" + clusterName); started = true; } }
Example #3
Source File: MiniTajoYarnCluster.java From incubator-tajo with Apache License 2.0 | 6 votes |
@Override public synchronized void start() { try { if (!getConfig().getBoolean( JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) { // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname() + ":0"); getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, MiniYARNCluster.getHostname() + ":0"); } super.start(); } catch (Throwable t) { throw new YarnRuntimeException(t); } LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)); }
Example #4
Source File: TestRMFailover.java From big-c with Apache License 2.0 | 6 votes |
@Before public void setup() throws IOException { fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0); conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE); setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE); conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true); cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1); }
Example #5
Source File: UnitTestCluster.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception { yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64); yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1); miniYarnCluster.init(yarnConf); yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ; miniYarnCluster.start(); return miniYarnCluster ; }
Example #6
Source File: YarnTestBinding.java From mr4c with Apache License 2.0 | 5 votes |
private void startMrCluster() throws IOException { Configuration conf = new JobConf(); FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri()); conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); String addr = MiniYARNCluster.getHostname() + ":0"; conf.set(YarnConfiguration.RM_ADDRESS, addr); conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr); m_mrCluster = MiniMRClientClusterFactory.create( HadoopTestUtils.class, "MR4CTests", 1, // num node managers conf ); // make sure startup is finished for ( int i=0; i<60; i++ ) { String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); if ( newAddr.equals(addr) ) { s_log.warn("MiniYARNCluster startup not complete"); try { Thread.sleep(1000); } catch (InterruptedException ie) { throw new IOException(ie); } } else { s_log.info("MiniYARNCluster now available at {}", newAddr); return; } } throw new IOException("MiniYARNCluster taking too long to startup"); }
Example #7
Source File: YarnLocalCluster.java From hadoop-mini-clusters with Apache License 2.0 | 5 votes |
@Override public void start() throws Exception { LOG.info("YARN: Starting MiniYarnCluster"); configure(); miniYARNCluster = new MiniYARNCluster(testName, numResourceManagers, numNodeManagers, numLocalDirs, numLogDirs, enableHa); miniYARNCluster.serviceInit(configuration); miniYARNCluster.init(configuration); miniYARNCluster.start(); }
Example #8
Source File: TestAMRMClient.java From big-c with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws Exception { // start minicluster conf = new YarnConfiguration(); conf.setLong( YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec); conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); // start rm client yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // get node info nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); priority = Priority.newInstance(1); priority2 = Priority.newInstance(2); capability = Resource.newInstance(1024, 1); node = nodeReports.get(0).getNodeId().getHost(); rack = nodeReports.get(0).getRackName(); nodes = new String[]{ node }; racks = new String[]{ rack }; }
Example #9
Source File: TestAMRMClient.java From hadoop with Apache License 2.0 | 5 votes |
@BeforeClass public static void setup() throws Exception { // start minicluster conf = new YarnConfiguration(); conf.setLong( YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, rolling_interval_sec); conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms); conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); // start rm client yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // get node info nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); priority = Priority.newInstance(1); priority2 = Priority.newInstance(2); capability = Resource.newInstance(1024, 1, 1); node = nodeReports.get(0).getNodeId().getHost(); rack = nodeReports.get(0).getRackName(); nodes = new String[]{ node }; racks = new String[]{ rack }; }
Example #10
Source File: YarnComponent.java From metron with Apache License 2.0 | 4 votes |
public MiniYARNCluster getYARNCluster(){ return yarnCluster; }
Example #11
Source File: MiniAthenaXCluster.java From AthenaX with Apache License 2.0 | 4 votes |
public MiniAthenaXCluster(String name) { this.yarnCluster = new MiniYARNCluster(name, 1, 1, 1, 1); this.workDir = new File("target", name); }
Example #12
Source File: UnitTestCluster.java From Scribengin with GNU Affero General Public License v3.0 | 4 votes |
protected MiniYARNCluster createMiniYARNCluster(int numOfNodeManagers) throws Exception { return createMiniYARNCluster(new YarnConfiguration(), numOfNodeManagers) ; }
Example #13
Source File: TestRMFailover.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testRMWebAppRedirect() throws YarnException, InterruptedException, IOException { cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1); conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster.init(conf); cluster.start(); getAdminService(0).transitionToActive(req); String rm1Url = "http://0.0.0.0:18088"; String rm2Url = "http://0.0.0.0:28088"; String redirectURL = getRedirectURL(rm2Url); // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url assertEquals(redirectURL,rm1Url+"/"); redirectURL = getRedirectURL(rm2Url + "/metrics"); assertEquals(redirectURL,rm1Url + "/metrics"); redirectURL = getRedirectURL(rm2Url + "/jmx"); assertEquals(redirectURL,rm1Url + "/jmx"); // standby RM links /conf, /stacks, /logLevel, /static, /logs, // /cluster/cluster as well as webService // /ws/v1/cluster/info should not be redirected to active RM redirectURL = getRedirectURL(rm2Url + "/cluster/cluster"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/conf"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/stacks"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/logLevel"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/static"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/logs"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps"); assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps"); redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId); assertNull(redirectURL); }
Example #14
Source File: GobblinYarnAppLauncherTest.java From incubator-gobblin with Apache License 2.0 | 4 votes |
@BeforeClass public void setUp() throws Exception { // Set java home in environment since it isn't set on some systems String javaHome = System.getProperty("java.home"); setEnv("JAVA_HOME", javaHome); final YarnConfiguration clusterConf = new YarnConfiguration(); clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000"); MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1)); miniYARNCluster.init(clusterConf); miniYARNCluster.start(); // YARN client should not be started before the Resource Manager is up AssertWithBackoff.create().logger(LOG).timeoutMs(10000) .assertTrue(new Predicate<Void>() { @Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM"); this.yarnClient = this.closer.register(YarnClient.createYarnClient()); this.yarnClient.init(clusterConf); this.yarnClient.start(); // Use a random ZK port TestingServer testingZKServer = this.closer.register(new TestingServer(-1)); LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString()); // the zk port is dynamically configured try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) { File dir = new File("target/dummydir"); // dummy directory specified in configuration dir.mkdir(); pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\""); pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\""); } // YARN config is dynamic and needs to be passed to other processes try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) { clusterConf.writeXml(os); } this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer); URL url = GobblinYarnAppLauncherTest.class.getClassLoader() .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); this.config = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .resolve(); String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); this.helixManager = HelixManagerFactory.getZKHelixManager( this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, zkConnectionString); this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf); this.configManagedHelix = ConfigFactory.parseURL(url) .withValue("gobblin.cluster.zk.connection.string", ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString())) .withValue(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY, ConfigValueFactory.fromAnyRef(MANAGED_HELIX_CLUSTER_NAME)) .withValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, ConfigValueFactory.fromAnyRef(TEST_HELIX_INSTANCE_NAME_MANAGED)) .withValue(GobblinClusterConfigurationKeys.IS_HELIX_CLUSTER_MANAGED, ConfigValueFactory.fromAnyRef("true")) .resolve(); this.helixManagerManagedHelix = HelixManagerFactory.getZKHelixManager( this.configManagedHelix.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TEST_HELIX_INSTANCE_NAME_MANAGED, InstanceType.PARTICIPANT, zkConnectionString); this.gobblinYarnAppLauncherManagedHelix = new GobblinYarnAppLauncher(this.configManagedHelix, clusterConf); }
Example #15
Source File: YarnServiceTestWithExpiration.java From incubator-gobblin with Apache License 2.0 | 4 votes |
@BeforeClass public void setUp() throws Exception { // Set java home in environment since it isn't set on some systems String javaHome = System.getProperty("java.home"); setEnv("JAVA_HOME", javaHome); this.clusterConf = new YarnConfiguration(); this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100"); this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000"); this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000"); this.clusterConf.set(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, "1000"); this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1)); this.yarnCluster.init(this.clusterConf); this.yarnCluster.start(); // YARN client should not be started before the Resource Manager is up AssertWithBackoff.create().logger(LOG).timeoutMs(10000) .assertTrue(new Predicate<Void>() { @Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM"); this.yarnClient = this.closer.register(YarnClient.createYarnClient()); this.yarnClient.init(this.clusterConf); this.yarnClient.start(); URL url = YarnServiceTest.class.getClassLoader() .getResource(YarnServiceTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); this.config = ConfigFactory.parseURL(url).resolve(); // Start a dummy application manager so that the YarnService can use the AM-RM token. startApp(); // create and start the test yarn service this.expiredYarnService = new TestExpiredYarnService(this.config, "testApp", "appId", this.clusterConf, FileSystem.getLocal(new Configuration()), this.eventBus); this.expiredYarnService.startUp(); }
Example #16
Source File: YarnServiceTest.java From incubator-gobblin with Apache License 2.0 | 4 votes |
@BeforeClass public void setUp() throws Exception { // Set java home in environment since it isn't set on some systems String javaHome = System.getProperty("java.home"); setEnv("JAVA_HOME", javaHome); this.clusterConf = new YarnConfiguration(); this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100"); this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000"); this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000"); this.yarnCluster = this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1, 1)); this.yarnCluster.init(this.clusterConf); this.yarnCluster.start(); // YARN client should not be started before the Resource Manager is up AssertWithBackoff.create().logger(LOG).timeoutMs(10000) .assertTrue(new Predicate<Void>() { @Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM"); this.yarnClient = this.closer.register(YarnClient.createYarnClient()); this.yarnClient.init(this.clusterConf); this.yarnClient.start(); URL url = YarnServiceTest.class.getClassLoader() .getResource(YarnServiceTest.class.getSimpleName() + ".conf"); Assert.assertNotNull(url, "Could not find resource " + url); this.config = ConfigFactory.parseURL(url).resolve(); // Start a dummy application manager so that the YarnService can use the AM-RM token. startApp(); // create and start the test yarn service this.yarnService = new TestYarnService(this.config, "testApp", "appId", this.clusterConf, FileSystem.getLocal(new Configuration()), this.eventBus); this.yarnService.startUp(); }
Example #17
Source File: EmbeddedGobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0 | 4 votes |
private static void setup(String[] args) throws Exception { // Parsing zk-string Preconditions.checkArgument(args.length == 1); zkString = args[0]; // Initialize necessary external components: Yarn and Helix Closer closer = Closer.create(); // Set java home in environment since it isn't set on some systems String javaHome = System.getProperty("java.home"); setEnv("JAVA_HOME", javaHome); final YarnConfiguration clusterConf = new YarnConfiguration(); clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000"); clusterConf.set("yarn.nodemanager.resource.memory-mb", "512"); clusterConf.set("yarn.scheduler.maximum-allocation-mb", "1024"); MiniYARNCluster miniYARNCluster = closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1)); miniYARNCluster.init(clusterConf); miniYARNCluster.start(); // YARN client should not be started before the Resource Manager is up AssertWithBackoff.create().logger(log).timeoutMs(10000).assertTrue(new Predicate<Void>() { @Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM"); try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH, "UTF-8")) { File dir = new File("target/dummydir"); // dummy directory specified in configuration if (!dir.mkdir()) { log.error("The dummy folder's creation is not successful"); } dir.deleteOnExit(); pw.println("gobblin.cluster.zk.connection.string=\"" + zkString + "\""); pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\""); } // YARN config is dynamic and needs to be passed to other processes try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) { clusterConf.writeXml(os); } /** Have to pass the same yarn-site.xml to the GobblinYarnAppLauncher to initialize Yarn Client. */ fileAddress = new File(YARN_SITE_XML_PATH).getAbsolutePath(); }
Example #18
Source File: TestNMClient.java From hadoop with Apache License 2.0 | 4 votes |
@Before public void setup() throws YarnException, IOException { // start minicluster conf = new YarnConfiguration(); yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); assertNotNull(yarnCluster); assertEquals(STATE.STARTED, yarnCluster.getServiceState()); // start rm client yarnClient = (YarnClientImpl) YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); assertNotNull(yarnClient); assertEquals(STATE.STARTED, yarnClient.getServiceState()); // get node info nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); // submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Priority.newInstance(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records .newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); // unmanaged AM appContext.setUnmanagedAM(true); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records .newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start int iterationsLeft = 30; RMAppAttempt appAttempt = null; while (iterationsLeft > 0) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } sleep(1000); --iterationsLeft; } if (iterationsLeft == 0) { fail("Application hasn't bee started"); } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser(UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); //creating an instance NMTokenCase nmTokenCache = new NMTokenCache(); // start am rm client rmClient = (AMRMClientImpl<ContainerRequest>) AMRMClient .<ContainerRequest> createAMRMClient(); //setting an instance NMTokenCase rmClient.setNMTokenCache(nmTokenCache); rmClient.init(conf); rmClient.start(); assertNotNull(rmClient); assertEquals(STATE.STARTED, rmClient.getServiceState()); // start am nm client nmClient = (NMClientImpl) NMClient.createNMClient(); //propagating the AMRMClient NMTokenCache instance nmClient.setNMTokenCache(rmClient.getNMTokenCache()); nmClient.init(conf); nmClient.start(); assertNotNull(nmClient); assertEquals(STATE.STARTED, nmClient.getServiceState()); }
Example #19
Source File: TestYarnClient.java From hadoop with Apache License 2.0 | 4 votes |
@Test (timeout = 30000) public void testSubmitIncorrectQueue() throws IOException { MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1); YarnClient rmClient = null; try { cluster.init(new YarnConfiguration()); cluster.start(); final Configuration yarnConf = cluster.getConfig(); rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); rmClient.start(); YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("nonexist"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); Assert.fail("Job submission should have thrown an exception"); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains("Failed to submit")); } finally { if (rmClient != null) { rmClient.stop(); } cluster.stop(); } }
Example #20
Source File: TwillTester.java From twill with Apache License 2.0 | 4 votes |
@Override protected void before() throws Throwable { tmpFolder.create(); // Starts Zookeeper zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build(); zkServer.startAndWait(); // Start YARN mini cluster File miniDFSDir = tmpFolder.newFolder(); LOG.info("Starting Mini DFS on path {}", miniDFSDir); Configuration fsConf = new HdfsConfiguration(new Configuration()); fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, miniDFSDir.getAbsolutePath()); for (Map.Entry<String, String> entry : extraConfig.entrySet()) { fsConf.set(entry.getKey(), entry.getValue()); } dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build(); Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf()); conf.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler"); conf.set("yarn.scheduler.capacity.resource-calculator", "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"); conf.setBoolean("yarn.scheduler.include-port-in-node-name", true); conf.set("yarn.nodemanager.vmem-pmem-ratio", "100.1"); conf.set("yarn.nodemanager.vmem-check-enabled", "false"); conf.set("yarn.scheduler.minimum-allocation-mb", "128"); conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600"); conf.set(Configs.Keys.LOCAL_STAGING_DIRECTORY, tmpFolder.newFolder().getAbsolutePath()); cluster = new MiniYARNCluster("test-cluster", 3, 1, 1); cluster.init(conf); cluster.start(); config = new YarnConfiguration(cluster.getConfig()); twillRunner = createTwillRunnerService(); twillRunner.start(); yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf); }
Example #21
Source File: MiniHadoopCluster.java From zeppelin with Apache License 2.0 | 4 votes |
public MiniYARNCluster getYarnCluster() { return yarnCluster; }
Example #22
Source File: MiniHadoopClusterManager.java From big-c with Apache License 2.0 | 4 votes |
/** * Starts DFS and MR clusters, as specified in member-variable options. Also * writes out configuration and details, if requested. * * @throws IOException * @throws FileNotFoundException * @throws URISyntaxException */ public void start() throws IOException, FileNotFoundException, URISyntaxException { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } if (!noMR) { if (fs == null && dfs != null) { fs = dfs.getFileSystem().getUri().toString(); } else if (fs == null) { fs = "file:///tmp/minimr-" + System.nanoTime(); } FileSystem.setDefaultUri(conf, new URI(fs)); // Instruct the minicluster to use fixed ports, so user will know which // ports to use when communicating with the cluster. conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); conf.set(YarnConfiguration.RM_ADDRESS, MiniYARNCluster.getHostname() + ":" + this.rmPort); conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname() + ":" + this.jhsPort); mr = MiniMRClientClusterFactory.create(this.getClass(), numNodeManagers, conf); LOG.info("Started MiniMRCluster"); } if (writeConfig != null) { FileOutputStream fos = new FileOutputStream(new File(writeConfig)); conf.writeXml(fos); fos.close(); } if (writeDetails != null) { Map<String, Object> map = new TreeMap<String, Object>(); if (dfs != null) { map.put("namenode_port", dfs.getNameNodePort()); } if (mr != null) { map.put("resourcemanager_port", mr.getConfig().get( YarnConfiguration.RM_ADDRESS).split(":")[1]); } FileWriter fw = new FileWriter(new File(writeDetails)); fw.write(new JSON().toJSON(map)); fw.close(); } }
Example #23
Source File: MiniMRYarnCluster.java From big-c with Apache License 2.0 | 4 votes |
@Override public synchronized void serviceStart() throws Exception { try { if (!getConfig().getBoolean( JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) { String hostname = MiniYARNCluster.getHostname(); // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, hostname + ":0"); MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname + ":0"); getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS, hostname + ":0"); } historyServer = new JobHistoryServer(); historyServer.init(getConfig()); new Thread() { public void run() { historyServer.start(); }; }.start(); while (historyServer.getServiceState() == STATE.INITED) { LOG.info("Waiting for HistoryServer to start..."); Thread.sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (historyServer.getServiceState() != STATE.STARTED) { throw new IOException("HistoryServer failed to start"); } super.serviceStart(); } catch (Throwable t) { throw new YarnRuntimeException(t); } //need to do this because historyServer.init creates a new Configuration getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig())); LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(), MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY)); }
Example #24
Source File: MiniMRYarnCluster.java From hadoop with Apache License 2.0 | 4 votes |
@Override public synchronized void serviceStart() throws Exception { try { if (!getConfig().getBoolean( JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) { String hostname = MiniYARNCluster.getHostname(); // pick free random ports. getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, hostname + ":0"); MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname + ":0"); getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS, hostname + ":0"); } historyServer = new JobHistoryServer(); historyServer.init(getConfig()); new Thread() { public void run() { historyServer.start(); }; }.start(); while (historyServer.getServiceState() == STATE.INITED) { LOG.info("Waiting for HistoryServer to start..."); Thread.sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (historyServer.getServiceState() != STATE.STARTED) { throw new IOException("HistoryServer failed to start"); } super.serviceStart(); } catch (Throwable t) { throw new YarnRuntimeException(t); } //need to do this because historyServer.init creates a new Configuration getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS, historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig())); LOG.info("MiniMRYARN ResourceManager address: " + getConfig().get(YarnConfiguration.RM_ADDRESS)); LOG.info("MiniMRYARN ResourceManager web address: " + WebAppUtils.getRMWebAppURLWithoutScheme(getConfig())); LOG.info("MiniMRYARN HistoryServer address: " + getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS)); LOG.info("MiniMRYARN HistoryServer web address: " + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(), MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY)); }
Example #25
Source File: TestYarnClient.java From big-c with Apache License 2.0 | 4 votes |
@Test (timeout = 30000) public void testSubmitIncorrectQueue() throws IOException { MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1); YarnClient rmClient = null; try { cluster.init(new YarnConfiguration()); cluster.start(); final Configuration yarnConf = cluster.getConfig(); rmClient = YarnClient.createYarnClient(); rmClient.init(yarnConf); rmClient.start(); YarnClientApplication newApp = rmClient.createApplication(); ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId(); // Create launch context for app master ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("test"); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("nonexist"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // appContext.setUnmanagedAM(unmanaged); // Submit the application to the applications manager rmClient.submitApplication(appContext); Assert.fail("Job submission should have thrown an exception"); } catch (YarnException e) { Assert.assertTrue(e.getMessage().contains("Failed to submit")); } finally { if (rmClient != null) { rmClient.stop(); } cluster.stop(); } }
Example #26
Source File: TestNMClient.java From big-c with Apache License 2.0 | 4 votes |
@Before public void setup() throws YarnException, IOException { // start minicluster conf = new YarnConfiguration(); yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); assertNotNull(yarnCluster); assertEquals(STATE.STARTED, yarnCluster.getServiceState()); // start rm client yarnClient = (YarnClientImpl) YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); assertNotNull(yarnClient); assertEquals(STATE.STARTED, yarnClient.getServiceState()); // get node info nodeReports = yarnClient.getNodeReports(NodeState.RUNNING); // submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Priority.newInstance(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records .newRecord(ContainerLaunchContext.class); appContext.setAMContainerSpec(amContainer); // unmanaged AM appContext.setUnmanagedAM(true); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records .newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start int iterationsLeft = 30; RMAppAttempt appAttempt = null; while (iterationsLeft > 0) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } sleep(1000); --iterationsLeft; } if (iterationsLeft == 0) { fail("Application hasn't bee started"); } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser(UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); //creating an instance NMTokenCase nmTokenCache = new NMTokenCache(); // start am rm client rmClient = (AMRMClientImpl<ContainerRequest>) AMRMClient .<ContainerRequest> createAMRMClient(); //setting an instance NMTokenCase rmClient.setNMTokenCache(nmTokenCache); rmClient.init(conf); rmClient.start(); assertNotNull(rmClient); assertEquals(STATE.STARTED, rmClient.getServiceState()); // start am nm client nmClient = (NMClientImpl) NMClient.createNMClient(); //propagating the AMRMClient NMTokenCache instance nmClient.setNMTokenCache(rmClient.getNMTokenCache()); nmClient.init(conf); nmClient.start(); assertNotNull(nmClient); assertEquals(STATE.STARTED, nmClient.getServiceState()); }
Example #27
Source File: TestRMFailover.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testRMWebAppRedirect() throws YarnException, InterruptedException, IOException { cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1); conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); cluster.init(conf); cluster.start(); getAdminService(0).transitionToActive(req); String rm1Url = "http://0.0.0.0:18088"; String rm2Url = "http://0.0.0.0:28088"; String redirectURL = getRedirectURL(rm2Url); // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url assertEquals(redirectURL,rm1Url+"/"); redirectURL = getRedirectURL(rm2Url + "/metrics"); assertEquals(redirectURL,rm1Url + "/metrics"); redirectURL = getRedirectURL(rm2Url + "/jmx"); assertEquals(redirectURL,rm1Url + "/jmx"); // standby RM links /conf, /stacks, /logLevel, /static, /logs, // /cluster/cluster as well as webService // /ws/v1/cluster/info should not be redirected to active RM redirectURL = getRedirectURL(rm2Url + "/cluster/cluster"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/conf"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/stacks"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/logLevel"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/static"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/logs"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info"); assertNull(redirectURL); redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps"); assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps"); redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId); assertNull(redirectURL); }
Example #28
Source File: MiniHadoopClusterManager.java From hadoop with Apache License 2.0 | 4 votes |
/** * Starts DFS and MR clusters, as specified in member-variable options. Also * writes out configuration and details, if requested. * * @throws IOException * @throws FileNotFoundException * @throws URISyntaxException */ public void start() throws IOException, FileNotFoundException, URISyntaxException { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort) .numDataNodes(numDataNodes).startupOption(dfsOpts).build(); LOG.info("Started MiniDFSCluster -- namenode on port " + dfs.getNameNodePort()); } if (!noMR) { if (fs == null && dfs != null) { fs = dfs.getFileSystem().getUri().toString(); } else if (fs == null) { fs = "file:///tmp/minimr-" + System.nanoTime(); } FileSystem.setDefaultUri(conf, new URI(fs)); // Instruct the minicluster to use fixed ports, so user will know which // ports to use when communicating with the cluster. conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); conf.set(YarnConfiguration.RM_ADDRESS, MiniYARNCluster.getHostname() + ":" + this.rmPort); conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname() + ":" + this.jhsPort); mr = MiniMRClientClusterFactory.create(this.getClass(), numNodeManagers, conf); LOG.info("Started MiniMRCluster"); } if (writeConfig != null) { FileOutputStream fos = new FileOutputStream(new File(writeConfig)); conf.writeXml(fos); fos.close(); } if (writeDetails != null) { Map<String, Object> map = new TreeMap<String, Object>(); if (dfs != null) { map.put("namenode_port", dfs.getNameNodePort()); } if (mr != null) { map.put("resourcemanager_port", mr.getConfig().get( YarnConfiguration.RM_ADDRESS).split(":")[1]); } FileWriter fw = new FileWriter(new File(writeDetails)); fw.write(new JSON().toJSON(map)); fw.close(); } }