org.apache.hadoop.yarn.server.MiniYARNCluster Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.MiniYARNCluster. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRMFailover.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);

  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
}
 
Example #2
Source File: HadoopTestCluster.java    From yarn-proto with Apache License 2.0 6 votes vote down vote up
public void start() throws IOException {
  log.info("Checking if cluster=" + clusterName + " needs to be started");
  synchronized (this.startupShutdownMonitor) {
    if (started) {
      return;
    }
    log.info("Starting cluster=" + clusterName);
    configuration = new YarnConfiguration();

    //configuration.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
    configuration.setBoolean("yarn.is.minicluster", true);
    configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs");

    dfsCluster = new MiniDFSCluster.Builder(configuration).
            numDataNodes(nodes).
            build();

    yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1);
    yarnCluster.init(configuration);
    yarnCluster.start();

    log.info("Started cluster=" + clusterName);
    started = true;
  }
}
 
Example #3
Source File: MiniTajoYarnCluster.java    From incubator-tajo with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void start() {
  try {
    if (!getConfig().getBoolean(
        JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
        JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
      // pick free random ports.
      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
          MiniYARNCluster.getHostname() + ":0");
      getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
          MiniYARNCluster.getHostname() + ":0");
    }
    super.start();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }

  LOG.info("MiniMRYARN ResourceManager address: " +
      getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniMRYARN ResourceManager web address: " +
      getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer address: " +
      getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer web address: " +
      getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS));
}
 
Example #4
Source File: TestRMFailover.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
  fakeAppId = ApplicationId.newInstance(System.currentTimeMillis(), 0);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);

  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 1, 1, 1);
}
 
Example #5
Source File: UnitTestCluster.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception {
  yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
  yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
  MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1);
  miniYarnCluster.init(yarnConf);
  yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ;
  miniYarnCluster.start();
  return miniYarnCluster ;
}
 
Example #6
Source File: YarnTestBinding.java    From mr4c with Apache License 2.0 5 votes vote down vote up
private void startMrCluster() throws IOException {
	Configuration conf = new JobConf();
	FileSystem.setDefaultUri(conf, HadoopTestUtils.getTestDFS().getUri());
	conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
	conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
	String addr = MiniYARNCluster.getHostname() + ":0";
	conf.set(YarnConfiguration.RM_ADDRESS, addr);
	conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, addr);
	m_mrCluster = MiniMRClientClusterFactory.create(
		HadoopTestUtils.class,
		"MR4CTests",
		1, // num node managers
		conf
	);

	// make sure startup is finished
	for ( int i=0; i<60; i++ ) {
		String newAddr = m_mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS);
		if ( newAddr.equals(addr) ) {
			s_log.warn("MiniYARNCluster startup not complete");
			try {
				Thread.sleep(1000);
			} catch (InterruptedException ie) {
				throw new IOException(ie);
			}
		} else {
			s_log.info("MiniYARNCluster now available at {}", newAddr);
			return;
		}
	}
	throw new IOException("MiniYARNCluster taking too long to startup");

}
 
Example #7
Source File: YarnLocalCluster.java    From hadoop-mini-clusters with Apache License 2.0 5 votes vote down vote up
@Override
public void start() throws Exception {
    LOG.info("YARN: Starting MiniYarnCluster");
    configure();
    miniYARNCluster = new MiniYARNCluster(testName, numResourceManagers, numNodeManagers,
            numLocalDirs, numLogDirs, enableHa);

    miniYARNCluster.serviceInit(configuration);
    miniYARNCluster.init(configuration);
    miniYARNCluster.start();
}
 
Example #8
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 
Example #9
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 
Example #10
Source File: YarnComponent.java    From metron with Apache License 2.0 4 votes vote down vote up
public MiniYARNCluster getYARNCluster(){
    return yarnCluster;
}
 
Example #11
Source File: MiniAthenaXCluster.java    From AthenaX with Apache License 2.0 4 votes vote down vote up
public MiniAthenaXCluster(String name) {
  this.yarnCluster = new MiniYARNCluster(name, 1, 1, 1, 1);
  this.workDir = new File("target", name);
}
 
Example #12
Source File: UnitTestCluster.java    From Scribengin with GNU Affero General Public License v3.0 4 votes vote down vote up
protected MiniYARNCluster createMiniYARNCluster(int numOfNodeManagers) throws Exception {
  return createMiniYARNCluster(new YarnConfiguration(), numOfNodeManagers) ;
}
 
Example #13
Source File: TestRMFailover.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testRMWebAppRedirect() throws YarnException,
    InterruptedException, IOException {
  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);

  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  String rm1Url = "http://0.0.0.0:18088";
  String rm2Url = "http://0.0.0.0:28088";
  String redirectURL = getRedirectURL(rm2Url);
  // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
  assertEquals(redirectURL,rm1Url+"/");

  redirectURL = getRedirectURL(rm2Url + "/metrics");
  assertEquals(redirectURL,rm1Url + "/metrics");

  redirectURL = getRedirectURL(rm2Url + "/jmx");
  assertEquals(redirectURL,rm1Url + "/jmx");

  // standby RM links /conf, /stacks, /logLevel, /static, /logs,
  // /cluster/cluster as well as webService
  // /ws/v1/cluster/info should not be redirected to active RM
  redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/conf");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/stacks");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logLevel");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/static");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logs");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
  assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");

  redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
  assertNull(redirectURL);
}
 
Example #14
Source File: GobblinYarnAppLauncherTest.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public void setUp() throws Exception {
  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  final YarnConfiguration clusterConf = new YarnConfiguration();
  clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

  MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
  miniYARNCluster.init(clusterConf);
  miniYARNCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(LOG).timeoutMs(10000)
      .assertTrue(new Predicate<Void>() {
        @Override public boolean apply(Void input) {
          return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
      }, "Waiting for RM");

  this.yarnClient = this.closer.register(YarnClient.createYarnClient());
  this.yarnClient.init(clusterConf);
  this.yarnClient.start();

  // Use a random ZK port
  TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
  LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

  // the zk port is dynamically configured
  try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) {
    File dir = new File("target/dummydir");

    // dummy directory specified in configuration
    dir.mkdir();

    pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
    pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
  }

  // YARN config is dynamic and needs to be passed to other processes
  try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
    clusterConf.writeXml(os);
  }

  this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

  URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
      .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);

  this.config = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
                 ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .resolve();

  String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
  this.helixManager = HelixManagerFactory.getZKHelixManager(
      this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME,
      InstanceType.CONTROLLER, zkConnectionString);

  this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);

  this.configManagedHelix = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
          ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .withValue(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY, ConfigValueFactory.fromAnyRef(MANAGED_HELIX_CLUSTER_NAME))
      .withValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, ConfigValueFactory.fromAnyRef(TEST_HELIX_INSTANCE_NAME_MANAGED))
      .withValue(GobblinClusterConfigurationKeys.IS_HELIX_CLUSTER_MANAGED, ConfigValueFactory.fromAnyRef("true"))
      .resolve();

  this.helixManagerManagedHelix = HelixManagerFactory.getZKHelixManager(
      this.configManagedHelix.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TEST_HELIX_INSTANCE_NAME_MANAGED,
      InstanceType.PARTICIPANT, zkConnectionString);

  this.gobblinYarnAppLauncherManagedHelix = new GobblinYarnAppLauncher(this.configManagedHelix, clusterConf);
}
 
Example #15
Source File: YarnServiceTestWithExpiration.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public void setUp() throws Exception {
  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  this.clusterConf = new YarnConfiguration();
  this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100");
  this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000");
  this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000");
  this.clusterConf.set(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, "1000");

  this.yarnCluster =
      this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1,
          1));
  this.yarnCluster.init(this.clusterConf);
  this.yarnCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(LOG).timeoutMs(10000)
      .assertTrue(new Predicate<Void>() {
        @Override public boolean apply(Void input) {
          return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
      }, "Waiting for RM");

  this.yarnClient = this.closer.register(YarnClient.createYarnClient());
  this.yarnClient.init(this.clusterConf);
  this.yarnClient.start();

  URL url = YarnServiceTest.class.getClassLoader()
      .getResource(YarnServiceTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);

  this.config = ConfigFactory.parseURL(url).resolve();

  // Start a dummy application manager so that the YarnService can use the AM-RM token.
  startApp();

  // create and start the test yarn service
  this.expiredYarnService = new TestExpiredYarnService(this.config, "testApp", "appId",
      this.clusterConf,
      FileSystem.getLocal(new Configuration()), this.eventBus);

  this.expiredYarnService.startUp();
}
 
Example #16
Source File: YarnServiceTest.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public void setUp() throws Exception {
  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  this.clusterConf = new YarnConfiguration();
  this.clusterConf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "100");
  this.clusterConf.set(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, "10000");
  this.clusterConf.set(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS, "60000");

  this.yarnCluster =
      this.closer.register(new MiniYARNCluster("YarnServiceTestCluster", 4, 1,
          1));
  this.yarnCluster.init(this.clusterConf);
  this.yarnCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(LOG).timeoutMs(10000)
      .assertTrue(new Predicate<Void>() {
        @Override public boolean apply(Void input) {
          return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
      }, "Waiting for RM");

  this.yarnClient = this.closer.register(YarnClient.createYarnClient());
  this.yarnClient.init(this.clusterConf);
  this.yarnClient.start();

  URL url = YarnServiceTest.class.getClassLoader()
      .getResource(YarnServiceTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);

  this.config = ConfigFactory.parseURL(url).resolve();

  // Start a dummy application manager so that the YarnService can use the AM-RM token.
  startApp();

  // create and start the test yarn service
  this.yarnService = new TestYarnService(this.config, "testApp", "appId",
      this.clusterConf,
      FileSystem.getLocal(new Configuration()), this.eventBus);

 this.yarnService.startUp();
}
 
Example #17
Source File: EmbeddedGobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
private static void setup(String[] args)
    throws Exception {
  // Parsing zk-string
  Preconditions.checkArgument(args.length == 1);
  zkString = args[0];

  // Initialize necessary external components: Yarn and Helix
  Closer closer = Closer.create();

  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  final YarnConfiguration clusterConf = new YarnConfiguration();
  clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");
  clusterConf.set("yarn.nodemanager.resource.memory-mb", "512");
  clusterConf.set("yarn.scheduler.maximum-allocation-mb", "1024");

  MiniYARNCluster miniYARNCluster = closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
  miniYARNCluster.init(clusterConf);
  miniYARNCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(log).timeoutMs(10000).assertTrue(new Predicate<Void>() {
    @Override
    public boolean apply(Void input) {
      return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
    }
  }, "Waiting for RM");

  try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH, "UTF-8")) {
    File dir = new File("target/dummydir");

    // dummy directory specified in configuration
    if (!dir.mkdir()) {
      log.error("The dummy folder's creation is not successful");
    }
    dir.deleteOnExit();

    pw.println("gobblin.cluster.zk.connection.string=\"" + zkString + "\"");
    pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
  }

  // YARN config is dynamic and needs to be passed to other processes
  try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
    clusterConf.writeXml(os);
  }

  /** Have to pass the same yarn-site.xml to the GobblinYarnAppLauncher to initialize Yarn Client. */
  fileAddress = new File(YARN_SITE_XML_PATH).getAbsolutePath();
}
 
Example #18
Source File: TestNMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 
Example #19
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 
Example #20
Source File: TwillTester.java    From twill with Apache License 2.0 4 votes vote down vote up
@Override
protected void before() throws Throwable {
  tmpFolder.create();

  // Starts Zookeeper
  zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
  zkServer.startAndWait();

  // Start YARN mini cluster
  File miniDFSDir = tmpFolder.newFolder();
  LOG.info("Starting Mini DFS on path {}", miniDFSDir);
  Configuration fsConf = new HdfsConfiguration(new Configuration());
  fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, miniDFSDir.getAbsolutePath());

  for (Map.Entry<String, String> entry : extraConfig.entrySet()) {
    fsConf.set(entry.getKey(), entry.getValue());
  }

  dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build();

  Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

  conf.set("yarn.resourcemanager.scheduler.class",
           "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
  conf.set("yarn.scheduler.capacity.resource-calculator",
           "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
  conf.setBoolean("yarn.scheduler.include-port-in-node-name", true);
  conf.set("yarn.nodemanager.vmem-pmem-ratio", "100.1");
  conf.set("yarn.nodemanager.vmem-check-enabled", "false");
  conf.set("yarn.scheduler.minimum-allocation-mb", "128");
  conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600");

  conf.set(Configs.Keys.LOCAL_STAGING_DIRECTORY, tmpFolder.newFolder().getAbsolutePath());

  cluster = new MiniYARNCluster("test-cluster", 3, 1, 1);
  cluster.init(conf);
  cluster.start();

  config = new YarnConfiguration(cluster.getConfig());

  twillRunner = createTwillRunnerService();
  twillRunner.start();

  yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
}
 
Example #21
Source File: MiniHadoopCluster.java    From zeppelin with Apache License 2.0 4 votes vote down vote up
public MiniYARNCluster getYarnCluster() {
  return yarnCluster;
}
 
Example #22
Source File: MiniHadoopClusterManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Starts DFS and MR clusters, as specified in member-variable options. Also
 * writes out configuration and details, if requested.
 *
 * @throws IOException
 * @throws FileNotFoundException
 * @throws URISyntaxException
 */
public void start() throws IOException, FileNotFoundException,
    URISyntaxException {
  if (!noDFS) {
    dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort)
        .numDataNodes(numDataNodes).startupOption(dfsOpts).build();
    LOG.info("Started MiniDFSCluster -- namenode on port "
        + dfs.getNameNodePort());
  }
  if (!noMR) {
    if (fs == null && dfs != null) {
      fs = dfs.getFileSystem().getUri().toString();
    } else if (fs == null) {
      fs = "file:///tmp/minimr-" + System.nanoTime();
    }
    FileSystem.setDefaultUri(conf, new URI(fs));
    // Instruct the minicluster to use fixed ports, so user will know which
    // ports to use when communicating with the cluster.
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    conf.set(YarnConfiguration.RM_ADDRESS, MiniYARNCluster.getHostname()
        + ":" + this.rmPort);
    conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname()
        + ":" + this.jhsPort);
    mr = MiniMRClientClusterFactory.create(this.getClass(), numNodeManagers,
        conf);
    LOG.info("Started MiniMRCluster");
  }

  if (writeConfig != null) {
    FileOutputStream fos = new FileOutputStream(new File(writeConfig));
    conf.writeXml(fos);
    fos.close();
  }

  if (writeDetails != null) {
    Map<String, Object> map = new TreeMap<String, Object>();
    if (dfs != null) {
      map.put("namenode_port", dfs.getNameNodePort());
    }
    if (mr != null) {
      map.put("resourcemanager_port", mr.getConfig().get(
          YarnConfiguration.RM_ADDRESS).split(":")[1]);
    }
    FileWriter fw = new FileWriter(new File(writeDetails));
    fw.write(new JSON().toJSON(map));
    fw.close();
  }
}
 
Example #23
Source File: MiniMRYarnCluster.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void serviceStart() throws Exception {
  try {
    if (!getConfig().getBoolean(
        JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
        JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
      String hostname = MiniYARNCluster.getHostname();
      // pick free random ports.
      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
        hostname + ":0");
      MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
          + ":0");
      getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
        hostname + ":0");
    }
    historyServer = new JobHistoryServer();
    historyServer.init(getConfig());
    new Thread() {
      public void run() {
        historyServer.start();
      };
    }.start();
    while (historyServer.getServiceState() == STATE.INITED) {
      LOG.info("Waiting for HistoryServer to start...");
      Thread.sleep(1500);
    }
    //TODO Add a timeout. State.STOPPED check ?
    if (historyServer.getServiceState() != STATE.STARTED) {
      throw new IOException("HistoryServer failed to start");
    }
    super.serviceStart();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }
  //need to do this because historyServer.init creates a new Configuration
  getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
                  historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
      MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));

  LOG.info("MiniMRYARN ResourceManager address: " +
           getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniMRYARN ResourceManager web address: " +
           WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
  LOG.info("MiniMRYARN HistoryServer address: " +
           getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer web address: "
      + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
          MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
}
 
Example #24
Source File: MiniMRYarnCluster.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void serviceStart() throws Exception {
  try {
    if (!getConfig().getBoolean(
        JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
        JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
      String hostname = MiniYARNCluster.getHostname();
      // pick free random ports.
      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
        hostname + ":0");
      MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
          + ":0");
      getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
        hostname + ":0");
    }
    historyServer = new JobHistoryServer();
    historyServer.init(getConfig());
    new Thread() {
      public void run() {
        historyServer.start();
      };
    }.start();
    while (historyServer.getServiceState() == STATE.INITED) {
      LOG.info("Waiting for HistoryServer to start...");
      Thread.sleep(1500);
    }
    //TODO Add a timeout. State.STOPPED check ?
    if (historyServer.getServiceState() != STATE.STARTED) {
      throw new IOException("HistoryServer failed to start");
    }
    super.serviceStart();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }
  //need to do this because historyServer.init creates a new Configuration
  getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
                  historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
      MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));

  LOG.info("MiniMRYARN ResourceManager address: " +
           getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniMRYARN ResourceManager web address: " +
           WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
  LOG.info("MiniMRYARN HistoryServer address: " +
           getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer web address: "
      + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
          MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
}
 
Example #25
Source File: TestYarnClient.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 
Example #26
Source File: TestNMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 
Example #27
Source File: TestRMFailover.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testRMWebAppRedirect() throws YarnException,
    InterruptedException, IOException {
  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);

  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  String rm1Url = "http://0.0.0.0:18088";
  String rm2Url = "http://0.0.0.0:28088";
  String redirectURL = getRedirectURL(rm2Url);
  // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
  assertEquals(redirectURL,rm1Url+"/");

  redirectURL = getRedirectURL(rm2Url + "/metrics");
  assertEquals(redirectURL,rm1Url + "/metrics");

  redirectURL = getRedirectURL(rm2Url + "/jmx");
  assertEquals(redirectURL,rm1Url + "/jmx");

  // standby RM links /conf, /stacks, /logLevel, /static, /logs,
  // /cluster/cluster as well as webService
  // /ws/v1/cluster/info should not be redirected to active RM
  redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/conf");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/stacks");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logLevel");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/static");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logs");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
  assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");

  redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
  assertNull(redirectURL);
}
 
Example #28
Source File: MiniHadoopClusterManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Starts DFS and MR clusters, as specified in member-variable options. Also
 * writes out configuration and details, if requested.
 *
 * @throws IOException
 * @throws FileNotFoundException
 * @throws URISyntaxException
 */
public void start() throws IOException, FileNotFoundException,
    URISyntaxException {
  if (!noDFS) {
    dfs = new MiniDFSCluster.Builder(conf).nameNodePort(nnPort)
        .numDataNodes(numDataNodes).startupOption(dfsOpts).build();
    LOG.info("Started MiniDFSCluster -- namenode on port "
        + dfs.getNameNodePort());
  }
  if (!noMR) {
    if (fs == null && dfs != null) {
      fs = dfs.getFileSystem().getUri().toString();
    } else if (fs == null) {
      fs = "file:///tmp/minimr-" + System.nanoTime();
    }
    FileSystem.setDefaultUri(conf, new URI(fs));
    // Instruct the minicluster to use fixed ports, so user will know which
    // ports to use when communicating with the cluster.
    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
    conf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true);
    conf.set(YarnConfiguration.RM_ADDRESS, MiniYARNCluster.getHostname()
        + ":" + this.rmPort);
    conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, MiniYARNCluster.getHostname()
        + ":" + this.jhsPort);
    mr = MiniMRClientClusterFactory.create(this.getClass(), numNodeManagers,
        conf);
    LOG.info("Started MiniMRCluster");
  }

  if (writeConfig != null) {
    FileOutputStream fos = new FileOutputStream(new File(writeConfig));
    conf.writeXml(fos);
    fos.close();
  }

  if (writeDetails != null) {
    Map<String, Object> map = new TreeMap<String, Object>();
    if (dfs != null) {
      map.put("namenode_port", dfs.getNameNodePort());
    }
    if (mr != null) {
      map.put("resourcemanager_port", mr.getConfig().get(
          YarnConfiguration.RM_ADDRESS).split(":")[1]);
    }
    FileWriter fw = new FileWriter(new File(writeDetails));
    fw.write(new JSON().toJSON(map));
    fw.close();
  }
}