Java Code Examples for org.apache.hadoop.yarn.server.MiniYARNCluster#init()

The following examples show how to use org.apache.hadoop.yarn.server.MiniYARNCluster#init() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HadoopTestCluster.java    From yarn-proto with Apache License 2.0 6 votes vote down vote up
public void start() throws IOException {
  log.info("Checking if cluster=" + clusterName + " needs to be started");
  synchronized (this.startupShutdownMonitor) {
    if (started) {
      return;
    }
    log.info("Starting cluster=" + clusterName);
    configuration = new YarnConfiguration();

    //configuration.setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
    configuration.setBoolean("yarn.is.minicluster", true);
    configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs");

    dfsCluster = new MiniDFSCluster.Builder(configuration).
            numDataNodes(nodes).
            build();

    yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1);
    yarnCluster.init(configuration);
    yarnCluster.start();

    log.info("Started cluster=" + clusterName);
    started = true;
  }
}
 
Example 2
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 
Example 3
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 
Example 4
Source File: YarnLocalCluster.java    From hadoop-mini-clusters with Apache License 2.0 5 votes vote down vote up
@Override
public void start() throws Exception {
    LOG.info("YARN: Starting MiniYarnCluster");
    configure();
    miniYARNCluster = new MiniYARNCluster(testName, numResourceManagers, numNodeManagers,
            numLocalDirs, numLogDirs, enableHa);

    miniYARNCluster.serviceInit(configuration);
    miniYARNCluster.init(configuration);
    miniYARNCluster.start();
}
 
Example 5
Source File: UnitTestCluster.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
protected MiniYARNCluster createMiniYARNCluster(Configuration yarnConf, int numOfNodeManagers) throws Exception {
  yarnConf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
  yarnConf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class);
  MiniYARNCluster miniYarnCluster = new MiniYARNCluster("yarn", numOfNodeManagers, 1, 1);
  miniYarnCluster.init(yarnConf);
  yarnConf.set("yarn.resourcemanager.scheduler.address", "0.0.0.0:8030") ;
  miniYarnCluster.start();
  return miniYarnCluster ;
}
 
Example 6
Source File: TestRMFailover.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testRMWebAppRedirect() throws YarnException,
    InterruptedException, IOException {
  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);

  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  String rm1Url = "http://0.0.0.0:18088";
  String rm2Url = "http://0.0.0.0:28088";
  String redirectURL = getRedirectURL(rm2Url);
  // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
  assertEquals(redirectURL,rm1Url+"/");

  redirectURL = getRedirectURL(rm2Url + "/metrics");
  assertEquals(redirectURL,rm1Url + "/metrics");

  redirectURL = getRedirectURL(rm2Url + "/jmx");
  assertEquals(redirectURL,rm1Url + "/jmx");

  // standby RM links /conf, /stacks, /logLevel, /static, /logs,
  // /cluster/cluster as well as webService
  // /ws/v1/cluster/info should not be redirected to active RM
  redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/conf");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/stacks");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logLevel");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/static");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logs");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
  assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");

  redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
  assertNull(redirectURL);
}
 
Example 7
Source File: TestNMClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 
Example 8
Source File: TestYarnClient.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 
Example 9
Source File: TwillTester.java    From twill with Apache License 2.0 4 votes vote down vote up
@Override
protected void before() throws Throwable {
  tmpFolder.create();

  // Starts Zookeeper
  zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
  zkServer.startAndWait();

  // Start YARN mini cluster
  File miniDFSDir = tmpFolder.newFolder();
  LOG.info("Starting Mini DFS on path {}", miniDFSDir);
  Configuration fsConf = new HdfsConfiguration(new Configuration());
  fsConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, miniDFSDir.getAbsolutePath());

  for (Map.Entry<String, String> entry : extraConfig.entrySet()) {
    fsConf.set(entry.getKey(), entry.getValue());
  }

  dfsCluster = new MiniDFSCluster.Builder(fsConf).numDataNodes(1).build();

  Configuration conf = new YarnConfiguration(dfsCluster.getFileSystem().getConf());

  conf.set("yarn.resourcemanager.scheduler.class",
           "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
  conf.set("yarn.scheduler.capacity.resource-calculator",
           "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
  conf.setBoolean("yarn.scheduler.include-port-in-node-name", true);
  conf.set("yarn.nodemanager.vmem-pmem-ratio", "100.1");
  conf.set("yarn.nodemanager.vmem-check-enabled", "false");
  conf.set("yarn.scheduler.minimum-allocation-mb", "128");
  conf.set("yarn.nodemanager.delete.debug-delay-sec", "3600");

  conf.set(Configs.Keys.LOCAL_STAGING_DIRECTORY, tmpFolder.newFolder().getAbsolutePath());

  cluster = new MiniYARNCluster("test-cluster", 3, 1, 1);
  cluster.init(conf);
  cluster.start();

  config = new YarnConfiguration(cluster.getConfig());

  twillRunner = createTwillRunnerService();
  twillRunner.start();

  yarnAppClient = new VersionDetectYarnAppClientFactory().create(conf);
}
 
Example 10
Source File: TestRMFailover.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testRMWebAppRedirect() throws YarnException,
    InterruptedException, IOException {
  cluster = new MiniYARNCluster(TestRMFailover.class.getName(), 2, 0, 1, 1);
  conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);

  cluster.init(conf);
  cluster.start();
  getAdminService(0).transitionToActive(req);
  String rm1Url = "http://0.0.0.0:18088";
  String rm2Url = "http://0.0.0.0:28088";
  String redirectURL = getRedirectURL(rm2Url);
  // if uri is null, RMWebAppFilter will append a slash at the trail of the redirection url
  assertEquals(redirectURL,rm1Url+"/");

  redirectURL = getRedirectURL(rm2Url + "/metrics");
  assertEquals(redirectURL,rm1Url + "/metrics");

  redirectURL = getRedirectURL(rm2Url + "/jmx");
  assertEquals(redirectURL,rm1Url + "/jmx");

  // standby RM links /conf, /stacks, /logLevel, /static, /logs,
  // /cluster/cluster as well as webService
  // /ws/v1/cluster/info should not be redirected to active RM
  redirectURL = getRedirectURL(rm2Url + "/cluster/cluster");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/conf");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/stacks");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logLevel");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/static");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/logs");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/info");
  assertNull(redirectURL);

  redirectURL = getRedirectURL(rm2Url + "/ws/v1/cluster/apps");
  assertEquals(redirectURL, rm1Url + "/ws/v1/cluster/apps");

  redirectURL = getRedirectURL(rm2Url + "/proxy/" + fakeAppId);
  assertNull(redirectURL);
}
 
Example 11
Source File: TestNMClient.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 
Example 12
Source File: TestYarnClient.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 
Example 13
Source File: EmbeddedGobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
private static void setup(String[] args)
    throws Exception {
  // Parsing zk-string
  Preconditions.checkArgument(args.length == 1);
  zkString = args[0];

  // Initialize necessary external components: Yarn and Helix
  Closer closer = Closer.create();

  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  final YarnConfiguration clusterConf = new YarnConfiguration();
  clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");
  clusterConf.set("yarn.nodemanager.resource.memory-mb", "512");
  clusterConf.set("yarn.scheduler.maximum-allocation-mb", "1024");

  MiniYARNCluster miniYARNCluster = closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
  miniYARNCluster.init(clusterConf);
  miniYARNCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(log).timeoutMs(10000).assertTrue(new Predicate<Void>() {
    @Override
    public boolean apply(Void input) {
      return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
    }
  }, "Waiting for RM");

  try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH, "UTF-8")) {
    File dir = new File("target/dummydir");

    // dummy directory specified in configuration
    if (!dir.mkdir()) {
      log.error("The dummy folder's creation is not successful");
    }
    dir.deleteOnExit();

    pw.println("gobblin.cluster.zk.connection.string=\"" + zkString + "\"");
    pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
  }

  // YARN config is dynamic and needs to be passed to other processes
  try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
    clusterConf.writeXml(os);
  }

  /** Have to pass the same yarn-site.xml to the GobblinYarnAppLauncher to initialize Yarn Client. */
  fileAddress = new File(YARN_SITE_XML_PATH).getAbsolutePath();
}
 
Example 14
Source File: GobblinYarnAppLauncherTest.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public void setUp() throws Exception {
  // Set java home in environment since it isn't set on some systems
  String javaHome = System.getProperty("java.home");
  setEnv("JAVA_HOME", javaHome);

  final YarnConfiguration clusterConf = new YarnConfiguration();
  clusterConf.set("yarn.resourcemanager.connect.max-wait.ms", "10000");

  MiniYARNCluster miniYARNCluster = this.closer.register(new MiniYARNCluster("TestCluster", 1, 1, 1));
  miniYARNCluster.init(clusterConf);
  miniYARNCluster.start();

  // YARN client should not be started before the Resource Manager is up
  AssertWithBackoff.create().logger(LOG).timeoutMs(10000)
      .assertTrue(new Predicate<Void>() {
        @Override public boolean apply(Void input) {
          return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0");
        }
      }, "Waiting for RM");

  this.yarnClient = this.closer.register(YarnClient.createYarnClient());
  this.yarnClient.init(clusterConf);
  this.yarnClient.start();

  // Use a random ZK port
  TestingServer testingZKServer = this.closer.register(new TestingServer(-1));
  LOG.info("Testing ZK Server listening on: " + testingZKServer.getConnectString());

  // the zk port is dynamically configured
  try (PrintWriter pw = new PrintWriter(DYNAMIC_CONF_PATH)) {
    File dir = new File("target/dummydir");

    // dummy directory specified in configuration
    dir.mkdir();

    pw.println("gobblin.cluster.zk.connection.string=\"" + testingZKServer.getConnectString() + "\"");
    pw.println("jobconf.fullyQualifiedPath=\"" + dir.getAbsolutePath() + "\"");
  }

  // YARN config is dynamic and needs to be passed to other processes
  try (OutputStream os = new FileOutputStream(new File(YARN_SITE_XML_PATH))) {
    clusterConf.writeXml(os);
  }

  this.curatorFramework = TestHelper.createZkClient(testingZKServer, this.closer);

  URL url = GobblinYarnAppLauncherTest.class.getClassLoader()
      .getResource(GobblinYarnAppLauncherTest.class.getSimpleName() + ".conf");
  Assert.assertNotNull(url, "Could not find resource " + url);

  this.config = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
                 ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .resolve();

  String zkConnectionString = this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY);
  this.helixManager = HelixManagerFactory.getZKHelixManager(
      this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TestHelper.TEST_HELIX_INSTANCE_NAME,
      InstanceType.CONTROLLER, zkConnectionString);

  this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(this.config, clusterConf);

  this.configManagedHelix = ConfigFactory.parseURL(url)
      .withValue("gobblin.cluster.zk.connection.string",
          ConfigValueFactory.fromAnyRef(testingZKServer.getConnectString()))
      .withValue(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY, ConfigValueFactory.fromAnyRef(MANAGED_HELIX_CLUSTER_NAME))
      .withValue(GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, ConfigValueFactory.fromAnyRef(TEST_HELIX_INSTANCE_NAME_MANAGED))
      .withValue(GobblinClusterConfigurationKeys.IS_HELIX_CLUSTER_MANAGED, ConfigValueFactory.fromAnyRef("true"))
      .resolve();

  this.helixManagerManagedHelix = HelixManagerFactory.getZKHelixManager(
      this.configManagedHelix.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), TEST_HELIX_INSTANCE_NAME_MANAGED,
      InstanceType.PARTICIPANT, zkConnectionString);

  this.gobblinYarnAppLauncherManagedHelix = new GobblinYarnAppLauncher(this.configManagedHelix, clusterConf);
}