Java Code Examples for org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer

The following examples show how to use org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: spydra   Source File: HistoryLogUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Starts a minimal JobHistoryServer.
 */
public static void startJhs(Configuration cfg) {
  try {
    JobHistoryServer jobHistoryServer = new JobHistoryServer();
    jobHistoryServer.init(cfg);
    logger.info(String.format(
        "Starting JobHistoryServer on: http://%s",
        cfg.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS)));
    jobHistoryServer.start();
  } catch (Exception e) {
    logger.error("Error starting JobHistoryServer", e);
    System.exit(1);
  }
}
 
Example 2
Source Project: hadoop   Source File: TestHsWebServices.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyHsInfoGeneric(String hadoopVersionBuiltOn,
    String hadoopBuildVersion, String hadoopVersion, long startedon) {
  WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
      VersionInfo.getDate(), hadoopVersionBuiltOn);
  WebServicesTestUtils.checkStringEqual("hadoopBuildVersion",
      VersionInfo.getBuildVersion(), hadoopBuildVersion);
  WebServicesTestUtils.checkStringMatch("hadoopVersion",
      VersionInfo.getVersion(), hadoopVersion);
  assertEquals("startedOn doesn't match: ",
      JobHistoryServer.historyServerTimeStamp, startedon);
}
 
Example 3
Source Project: big-c   Source File: TestHsWebServices.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyHsInfoGeneric(String hadoopVersionBuiltOn,
    String hadoopBuildVersion, String hadoopVersion, long startedon) {
  WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
      VersionInfo.getDate(), hadoopVersionBuiltOn);
  WebServicesTestUtils.checkStringEqual("hadoopBuildVersion",
      VersionInfo.getBuildVersion(), hadoopBuildVersion);
  WebServicesTestUtils.checkStringMatch("hadoopVersion",
      VersionInfo.getVersion(), hadoopVersion);
  assertEquals("startedOn doesn't match: ",
      JobHistoryServer.historyServerTimeStamp, startedon);
}
 
Example 4
Source Project: hadoop   Source File: MiniMRYarnCluster.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void serviceStart() throws Exception {
  try {
    if (!getConfig().getBoolean(
        JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
        JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
      String hostname = MiniYARNCluster.getHostname();
      // pick free random ports.
      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
        hostname + ":0");
      MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
          + ":0");
      getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
        hostname + ":0");
    }
    historyServer = new JobHistoryServer();
    historyServer.init(getConfig());
    new Thread() {
      public void run() {
        historyServer.start();
      };
    }.start();
    while (historyServer.getServiceState() == STATE.INITED) {
      LOG.info("Waiting for HistoryServer to start...");
      Thread.sleep(1500);
    }
    //TODO Add a timeout. State.STOPPED check ?
    if (historyServer.getServiceState() != STATE.STARTED) {
      throw new IOException("HistoryServer failed to start");
    }
    super.serviceStart();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }
  //need to do this because historyServer.init creates a new Configuration
  getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
                  historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
      MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));

  LOG.info("MiniMRYARN ResourceManager address: " +
           getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniMRYARN ResourceManager web address: " +
           WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
  LOG.info("MiniMRYARN HistoryServer address: " +
           getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer web address: "
      + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
          MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
}
 
Example 5
Source Project: hadoop   Source File: MiniMRYarnCluster.java    License: Apache License 2.0 4 votes vote down vote up
public JobHistoryServer getHistoryServer() {
  return this.historyServer;
}
 
Example 6
Source Project: hadoop   Source File: HistoryInfo.java    License: Apache License 2.0 4 votes vote down vote up
public HistoryInfo() {
  this.startedOn = JobHistoryServer.historyServerTimeStamp;
  this.hadoopVersion = VersionInfo.getVersion();
  this.hadoopBuildVersion = VersionInfo.getBuildVersion();
  this.hadoopVersionBuiltOn = VersionInfo.getDate();
}
 
Example 7
Source Project: big-c   Source File: MiniMRYarnCluster.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void serviceStart() throws Exception {
  try {
    if (!getConfig().getBoolean(
        JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
        JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
      String hostname = MiniYARNCluster.getHostname();
      // pick free random ports.
      getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
        hostname + ":0");
      MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(), hostname
          + ":0");
      getConfig().set(JHAdminConfig.JHS_ADMIN_ADDRESS,
        hostname + ":0");
    }
    historyServer = new JobHistoryServer();
    historyServer.init(getConfig());
    new Thread() {
      public void run() {
        historyServer.start();
      };
    }.start();
    while (historyServer.getServiceState() == STATE.INITED) {
      LOG.info("Waiting for HistoryServer to start...");
      Thread.sleep(1500);
    }
    //TODO Add a timeout. State.STOPPED check ?
    if (historyServer.getServiceState() != STATE.STARTED) {
      throw new IOException("HistoryServer failed to start");
    }
    super.serviceStart();
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }
  //need to do this because historyServer.init creates a new Configuration
  getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
                  historyServer.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  MRWebAppUtil.setJHSWebappURLWithoutScheme(getConfig(),
      MRWebAppUtil.getJHSWebappURLWithoutScheme(historyServer.getConfig()));

  LOG.info("MiniMRYARN ResourceManager address: " +
           getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniMRYARN ResourceManager web address: " +
           WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
  LOG.info("MiniMRYARN HistoryServer address: " +
           getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS));
  LOG.info("MiniMRYARN HistoryServer web address: "
      + getResolvedMRHistoryWebAppURLWithoutScheme(getConfig(),
          MRWebAppUtil.getJHSHttpPolicy() == HttpConfig.Policy.HTTPS_ONLY));
}
 
Example 8
Source Project: big-c   Source File: MiniMRYarnCluster.java    License: Apache License 2.0 4 votes vote down vote up
public JobHistoryServer getHistoryServer() {
  return this.historyServer;
}
 
Example 9
Source Project: big-c   Source File: HistoryInfo.java    License: Apache License 2.0 4 votes vote down vote up
public HistoryInfo() {
  this.startedOn = JobHistoryServer.historyServerTimeStamp;
  this.hadoopVersion = VersionInfo.getVersion();
  this.hadoopBuildVersion = VersionInfo.getBuildVersion();
  this.hadoopVersionBuiltOn = VersionInfo.getDate();
}