Java Code Examples for org.apache.hadoop.yarn.webapp.util.WebAppUtils

The following examples show how to use org.apache.hadoop.yarn.webapp.util.WebAppUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   Source File: SecurityUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Modify config object by adding SSL related parameters into a resource for WebApp's use
 *
 * @param config  Configuration to be modified
 * @param sslConfig
 */
private static void addSSLConfigResource(Configuration config, SSLConfig sslConfig)
{
  String nodeLocalConfig = sslConfig.getConfigPath();
  if (StringUtils.isNotEmpty(nodeLocalConfig)) {
    config.addResource(new Path(nodeLocalConfig));
  } else {
    // create a configuration object and add it as a resource
    Configuration sslConfigResource = new Configuration(false);
    final String SSL_CONFIG_LONG_NAME = Context.DAGContext.SSL_CONFIG.getLongName();
    sslConfigResource.set(SSL_SERVER_KEYSTORE_LOCATION, new Path(sslConfig.getKeyStorePath()).getName(), SSL_CONFIG_LONG_NAME);
    sslConfigResource.set(WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY, sslConfig.getKeyStorePassword(), SSL_CONFIG_LONG_NAME);
    sslConfigResource.set(WebAppUtils.WEB_APP_KEY_PASSWORD_KEY, sslConfig.getKeyStoreKeyPassword(), SSL_CONFIG_LONG_NAME);
    config.addResource(sslConfigResource);
  }
}
 
Example 2
Source Project: hadoop   Source File: TestWebAppUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
  Configuration configuration = new Configuration();
  final String rmAddress = "host1:8088";
  configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
  final String rm1Address = "host2:8088";
  final String rm2Address = "host3:8088";
  configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
  configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
  configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);

  String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
  Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);

  String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
  Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
}
 
Example 3
Source Project: hadoop   Source File: TestYarnConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRMWebUrlSpecified() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  // seems a bit odd but right now we are forcing webapp for RM to be
  // RM_ADDRESS
  // for host and use the port from the RM_WEBAPP_ADDRESS
  conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "fortesting:24543");
  conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
  String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
  String[] parts = rmWebUrl.split(":");
  Assert.assertEquals("RM Web URL Port is incrrect", 24543,
      Integer.valueOf(parts[parts.length - 1]).intValue());
  Assert.assertNotSame(
      "RM Web Url not resolved correctly. Should not be rmtesting",
      "http://rmtesting:24543", rmWebUrl);
}
 
Example 4
Source Project: hadoop   Source File: WebServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  String bindAddress = WebAppUtils.getWebAppBindURL(getConfig(),
                        YarnConfiguration.NM_BIND_HOST,
                        WebAppUtils.getNMWebAppURLWithoutScheme(getConfig()));
  
  LOG.info("Instantiating NMWebApp at " + bindAddress);
  try {
    this.webApp =
        WebApps
          .$for("node", Context.class, this.nmContext, "ws")
          .at(bindAddress)
          .with(getConfig())
          .withHttpSpnegoPrincipalKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
          .withHttpSpnegoKeytabKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
          .start(this.nmWebApp);
    this.port = this.webApp.httpServer().getConnectorAddress(0).getPort();
  } catch (Exception e) {
    String msg = "NMWebapps failed to start.";
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
  super.serviceStart();
}
 
Example 5
Source Project: hadoop   Source File: AppAttemptInfo.java    License: Apache License 2.0 6 votes vote down vote up
public AppAttemptInfo(RMAppAttempt attempt, String user) {
  this.startTime = 0;
  this.containerId = "";
  this.nodeHttpAddress = "";
  this.nodeId = "";
  this.logsLink = "";
  if (attempt != null) {
    this.id = attempt.getAppAttemptId().getAttemptId();
    this.startTime = attempt.getStartTime();
    Container masterContainer = attempt.getMasterContainer();
    if (masterContainer != null) {
      this.containerId = masterContainer.getId().toString();
      this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
      this.nodeId = masterContainer.getNodeId().toString();
      this.logsLink =
          WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()), user);
    }
  }
}
 
Example 6
Source Project: hadoop   Source File: RMContainerImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String getNodeHttpAddress() {
  try {
    readLock.lock();
    if (container.getNodeHttpAddress() != null) {
      StringBuilder httpAddress = new StringBuilder();
      httpAddress.append(WebAppUtils.getHttpSchemePrefix(rmContext
          .getYarnConfiguration()));
      httpAddress.append(container.getNodeHttpAddress());
      return httpAddress.toString();
    } else {
      return null;
    }
  } finally {
    readLock.unlock();
  }
}
 
Example 7
Source Project: hadoop   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
private String generateProxyUriWithScheme() {
  this.readLock.lock();
  try {
    final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
    String proxy = WebAppUtils.getProxyHostAndPort(conf);
    URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
    URI result = ProxyUriUtils.getProxyUri(null, proxyUri,
        applicationAttemptId.getApplicationId());
    return result.toASCIIString();
  } catch (URISyntaxException e) {
    LOG.warn("Could not proxify the uri for "
        + applicationAttemptId.getApplicationId(), e);
    return null;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 8
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  if (this.rmContext.isHAEnabled()) {
    transitionToStandby(true);
  } else {
    transitionToActive();
  }

  startWepApp();
  if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
      false)) {
    int port = webApp.port();
    WebAppUtils.setRMWebAppPort(conf, port);
  }
  super.serviceStart();
}
 
Example 9
Source Project: hadoop   Source File: ApplicationHistoryManagerImpl.java    License: Apache License 2.0 6 votes vote down vote up
private ContainerReport convertToContainerReport(
    ContainerHistoryData containerHistory, String user) {
  // If the container has the aggregated log, add the server root url
  String logUrl = WebAppUtils.getAggregatedLogURL(
      serverHttpAddress,
      containerHistory.getAssignedNode().toString(),
      containerHistory.getContainerId().toString(),
      containerHistory.getContainerId().toString(),
      user);
  return ContainerReport.newInstance(containerHistory.getContainerId(),
    containerHistory.getAllocatedResource(),
    containerHistory.getAssignedNode(), containerHistory.getPriority(),
    containerHistory.getStartTime(), containerHistory.getFinishTime(),
    containerHistory.getDiagnosticsInfo(), logUrl,
    containerHistory.getContainerExitStatus(),
    containerHistory.getContainerState(), null);
}
 
Example 10
Source Project: big-c   Source File: TestWebAppUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void TestRMWebAppURLRemoteAndLocal() throws UnknownHostException {
  Configuration configuration = new Configuration();
  final String rmAddress = "host1:8088";
  configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS, rmAddress);
  final String rm1Address = "host2:8088";
  final String rm2Address = "host3:8088";
  configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID, rm1Address);
  configuration.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
  configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);

  String rmRemoteUrl = WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(configuration);
  Assert.assertEquals("ResolvedRemoteRMWebAppUrl should resolve to the first HA RM address", rm1Address, rmRemoteUrl);

  String rmLocalUrl = WebAppUtils.getResolvedRMWebAppURLWithoutScheme(configuration);
  Assert.assertEquals("ResolvedRMWebAppUrl should resolve to the default RM webapp address", rmAddress, rmLocalUrl);
}
 
Example 11
Source Project: big-c   Source File: TestYarnConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRMWebUrlSpecified() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  // seems a bit odd but right now we are forcing webapp for RM to be
  // RM_ADDRESS
  // for host and use the port from the RM_WEBAPP_ADDRESS
  conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "fortesting:24543");
  conf.set(YarnConfiguration.RM_ADDRESS, "rmtesting:9999");
  String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
  String[] parts = rmWebUrl.split(":");
  Assert.assertEquals("RM Web URL Port is incrrect", 24543,
      Integer.valueOf(parts[parts.length - 1]).intValue());
  Assert.assertNotSame(
      "RM Web Url not resolved correctly. Should not be rmtesting",
      "http://rmtesting:24543", rmWebUrl);
}
 
Example 12
Source Project: big-c   Source File: WebServer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  String bindAddress = WebAppUtils.getWebAppBindURL(getConfig(),
                        YarnConfiguration.NM_BIND_HOST,
                        WebAppUtils.getNMWebAppURLWithoutScheme(getConfig()));
  
  LOG.info("Instantiating NMWebApp at " + bindAddress);
  try {
    this.webApp =
        WebApps
          .$for("node", Context.class, this.nmContext, "ws")
          .at(bindAddress)
          .with(getConfig())
          .withHttpSpnegoPrincipalKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
          .withHttpSpnegoKeytabKey(
            YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
          .start(this.nmWebApp);
    this.port = this.webApp.httpServer().getConnectorAddress(0).getPort();
  } catch (Exception e) {
    String msg = "NMWebapps failed to start.";
    LOG.error(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
  super.serviceStart();
}
 
Example 13
Source Project: big-c   Source File: AppAttemptInfo.java    License: Apache License 2.0 6 votes vote down vote up
public AppAttemptInfo(RMAppAttempt attempt, String user) {
  this.startTime = 0;
  this.containerId = "";
  this.nodeHttpAddress = "";
  this.nodeId = "";
  this.logsLink = "";
  if (attempt != null) {
    this.id = attempt.getAppAttemptId().getAttemptId();
    this.startTime = attempt.getStartTime();
    Container masterContainer = attempt.getMasterContainer();
    if (masterContainer != null) {
      this.containerId = masterContainer.getId().toString();
      this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
      this.nodeId = masterContainer.getNodeId().toString();
      this.logsLink =
          WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()), user);
    }
  }
}
 
Example 14
Source Project: big-c   Source File: RMContainerImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String getNodeHttpAddress() {
  try {
    readLock.lock();
    if (container.getNodeHttpAddress() != null) {
      StringBuilder httpAddress = new StringBuilder();
      httpAddress.append(WebAppUtils.getHttpSchemePrefix(rmContext
          .getYarnConfiguration()));
      httpAddress.append(container.getNodeHttpAddress());
      return httpAddress.toString();
    } else {
      return null;
    }
  } finally {
    readLock.unlock();
  }
}
 
Example 15
Source Project: big-c   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
private String generateProxyUriWithScheme() {
  this.readLock.lock();
  try {
    final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
    String proxy = WebAppUtils.getProxyHostAndPort(conf);
    URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
    URI result = ProxyUriUtils.getProxyUri(null, proxyUri,
        applicationAttemptId.getApplicationId());
    return result.toASCIIString();
  } catch (URISyntaxException e) {
    LOG.warn("Could not proxify the uri for "
        + applicationAttemptId.getApplicationId(), e);
    return null;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 16
Source Project: big-c   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  if (this.rmContext.isHAEnabled()) {
    transitionToStandby(true);
  } else {
    transitionToActive();
  }

  startWepApp();
  if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
      false)) {
    int port = webApp.port();
    WebAppUtils.setRMWebAppPort(conf, port);
  }
  super.serviceStart();
}
 
Example 17
Source Project: big-c   Source File: ApplicationHistoryManagerImpl.java    License: Apache License 2.0 6 votes vote down vote up
private ContainerReport convertToContainerReport(
    ContainerHistoryData containerHistory, String user) {
  // If the container has the aggregated log, add the server root url
  String logUrl = WebAppUtils.getAggregatedLogURL(
      serverHttpAddress,
      containerHistory.getAssignedNode().toString(),
      containerHistory.getContainerId().toString(),
      containerHistory.getContainerId().toString(),
      user);
  return ContainerReport.newInstance(containerHistory.getContainerId(),
    containerHistory.getAllocatedResource(),
    containerHistory.getAssignedNode(), containerHistory.getPriority(),
    containerHistory.getStartTime(), containerHistory.getFinishTime(),
    containerHistory.getDiagnosticsInfo(), logUrl,
    containerHistory.getContainerExitStatus(),
    containerHistory.getContainerState(), null);
}
 
Example 18
Source Project: attic-apex-core   Source File: SecurityUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Modify config object by adding SSL related parameters into a resource for WebApp's use
 *
 * @param config  Configuration to be modified
 * @param sslConfig
 */
private static void addSSLConfigResource(Configuration config, SSLConfig sslConfig)
{
  String nodeLocalConfig = sslConfig.getConfigPath();
  if (StringUtils.isNotEmpty(nodeLocalConfig)) {
    config.addResource(new Path(nodeLocalConfig));
  } else {
    // create a configuration object and add it as a resource
    Configuration sslConfigResource = new Configuration(false);
    final String SSL_CONFIG_LONG_NAME = Context.DAGContext.SSL_CONFIG.getLongName();
    sslConfigResource.set(SSL_SERVER_KEYSTORE_LOCATION, new Path(sslConfig.getKeyStorePath()).getName(), SSL_CONFIG_LONG_NAME);
    sslConfigResource.set(WebAppUtils.WEB_APP_KEYSTORE_PASSWORD_KEY, sslConfig.getKeyStorePassword(), SSL_CONFIG_LONG_NAME);
    sslConfigResource.set(WebAppUtils.WEB_APP_KEY_PASSWORD_KEY, sslConfig.getKeyStoreKeyPassword(), SSL_CONFIG_LONG_NAME);
    config.addResource(sslConfigResource);
  }
}
 
Example 19
Source Project: hadoop   Source File: TestYarnConfiguration.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDefaultRMWebUrl() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  String rmWebUrl = WebAppUtils.getRMWebAppURLWithScheme(conf);
  // shouldn't have a "/" on the end of the url as all the other uri routinnes
  // specifically add slashes and Jetty doesn't handle double slashes.
  Assert.assertNotSame("RM Web Url is not correct", "http://0.0.0.0:8088",
      rmWebUrl);
}
 
Example 20
Source Project: hadoop   Source File: WebAppProxyServlet.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Default constructor
 */
public WebAppProxyServlet() {
  super();
  conf = new YarnConfiguration();
  this.trackingUriPlugins =
      conf.getInstances(YarnConfiguration.YARN_TRACKING_URL_GENERATOR,
          TrackingUriPlugin.class);
  this.rmAppPageUrlBase = StringHelper.pjoin(
      WebAppUtils.getResolvedRMWebAppURLWithScheme(conf), "cluster", "app");
}
 
Example 21
Source Project: hadoop   Source File: WebAppProxy.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  String auth =  conf.get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION);
  if (auth == null || "simple".equals(auth)) {
    isSecurityEnabled = false;
  } else if ("kerberos".equals(auth)) {
    isSecurityEnabled = true;
  } else {
    LOG.warn("Unrecongized attribute value for " +
        CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION +
        " of " + auth);
  }
  String proxy = WebAppUtils.getProxyHostAndPort(conf);
  String[] proxyParts = proxy.split(":");
  proxyHost = proxyParts[0];

  fetcher = new AppReportFetcher(conf);
  bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
  if(bindAddress == null || bindAddress.isEmpty()) {
    throw new YarnRuntimeException(YarnConfiguration.PROXY_ADDRESS + 
        " is not set so the proxy will not run.");
  }
  LOG.info("Instantiating Proxy at " + bindAddress);
  String[] parts = StringUtils.split(bindAddress, ':');
  port = 0;
  if (parts.length == 2) {
    bindAddress = parts[0];
    port = Integer.parseInt(parts[1]);
  }
  acl = new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL, 
      YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
  super.serviceInit(conf);
}
 
Example 22
Source Project: hadoop   Source File: WebAppProxy.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  try {
    Configuration conf = getConfig();
    HttpServer2.Builder b = new HttpServer2.Builder()
        .setName("proxy")
        .addEndpoint(
            URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
                + ":" + port)).setFindPort(port == 0).setConf(getConfig())
        .setACL(acl);
    if (YarnConfiguration.useHttps(conf)) {
      WebAppUtils.loadSslConfiguration(b);
    }
    proxyServer = b.build();
    proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
        ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);
    proxyServer.setAttribute(FETCHER_ATTRIBUTE, fetcher);
    proxyServer
        .setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, isSecurityEnabled);
    proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
    proxyServer.start();
  } catch (IOException e) {
    LOG.error("Could not start proxy web server",e);
    throw e;
  }
  super.serviceStart();
}
 
Example 23
Source Project: hadoop   Source File: TestWebAppProxyServlet.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  String bindAddress = conf.get(YarnConfiguration.PROXY_ADDRESS);
  bindAddress = StringUtils.split(bindAddress, ':')[0];
  AccessControlList acl = new AccessControlList(
      conf.get(YarnConfiguration.YARN_ADMIN_ACL, 
      YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
  proxyServer = new HttpServer2.Builder()
      .setName("proxy")
      .addEndpoint(
          URI.create(WebAppUtils.getHttpSchemePrefix(conf) + bindAddress
              + ":0")).setFindPort(true)
      .setConf(conf)
      .setACL(acl)
      .build();
  proxyServer.addServlet(ProxyUriUtils.PROXY_SERVLET_NAME,
      ProxyUriUtils.PROXY_PATH_SPEC, WebAppProxyServlet.class);

  appReportFetcher = new AppReportFetcherForTest(conf);
  proxyServer.setAttribute(FETCHER_ATTRIBUTE,
      appReportFetcher );
  proxyServer.setAttribute(IS_SECURITY_ENABLED_ATTRIBUTE, Boolean.TRUE);
  
  String proxy = WebAppUtils.getProxyHostAndPort(conf);
  String[] proxyParts = proxy.split(":");
  String proxyHost = proxyParts[0];
  
  proxyServer.setAttribute(PROXY_HOST_ATTRIBUTE, proxyHost);
  proxyServer.start();
  LOG.info("Proxy server is started at port {}",
      proxyServer.getConnectorAddress(0).getPort());
}
 
Example 24
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 5 votes vote down vote up
private void setNonHARMConfiguration(Configuration conf) {
  String hostname = MiniYARNCluster.getHostname();
  conf.set(YarnConfiguration.RM_ADDRESS, hostname + ":0");
  conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, hostname + ":0");
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, hostname + ":0");
  conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, hostname + ":0");
  WebAppUtils.setRMWebAppHostnameAndPort(conf, hostname, 0);
}
 
Example 25
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void startResourceManager(final int index) {
  try {
    Thread rmThread = new Thread() {
      public void run() {
        resourceManagers[index].start();
      }
    };
    rmThread.setName("RM-" + index);
    rmThread.start();
    int waitCount = 0;
    while (resourceManagers[index].getServiceState() == STATE.INITED
        && waitCount++ < 60) {
      LOG.info("Waiting for RM to start...");
      Thread.sleep(1500);
    }
    if (resourceManagers[index].getServiceState() != STATE.STARTED) {
      // RM could have failed.
      throw new IOException(
          "ResourceManager failed to start. Final state is "
              + resourceManagers[index].getServiceState());
    }
  } catch (Throwable t) {
    throw new YarnRuntimeException(t);
  }
  LOG.info("MiniYARN ResourceManager address: " +
      getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniYARN ResourceManager web address: " +
      WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
}
 
Example 26
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected synchronized void serviceStart() throws Exception {
  startResourceManager(index);
  LOG.info("MiniYARN ResourceManager address: " +
           getConfig().get(YarnConfiguration.RM_ADDRESS));
  LOG.info("MiniYARN ResourceManager web address: " +
           WebAppUtils.getRMWebAppURLWithoutScheme(getConfig()));
  super.serviceStart();
}
 
Example 27
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 5 votes vote down vote up
protected synchronized void serviceInit(Configuration conf)
    throws Exception {
  Configuration config = new YarnConfiguration(conf);
  // create nm-local-dirs and configure them for the nodemanager
  String localDirsString = prepareDirs("local", numLocalDirs);
  config.set(YarnConfiguration.NM_LOCAL_DIRS, localDirsString);
  // create nm-log-dirs and configure them for the nodemanager
  String logDirsString = prepareDirs("log", numLogDirs);
  config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString);

  config.setInt(YarnConfiguration.NM_PMEM_MB, config.getInt(
      YarnConfiguration.YARN_MINICLUSTER_NM_PMEM_MB,
      YarnConfiguration.DEFAULT_YARN_MINICLUSTER_NM_PMEM_MB));

  config.set(YarnConfiguration.NM_ADDRESS,
      MiniYARNCluster.getHostname() + ":0");
  config.set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
      MiniYARNCluster.getHostname() + ":0");
  WebAppUtils
      .setNMWebAppHostNameAndPort(config,
          MiniYARNCluster.getHostname(), 0);

  // Disable resource checks by default
  if (!config.getBoolean(
      YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
      YarnConfiguration.
          DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
    config.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    config.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
  }

  LOG.info("Starting NM: " + index);
  nodeManagers[index].init(config);
  super.serviceInit(config);
}
 
Example 28
Source Project: hadoop   Source File: NavBlock.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void render(Block html) {
	
  String RMWebAppURL =
      WebAppUtils.getResolvedRemoteRMWebAppURLWithScheme(this.conf);
 html
    .div("#nav")
    .h3()._("ResourceManager")._()
      .ul()
        .li().a(RMWebAppURL, "RM Home")._()._()
    .h3()._("NodeManager")._() // TODO: Problem if no header like this
      .ul()
        .li()
          .a(url("node"), "Node Information")._()
        .li()
          .a(url("allApplications"), "List of Applications")
          ._()
        .li()
          .a(url("allContainers"), "List of Containers")._()
      ._()
    .h3("Tools")
      .ul()
        .li().a("/conf", "Configuration")._()
        .li().a("/logs", "Local logs")._()
        .li().a("/stacks", "Server stacks")._()
        .li().a("/jmx?qry=Hadoop:*", "Server metrics")._()._()._();
}
 
Example 29
Source Project: hadoop   Source File: RMContainerImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String getLogURL() {
  try {
    readLock.lock();
    StringBuilder logURL = new StringBuilder();
    logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
        .getYarnConfiguration()));
    logURL.append(WebAppUtils.getRunningLogURL(
        container.getNodeHttpAddress(), ConverterUtils.toString(containerId),
        user));
    return logURL.toString();
  } finally {
    readLock.unlock();
  }
}
 
Example 30
Source Project: hadoop   Source File: RMAppImpl.java    License: Apache License 2.0 5 votes vote down vote up
private String getDefaultProxyTrackingUrl() {
  try {
    final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
    String proxy = WebAppUtils.getProxyHostAndPort(conf);
    URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
    URI result = ProxyUriUtils.getProxyUri(null, proxyUri, applicationId);
    return result.toASCIIString();
  } catch (URISyntaxException e) {
    LOG.warn("Could not generate default proxy tracking URL for "
        + applicationId);
    return UNAVAILABLE;
  }
}