org.apache.hadoop.http.HttpConfig.Policy Java Examples

The following examples show how to use org.apache.hadoop.http.HttpConfig.Policy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: WebAppUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);      
  }
  return getResolvedAddress(address);
}
 
Example #2
Source File: WebAppUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);      
  }
  return getResolvedAddress(address);
}
 
Example #3
Source File: TestOzoneManagerHttpServer.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Parameters public static Collection<Object[]> policy() {
  Object[][] params = new Object[][] {
      {HttpConfig.Policy.HTTP_ONLY},
      {HttpConfig.Policy.HTTPS_ONLY},
      {HttpConfig.Policy.HTTP_AND_HTTPS} };
  return Arrays.asList(params);
}
 
Example #4
Source File: TestAMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRWebAppRedirection() throws Exception {

  String[] schemePrefix =
      { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
  for (String scheme : schemePrefix) {
    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
      @Override
      protected ClientService createClientService(AppContext context) {
        return new MRClientService(context);
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme
      .equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name()
        : Policy.HTTP_ONLY.name());
    webProxyBase = "/proxy/" + app.getAppID();
    conf.set("hadoop.http.filter.initializers",
      TestAMFilterInitializer.class.getName());
    Job job = app.submit(conf);
    String hostPort =
        NetUtils.getHostPortString(((MRClientService) app.getClientService())
          .getWebApp().getListenerAddress());
    URL httpUrl = new URL("http://" + hostPort + "/mapreduce");

    HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
    conn.setInstanceFollowRedirects(false);
    conn.connect();
    String expectedURL =
        scheme + conf.get(YarnConfiguration.PROXY_ADDRESS)
            + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce");
    Assert.assertEquals(expectedURL,
      conn.getHeaderField(HttpHeaders.LOCATION));
    Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY,
      conn.getResponseCode());
    app.waitForState(job, JobState.SUCCEEDED);
    app.verifyCompleted();
  }
}
 
Example #5
Source File: TestAMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRWebAppSSLDisabled() throws Exception {
  MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
    @Override
    protected ClientService createClientService(AppContext context) {
      return new MRClientService(context);
    }
  };
  Configuration conf = new Configuration();
  // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
  Job job = app.submit(conf);

  String hostPort =
      NetUtils.getHostPortString(((MRClientService) app.getClientService())
        .getWebApp().getListenerAddress());
  // http:// should be accessible
  URL httpUrl = new URL("http://" + hostPort);
  HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
  InputStream in = conn.getInputStream();
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  IOUtils.copyBytes(in, out, 1024);
  Assert.assertTrue(out.toString().contains("MapReduce Application"));

  // https:// is not accessible.
  URL httpsUrl = new URL("https://" + hostPort);
  try {
    HttpURLConnection httpsConn =
        (HttpURLConnection) httpsUrl.openConnection();
    httpsConn.getInputStream();
    Assert.fail("https:// is not accessible, expected to fail");
  } catch (Exception e) {
    Assert.assertTrue(e instanceof SSLException);
  }

  app.waitForState(job, JobState.SUCCEEDED);
  app.verifyCompleted();
}
 
Example #6
Source File: MRClientService.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(0);

  server =
      rpc.getServer(MRClientProtocol.class, protocolHandler, address,
          conf, appContext.getClientToAMTokenSecretManager(),
          conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
              MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
              MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new MRAMPolicyProvider());
  }

  server.start();
  this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
      server.getListenerAddress().getPort());
  LOG.info("Instantiated MRClientService at " + this.bindAddress);
  try {
    // Explicitly disabling SSL for map reduce task as we can't allow MR users
    // to gain access to keystore file for opening SSL listener. We can trust
    // RM/NM to issue SSL certificates but definitely not MR-AM as it is
    // running in user-land.
    webApp =
        WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
          .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
  super.serviceStart();
}
 
Example #7
Source File: WebAppUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  String rmId = null;
  if (HAUtil.isHAEnabled(conf)) {
    // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
    // the Active RM
    rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
  }

  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
  }
  return getResolvedAddress(address);
}
 
Example #8
Source File: MRClientService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(0);

  server =
      rpc.getServer(MRClientProtocol.class, protocolHandler, address,
          conf, appContext.getClientToAMTokenSecretManager(),
          conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
              MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
              MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new MRAMPolicyProvider());
  }

  server.start();
  this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
      server.getListenerAddress().getPort());
  LOG.info("Instantiated MRClientService at " + this.bindAddress);
  try {
    // Explicitly disabling SSL for map reduce task as we can't allow MR users
    // to gain access to keystore file for opening SSL listener. We can trust
    // RM/NM to issue SSL certificates but definitely not MR-AM as it is
    // running in user-land.
    webApp =
        WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
          .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
  super.serviceStart();
}
 
Example #9
Source File: TestAMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRWebAppSSLDisabled() throws Exception {
  MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
    @Override
    protected ClientService createClientService(AppContext context) {
      return new MRClientService(context);
    }
  };
  Configuration conf = new Configuration();
  // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
  Job job = app.submit(conf);

  String hostPort =
      NetUtils.getHostPortString(((MRClientService) app.getClientService())
        .getWebApp().getListenerAddress());
  // http:// should be accessible
  URL httpUrl = new URL("http://" + hostPort);
  HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
  InputStream in = conn.getInputStream();
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  IOUtils.copyBytes(in, out, 1024);
  Assert.assertTrue(out.toString().contains("MapReduce Application"));

  // https:// is not accessible.
  URL httpsUrl = new URL("https://" + hostPort);
  try {
    HttpURLConnection httpsConn =
        (HttpURLConnection) httpsUrl.openConnection();
    httpsConn.getInputStream();
    Assert.fail("https:// is not accessible, expected to fail");
  } catch (Exception e) {
    Assert.assertTrue(e instanceof SSLException);
  }

  app.waitForState(job, JobState.SUCCEEDED);
  app.verifyCompleted();
}
 
Example #10
Source File: TestAMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMRWebAppRedirection() throws Exception {

  String[] schemePrefix =
      { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
  for (String scheme : schemePrefix) {
    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
      @Override
      protected ClientService createClientService(AppContext context) {
        return new MRClientService(context);
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme
      .equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name()
        : Policy.HTTP_ONLY.name());
    webProxyBase = "/proxy/" + app.getAppID();
    conf.set("hadoop.http.filter.initializers",
      TestAMFilterInitializer.class.getName());
    Job job = app.submit(conf);
    String hostPort =
        NetUtils.getHostPortString(((MRClientService) app.getClientService())
          .getWebApp().getListenerAddress());
    URL httpUrl = new URL("http://" + hostPort + "/mapreduce");

    HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
    conn.setInstanceFollowRedirects(false);
    conn.connect();
    String expectedURL =
        scheme + conf.get(YarnConfiguration.PROXY_ADDRESS)
            + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce");
    Assert.assertEquals(expectedURL,
      conn.getHeaderField(HttpHeaders.LOCATION));
    Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY,
      conn.getResponseCode());
    app.waitForState(job, JobState.SUCCEEDED);
    app.verifyCompleted();
  }
}
 
Example #11
Source File: TestStorageContainerManagerHttpServer.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up
@Parameters public static Collection<Object[]> policy() {
  Object[][] params = new Object[][] {
      {HttpConfig.Policy.HTTP_ONLY},
      {HttpConfig.Policy.HTTPS_ONLY},
      {HttpConfig.Policy.HTTP_AND_HTTPS} };
  return Arrays.asList(params);
}
 
Example #12
Source File: WebAppUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  String rmId = null;
  if (HAUtil.isHAEnabled(conf)) {
    // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
    // the Active RM
    rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
  }

  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
  }
  return getResolvedAddress(address);
}
 
Example #13
Source File: TestNameNodeHttpServer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Parameters
public static Collection<Object[]> policy() {
  Object[][] params = new Object[][] { { HttpConfig.Policy.HTTP_ONLY },
      { HttpConfig.Policy.HTTPS_ONLY }, { HttpConfig.Policy.HTTP_AND_HTTPS } };
  return Arrays.asList(params);
}
 
Example #14
Source File: TestNameNodeHttpServer.java    From big-c with Apache License 2.0 4 votes vote down vote up
public TestNameNodeHttpServer(Policy policy) {
  super();
  this.policy = policy;
}
 
Example #15
Source File: TestNameNodeHttpServer.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Parameters
public static Collection<Object[]> policy() {
  Object[][] params = new Object[][] { { HttpConfig.Policy.HTTP_ONLY },
      { HttpConfig.Policy.HTTPS_ONLY }, { HttpConfig.Policy.HTTP_AND_HTTPS } };
  return Arrays.asList(params);
}
 
Example #16
Source File: WebAppUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf) {
  return getResolvedRMWebAppURLWithoutScheme(conf,
      YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
}
 
Example #17
Source File: WebAppUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up
public static String getResolvedRemoteRMWebAppURLWithoutScheme(
    Configuration conf) {
  return getResolvedRemoteRMWebAppURLWithoutScheme(conf,
      YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
}
 
Example #18
Source File: WebApps.java    From big-c with Apache License 2.0 4 votes vote down vote up
public Builder<T> withHttpPolicy(Configuration conf, Policy httpPolicy) {
  this.conf = conf;
  this.httpPolicy = httpPolicy;
  return this;
}
 
Example #19
Source File: TestNameNodeHttpServer.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public TestNameNodeHttpServer(Policy policy) {
  super();
  this.policy = policy;
}
 
Example #20
Source File: WebAppUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf) {
  return getResolvedRMWebAppURLWithoutScheme(conf,
      YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
}
 
Example #21
Source File: WebAppUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public static String getResolvedRemoteRMWebAppURLWithoutScheme(
    Configuration conf) {
  return getResolvedRemoteRMWebAppURLWithoutScheme(conf,
      YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY : Policy.HTTP_ONLY);
}
 
Example #22
Source File: WebApps.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public Builder<T> withHttpPolicy(Configuration conf, Policy httpPolicy) {
  this.conf = conf;
  this.httpPolicy = httpPolicy;
  return this;
}
 
Example #23
Source File: TestOzoneManagerHttpServer.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up
public TestOzoneManagerHttpServer(Policy policy) {
  super();
  this.policy = policy;
}
 
Example #24
Source File: TestStorageContainerManagerHttpServer.java    From hadoop-ozone with Apache License 2.0 4 votes vote down vote up
public TestStorageContainerManagerHttpServer(Policy policy) {
  super();
  this.policy = policy;
}