Java Code Examples for org.apache.hadoop.yarn.ipc.YarnRPC#getServer()

The following examples show how to use org.apache.hadoop.yarn.ipc.YarnRPC#getServer() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ResourceLocalizationService.java From big-c with Apache License 2.0

6 votes

Server createServer() {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    secretManager = new LocalizerTokenSecretManager();      
  }
  
  Server server = rpc.getServer(LocalizationProtocol.class, this,
      localizationServerAddress, conf, secretManager, 
      conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
          YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    server.refreshServiceAcl(conf, new NMPolicyProvider());
  }
  
  return server;
}

Example 2

Source File: SharedCacheUploaderService.java From big-c with Apache License 2.0

6 votes

@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = SharedCacheUploaderMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMUploaderProtocol.class, this, bindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_UPLOADER_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  bindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}

Example 3

Source File: SharedCacheUploaderService.java From hadoop with Apache License 2.0

6 votes

@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = SharedCacheUploaderMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMUploaderProtocol.class, this, bindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_UPLOADER_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  bindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}

Example 4

Source File: ClientProtocolService.java From hadoop with Apache License 2.0

6 votes

@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = ClientSCMMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(ClientSCMProtocol.class, this,
          clientBindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_CLIENT_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  clientBindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}

Example 5

Source File: SCMAdminProtocolService.java From hadoop with Apache License 2.0

6 votes

@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMAdminProtocol.class, this,
          clientBindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT));

  // TODO: Enable service authorization (see YARN-2774)

  this.server.start();
  clientBindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}

Example 6

Source File: ResourceLocalizationService.java From hadoop with Apache License 2.0

6 votes

Server createServer() {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    secretManager = new LocalizerTokenSecretManager();      
  }
  
  Server server = rpc.getServer(LocalizationProtocol.class, this,
      localizationServerAddress, conf, secretManager, 
      conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
          YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    server.refreshServiceAcl(conf, new NMPolicyProvider());
  }
  
  return server;
}

Example 7

Source File: TestClientRedirect.java From big-c with Apache License 2.0

6 votes

public void start(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  //TODO : use fixed port ??
  InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
  InetAddress hostNameResolved = null;
  try {
    address.getAddress();
    hostNameResolved = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    throw new YarnRuntimeException(e);
  }

  server =
      rpc.getServer(protocol, this, address,
          conf, null, 1);
  server.start();
  this.bindAddress = NetUtils.getConnectAddress(server);
   super.start();
   amRunning = true;
}

Example 8

Source File: MRClientService.java From big-c with Apache License 2.0

5 votes

protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(0);

  server =
      rpc.getServer(MRClientProtocol.class, protocolHandler, address,
          conf, appContext.getClientToAMTokenSecretManager(),
          conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
              MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
              MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new MRAMPolicyProvider());
  }

  server.start();
  this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
      server.getListenerAddress().getPort());
  LOG.info("Instantiated MRClientService at " + this.bindAddress);
  try {
    // Explicitly disabling SSL for map reduce task as we can't allow MR users
    // to gain access to keystore file for opening SSL listener. We can trust
    // RM/NM to issue SSL certificates but definitely not MR-AM as it is
    // running in user-land.
    webApp =
        WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
          .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
  super.serviceStart();
}

Example 9

Source File: HistoryClientService.java From hadoop with Apache License 2.0

5 votes

protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  initializeWebApp(conf);
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.MR_HISTORY_BIND_HOST,
      JHAdminConfig.MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_PORT);

  server =
      rpc.getServer(HSClientProtocol.class, protocolHandler, address,
          conf, jhsDTSecretManager,
          conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
              JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
      false)) {
    server.refreshServiceAcl(conf, new ClientHSPolicyProvider());
  }
  
  server.start();
  this.bindAddress = conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_BIND_HOST,
                                            JHAdminConfig.MR_HISTORY_ADDRESS,
                                            JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
                                            server.getListenerAddress());
  LOG.info("Instantiated HistoryClientService at " + this.bindAddress);

  super.serviceStart();
}

Example 10

Source File: ApplicationHistoryClientService.java From big-c with Apache License 2.0

5 votes

protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = conf.getSocketAddr(
      YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
      YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);

  Preconditions.checkArgument(conf.getInt(
      YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0,
      "%s property value should be greater than zero",
      YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT);

  server =
      rpc.getServer(ApplicationHistoryProtocol.class, this,
        address, conf, null, conf.getInt(
          YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
    refreshServiceAcls(conf, new TimelinePolicyProvider());
  }

  server.start();
  this.bindAddress =
      conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
                             YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
                             YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
                             server.getListenerAddress());
  LOG.info("Instantiated ApplicationHistoryClientService at "
      + this.bindAddress);

  super.serviceStart();
}

Example 11

Source File: MRClientService.java From hadoop with Apache License 2.0

5 votes

protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(0);

  server =
      rpc.getServer(MRClientProtocol.class, protocolHandler, address,
          conf, appContext.getClientToAMTokenSecretManager(),
          conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
              MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
              MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new MRAMPolicyProvider());
  }

  server.start();
  this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
      server.getListenerAddress().getPort());
  LOG.info("Instantiated MRClientService at " + this.bindAddress);
  try {
    // Explicitly disabling SSL for map reduce task as we can't allow MR users
    // to gain access to keystore file for opening SSL listener. We can trust
    // RM/NM to issue SSL certificates but definitely not MR-AM as it is
    // running in user-land.
    webApp =
        WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
          .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
  super.serviceStart();
}

Example 12

Source File: ApplicationHistoryClientService.java From hadoop with Apache License 2.0

5 votes

protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = conf.getSocketAddr(
      YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
      YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);

  Preconditions.checkArgument(conf.getInt(
      YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT) > 0,
      "%s property value should be greater than zero",
      YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT);

  server =
      rpc.getServer(ApplicationHistoryProtocol.class, this,
        address, conf, null, conf.getInt(
          YarnConfiguration.TIMELINE_SERVICE_HANDLER_THREAD_COUNT,
          YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) {
    refreshServiceAcls(conf, new TimelinePolicyProvider());
  }

  server.start();
  this.bindAddress =
      conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
                             YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
                             YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
                             server.getListenerAddress());
  LOG.info("Instantiated ApplicationHistoryClientService at "
      + this.bindAddress);

  super.serviceStart();
}

Example 13

Source File: ClientRMService.java From hadoop with Apache License 2.0

5 votes

@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server =   
    rpc.getServer(ApplicationClientProtocol.class, this,
          clientBindAddress,
          conf, this.rmDTSecretManager,
          conf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT, 
              YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    InputStream inputStream =
        this.rmContext.getConfigurationProvider()
            .getConfigurationInputStream(conf,
                YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
    if (inputStream != null) {
      conf.addResource(inputStream);
    }
    refreshServiceAcls(conf, RMPolicyProvider.getInstance());
  }
  
  this.server.start();
  clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST,
                                             YarnConfiguration.RM_ADDRESS,
                                             YarnConfiguration.DEFAULT_RM_ADDRESS,
                                             server.getListenerAddress());
  super.serviceStart();
}

Example 14

Source File: TestClientRedirect.java From big-c with Apache License 2.0

5 votes

@Override
protected void serviceStart() throws Exception {
  // All the clients to appsManager are supposed to be authenticated via
  // Kerberos if security is enabled, so no secretManager.
  YarnRPC rpc = YarnRPC.create(getConfig());
  Configuration clientServerConf = new Configuration(getConfig());
  this.server = rpc.getServer(ApplicationClientProtocol.class, this,
      clientBindAddress, clientServerConf, null, 1);
  this.server.start();
  super.serviceStart();
}

Example 15

Source File: ContainerManagerImpl.java From big-c with Apache License 2.0

4 votes

@Override
protected void serviceStart() throws Exception {

  // Enqueue user dirs in deletion context

  Configuration conf = getConfig();
  final InetSocketAddress initialAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_PORT);
  boolean usingEphemeralPort = (initialAddress.getPort() == 0);
  if (context.getNMStateStore().canRecover() && usingEphemeralPort) {
    throw new IllegalArgumentException("Cannot support recovery with an "
        + "ephemeral server port. Check the setting of "
        + YarnConfiguration.NM_ADDRESS);
  }
  // If recovering then delay opening the RPC service until the recovery
  // of resources and containers have completed, otherwise requests from
  // clients during recovery can interfere with the recovery process.
  final boolean delayedRpcServerStart =
      context.getNMStateStore().canRecover();

  Configuration serverConf = new Configuration(conf);

  // always enforce it to be token-based.
  serverConf.set(
    CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    SaslRpcServer.AuthMethod.TOKEN.toString());
  
  YarnRPC rpc = YarnRPC.create(conf);

  server =
      rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, 
          serverConf, this.context.getNMTokenSecretManager(),
          conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
              YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new NMPolicyProvider());
  }
  
  LOG.info("Blocking new container-requests as container manager rpc" +
  		" server is still starting.");
  this.setBlockNewContainerRequests(true);

  String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
  String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
  String hostOverride = null;
  if (bindHost != null && !bindHost.isEmpty()
      && nmAddress != null && !nmAddress.isEmpty()) {
    //a bind-host case with an address, to support overriding the first
    //hostname found when querying for our hostname with the specified
    //address, combine the specified address with the actual port listened
    //on by the server
    hostOverride = nmAddress.split(":")[0];
  }

  // setup node ID
  InetSocketAddress connectAddress;
  if (delayedRpcServerStart) {
    connectAddress = NetUtils.getConnectAddress(initialAddress);
  } else {
    server.start();
    connectAddress = NetUtils.getConnectAddress(server);
  }
  NodeId nodeId = buildNodeId(connectAddress, hostOverride);
  ((NodeManager.NMContext)context).setNodeId(nodeId);
  this.context.getNMTokenSecretManager().setNodeId(nodeId);
  this.context.getContainerTokenSecretManager().setNodeId(nodeId);

  // start remaining services
  super.serviceStart();

  if (delayedRpcServerStart) {
    waitForRecoveredContainers();
    server.start();

    // check that the node ID is as previously advertised
    connectAddress = NetUtils.getConnectAddress(server);
    NodeId serverNode = buildNodeId(connectAddress, hostOverride);
    if (!serverNode.equals(nodeId)) {
      throw new IOException("Node mismatch after server started, expected '"
          + nodeId + "' but found '" + serverNode + "'");
    }
  }

  LOG.info("ContainerManager started at " + connectAddress);
  LOG.info("ContainerManager bound to " + initialAddress);
}

Example 16

Source File: TestContainerLaunchRPC.java From big-c with Apache License 2.0

4 votes

private void testRPCTimeout(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);

  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class,
      new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  try {

    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(
        ContainerManagementProtocol.class,
        server.getListenerAddress(), conf);
    ContainerLaunchContext containerLaunchContext = recordFactory
        .newRecordInstance(ContainerLaunchContext.class);

    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId =
        ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId =
        ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2);
    ContainerTokenIdentifier containerTokenIdentifier =
        new ContainerTokenIdentifier(containerId, "localhost", "user",
          resource, System.currentTimeMillis() + 10000, 42, 42,
          Priority.newInstance(0), 0);
    Token containerToken =
        TestRPC.newContainerToken(nodeId, "password".getBytes(),
          containerTokenIdentifier);

    StartContainerRequest scRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests =
        StartContainersRequest.newInstance(list);
    try {
      proxy.startContainers(allRequests);
    } catch (Exception e) {
      LOG.info(StringUtils.stringifyException(e));
      Assert.assertEquals("Error, exception is not: "
          + SocketTimeoutException.class.getName(),
          SocketTimeoutException.class.getName(), e.getClass().getName());
      return;
    }
  } finally {
    server.stop();
  }

  Assert.fail("timeout exception should have occurred!");
}

Example 17

Source File: TestContainerLauncher.java From hadoop with Apache License 2.0

4 votes

@Test(timeout = 15000)
public void testSlowNM() throws Exception {

  conf = new Configuration();
  int maxAttempts = 1;
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts);
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);
  conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  NMTokenSecretManagerInNM tokenSecretManager =
      new NMTokenSecretManagerInNM();
  MasterKey masterKey = Records.newRecord(MasterKey.class);
  masterKey.setBytes(ByteBuffer.wrap("key".getBytes()));
  tokenSecretManager.setMasterKey(masterKey);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "token");
  server =
      rpc.getServer(ContainerManagementProtocol.class,
        new DummyContainerManager(), addr, conf, tokenSecretManager, 1);
  server.start();

  MRApp app = new MRAppWithSlowNM(tokenSecretManager);

  try {
  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);

  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());

  Task task = tasks.values().iterator().next();
  app.waitForState(task, TaskState.SCHEDULED);

  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator()
      .next().getAttempts();
    Assert.assertEquals("Num attempts is not correct", maxAttempts,
        attempts.size());

  TaskAttempt attempt = attempts.values().iterator().next();
    app.waitForInternalState((TaskAttemptImpl) attempt,
        TaskAttemptStateInternal.ASSIGNED);

  app.waitForState(job, JobState.FAILED);

  String diagnostics = attempt.getDiagnostics().toString();
  LOG.info("attempt.getDiagnostics: " + diagnostics);

    Assert.assertTrue(diagnostics.contains("Container launch failed for "
        + "container_0_0000_01_000000 : "));
    Assert
        .assertTrue(diagnostics
            .contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"));

  } finally {
    server.stop();
  app.stop();
}
}

Example 18

Source File: ContainerManagerImpl.java From hadoop with Apache License 2.0

4 votes

@Override
protected void serviceStart() throws Exception {

  // Enqueue user dirs in deletion context

  Configuration conf = getConfig();
  final InetSocketAddress initialAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_PORT);
  boolean usingEphemeralPort = (initialAddress.getPort() == 0);
  if (context.getNMStateStore().canRecover() && usingEphemeralPort) {
    throw new IllegalArgumentException("Cannot support recovery with an "
        + "ephemeral server port. Check the setting of "
        + YarnConfiguration.NM_ADDRESS);
  }
  // If recovering then delay opening the RPC service until the recovery
  // of resources and containers have completed, otherwise requests from
  // clients during recovery can interfere with the recovery process.
  final boolean delayedRpcServerStart =
      context.getNMStateStore().canRecover();

  Configuration serverConf = new Configuration(conf);

  // always enforce it to be token-based.
  serverConf.set(
    CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    SaslRpcServer.AuthMethod.TOKEN.toString());
  
  YarnRPC rpc = YarnRPC.create(conf);

  server =
      rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, 
          serverConf, this.context.getNMTokenSecretManager(),
          conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
              YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new NMPolicyProvider());
  }
  
  LOG.info("Blocking new container-requests as container manager rpc" +
  		" server is still starting.");
  this.setBlockNewContainerRequests(true);

  String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
  String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
  String hostOverride = null;
  if (bindHost != null && !bindHost.isEmpty()
      && nmAddress != null && !nmAddress.isEmpty()) {
    //a bind-host case with an address, to support overriding the first
    //hostname found when querying for our hostname with the specified
    //address, combine the specified address with the actual port listened
    //on by the server
    hostOverride = nmAddress.split(":")[0];
  }

  // setup node ID
  InetSocketAddress connectAddress;
  if (delayedRpcServerStart) {
    connectAddress = NetUtils.getConnectAddress(initialAddress);
  } else {
    server.start();
    connectAddress = NetUtils.getConnectAddress(server);
  }
  NodeId nodeId = buildNodeId(connectAddress, hostOverride);
  ((NodeManager.NMContext)context).setNodeId(nodeId);
  this.context.getNMTokenSecretManager().setNodeId(nodeId);
  this.context.getContainerTokenSecretManager().setNodeId(nodeId);

  // start remaining services
  super.serviceStart();

  if (delayedRpcServerStart) {
    waitForRecoveredContainers();
    server.start();

    // check that the node ID is as previously advertised
    connectAddress = NetUtils.getConnectAddress(server);
    NodeId serverNode = buildNodeId(connectAddress, hostOverride);
    if (!serverNode.equals(nodeId)) {
      throw new IOException("Node mismatch after server started, expected '"
          + nodeId + "' but found '" + serverNode + "'");
    }
  }

  LOG.info("ContainerManager started at " + connectAddress);
  LOG.info("ContainerManager bound to " + initialAddress);
}

Example 19

Source File: TestContainerLaunchRPC.java From hadoop with Apache License 2.0

4 votes

private void testRPCTimeout(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);

  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class,
      new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  try {

    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(
        ContainerManagementProtocol.class,
        server.getListenerAddress(), conf);
    ContainerLaunchContext containerLaunchContext = recordFactory
        .newRecordInstance(ContainerLaunchContext.class);

    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId =
        ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId =
        ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2, 3);
    ContainerTokenIdentifier containerTokenIdentifier =
        new ContainerTokenIdentifier(containerId, "localhost", "user",
          resource, System.currentTimeMillis() + 10000, 42, 42,
          Priority.newInstance(0), 0);
    Token containerToken =
        TestRPC.newContainerToken(nodeId, "password".getBytes(),
          containerTokenIdentifier);

    StartContainerRequest scRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests =
        StartContainersRequest.newInstance(list);
    try {
      proxy.startContainers(allRequests);
    } catch (Exception e) {
      LOG.info(StringUtils.stringifyException(e));
      Assert.assertEquals("Error, exception is not: "
          + SocketTimeoutException.class.getName(),
          SocketTimeoutException.class.getName(), e.getClass().getName());
      return;
    }
  } finally {
    server.stop();
  }

  Assert.fail("timeout exception should have occurred!");
}

Example 20

Source File: TestContainerLauncher.java From big-c with Apache License 2.0

4 votes

@Test(timeout = 15000)
public void testSlowNM() throws Exception {

  conf = new Configuration();
  int maxAttempts = 1;
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts);
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);
  conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  NMTokenSecretManagerInNM tokenSecretManager =
      new NMTokenSecretManagerInNM();
  MasterKey masterKey = Records.newRecord(MasterKey.class);
  masterKey.setBytes(ByteBuffer.wrap("key".getBytes()));
  tokenSecretManager.setMasterKey(masterKey);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "token");
  server =
      rpc.getServer(ContainerManagementProtocol.class,
        new DummyContainerManager(), addr, conf, tokenSecretManager, 1);
  server.start();

  MRApp app = new MRAppWithSlowNM(tokenSecretManager);

  try {
  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);

  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());

  Task task = tasks.values().iterator().next();
  app.waitForState(task, TaskState.SCHEDULED);

  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator()
      .next().getAttempts();
    Assert.assertEquals("Num attempts is not correct", maxAttempts,
        attempts.size());

  TaskAttempt attempt = attempts.values().iterator().next();
    app.waitForInternalState((TaskAttemptImpl) attempt,
        TaskAttemptStateInternal.ASSIGNED);

  app.waitForState(job, JobState.FAILED);

  String diagnostics = attempt.getDiagnostics().toString();
  LOG.info("attempt.getDiagnostics: " + diagnostics);

    Assert.assertTrue(diagnostics.contains("Container launch failed for "
        + "container_0_0000_01_000000 : "));
    Assert
        .assertTrue(diagnostics
            .contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"));

  } finally {
    server.stop();
  app.stop();
}
}