org.apache.hadoop.yarn.ipc.YarnRPC Java Examples

The following examples show how to use org.apache.hadoop.yarn.ipc.YarnRPC. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestJHSSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private MRClientProtocol getMRClientProtocol(Token token,
    final InetSocketAddress hsAddress, String user, final Configuration conf) {
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress));

  final YarnRPC rpc = YarnRPC.create(conf);
  MRClientProtocol hsWithDT = ugi
      .doAs(new PrivilegedAction<MRClientProtocol>() {

        @Override
        public MRClientProtocol run() {
          return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
              hsAddress, conf);
        }
      });
  return hsWithDT;
}
 
Example #2
Source File: SharedCacheUploaderService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = SharedCacheUploaderMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMUploaderProtocol.class, this, bindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_UPLOADER_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  bindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 
Example #3
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void stopContainer(YarnRPC rpc, Token nmToken,
    List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
    NodeId nodeId) throws Exception {
  StopContainersRequest request =
      StopContainersRequest.newInstance(containerId);
  ContainerManagementProtocol proxy = null;
  try {
    proxy =
        getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
            appAttemptId.toString());
    StopContainersResponse response = proxy.stopContainers(request);
    if (response.getFailedRequests() != null &&
        response.getFailedRequests().containsKey(containerId)) {
      parseAndThrowException(response.getFailedRequests().get(containerId)
          .deSerialize());
    }
  } catch (Exception e) {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #4
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private String testStartContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    boolean isExceptionExpected) {
  try {
    startContainer(rpc, nmToken, containerToken, nodeId,
        appAttemptId.toString());
    if (isExceptionExpected){
      fail("Exception was expected!!");        
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #5
Source File: YarnResourceAllocator.java    From incubator-tajo with Apache License 2.0 6 votes vote down vote up
@Override
public void init(Configuration conf) {
  systemConf = (TajoConf)conf;

  yarnRPC = YarnRPC.create(systemConf);

  connectYarnClient();

  taskRunnerLauncher = new YarnTaskRunnerLauncherImpl(queryTaskContext, yarnRPC);
  addService((Service) taskRunnerLauncher);
  queryTaskContext.getDispatcher().register(TaskRunnerGroupEvent.EventType.class, taskRunnerLauncher);

  rmAllocator = new YarnRMContainerAllocator(queryTaskContext);
  addService(rmAllocator);
  queryTaskContext.getDispatcher().register(ContainerAllocatorEventType.class, rmAllocator);
  super.init(conf);
}
 
Example #6
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private String testStopContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
  try {
    stopContainer(rpc, nmToken,
        Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
        nodeId);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #7
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private String testGetContainer(YarnRPC rpc,
    ApplicationAttemptId appAttemptId, NodeId nodeId,
    ContainerId containerId,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    boolean isExceptionExpected) {
  try {
    getContainerStatus(rpc, nmToken, containerId, appAttemptId, nodeId,
        isExceptionExpected);
    if (isExceptionExpected) {
      fail("Exception was expected!!");
    }
    return "";
  } catch (Exception e) {
    e.printStackTrace();
    return e.getMessage();
  }
}
 
Example #8
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void stopContainer(YarnRPC rpc, Token nmToken,
    List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
    NodeId nodeId) throws Exception {
  StopContainersRequest request =
      StopContainersRequest.newInstance(containerId);
  ContainerManagementProtocol proxy = null;
  try {
    proxy =
        getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
            appAttemptId.toString());
    StopContainersResponse response = proxy.stopContainers(request);
    if (response.getFailedRequests() != null &&
        response.getFailedRequests().containsKey(containerId)) {
      parseAndThrowException(response.getFailedRequests().get(containerId)
          .deSerialize());
    }
  } catch (Exception e) {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #9
Source File: TestContainerManagerSecurity.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #10
Source File: ClientProtocolService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = ClientSCMMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(ClientSCMProtocol.class, this,
          clientBindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_CLIENT_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  clientBindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 
Example #11
Source File: TestContainerManagerSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void startContainer(final YarnRPC rpc,
    org.apache.hadoop.yarn.api.records.Token nmToken,
    org.apache.hadoop.yarn.api.records.Token containerToken,
    NodeId nodeId, String user) throws Exception {

  ContainerLaunchContext context =
      Records.newRecord(ContainerLaunchContext.class);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(context,containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  ContainerManagementProtocol proxy = null;
  try {
    proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
    StartContainersResponse response = proxy.startContainers(allRequests);
    for(SerializedException ex : response.getFailedRequests().values()){
      parseAndThrowException(ex.deSerialize());
    }
  } finally {
    if (proxy != null) {
      rpc.stopProxy(proxy, conf);
    }
  }
}
 
Example #12
Source File: SharedCacheUploaderService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = SharedCacheUploaderMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMUploaderProtocol.class, this, bindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_UPLOADER_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  bindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 
Example #13
Source File: MRDelegationTokenRenewer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected MRClientProtocol instantiateHistoryProxy(final Configuration conf,
    final InetSocketAddress hsAddress) throws IOException {

  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to MRHistoryServer at: " + hsAddress);
  }
  final YarnRPC rpc = YarnRPC.create(conf);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          hsAddress, conf);
    }
  });
}
 
Example #14
Source File: YarnTajoResourceManager.java    From incubator-tajo with Apache License 2.0 6 votes vote down vote up
@Override
public void init(Configuration conf) {
  this.conf = conf;
  connectYarnClient();

  final YarnConfiguration yarnConf = new YarnConfiguration(conf);
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress rmAddress = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser;
  try {
    currentUser = UserGroupInformation.getCurrentUser();
  } catch (IOException e) {
    throw new YarnRuntimeException(e);
  }

  rmClient = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
    @Override
    public ApplicationMasterProtocol run() {
      return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmAddress, yarnConf);
    }
  });
}
 
Example #15
Source File: TestPBLocalizerRPC.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testLocalizerRPC() throws Exception {
  InetSocketAddress locAddr = new InetSocketAddress("0.0.0.0", 8040);
  LocalizerService server = new LocalizerService(locAddr);
  try {
    server.start();
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    LocalizationProtocol client = (LocalizationProtocol)
      rpc.getProxy(LocalizationProtocol.class, locAddr, conf);
    LocalizerStatus status =
      recordFactory.newRecordInstance(LocalizerStatus.class);
    status.setLocalizerId("localizer0");
    LocalizerHeartbeatResponse response = client.heartbeat(status);
    assertEquals(dieHBResponse(), response);
  } finally {
    server.stop();
  }
  assertTrue(true);
}
 
Example #16
Source File: TestJHSSecurity.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private MRClientProtocol getMRClientProtocol(Token token,
    final InetSocketAddress hsAddress, String user, final Configuration conf) {
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress));

  final YarnRPC rpc = YarnRPC.create(conf);
  MRClientProtocol hsWithDT = ugi
      .doAs(new PrivilegedAction<MRClientProtocol>() {

        @Override
        public MRClientProtocol run() {
          return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
              hsAddress, conf);
        }
      });
  return hsWithDT;
}
 
Example #17
Source File: TestClientRedirect.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void start(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  //TODO : use fixed port ??
  InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
  InetAddress hostNameResolved = null;
  try {
    address.getAddress();
    hostNameResolved = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    throw new YarnRuntimeException(e);
  }

  server =
      rpc.getServer(protocol, this, address,
          conf, null, 1);
  server.start();
  this.bindAddress = NetUtils.getConnectAddress(server);
   super.start();
   amRunning = true;
}
 
Example #18
Source File: TestClientSCMProtocolService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new ClientProtocolService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_PORT);

  clientSCMProxy =
      (ClientSCMProtocol) rpc.getProxy(ClientSCMProtocol.class, scmAddress,
          conf);
}
 
Example #19
Source File: TestClientRedirect.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void start(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  //TODO : use fixed port ??
  InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
  InetAddress hostNameResolved = null;
  try {
    address.getAddress();
    hostNameResolved = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    throw new YarnRuntimeException(e);
  }

  server =
      rpc.getServer(protocol, this, address,
          conf, null, 1);
  server.start();
  this.bindAddress = NetUtils.getConnectAddress(server);
   super.start();
   amRunning = true;
}
 
Example #20
Source File: SCMAdminProtocolService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMAdminProtocol.class, this,
          clientBindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT));

  // TODO: Enable service authorization (see YARN-2774)

  this.server.start();
  clientBindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 
Example #21
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 6 votes vote down vote up
Server createServer() {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    secretManager = new LocalizerTokenSecretManager();      
  }
  
  Server server = rpc.getServer(LocalizationProtocol.class, this,
      localizationServerAddress, conf, secretManager, 
      conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
          YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    server.refreshServiceAcl(conf, new NMPolicyProvider());
  }
  
  return server;
}
 
Example #22
Source File: MRDelegationTokenRenewer.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected MRClientProtocol instantiateHistoryProxy(final Configuration conf,
    final InetSocketAddress hsAddress) throws IOException {

  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to MRHistoryServer at: " + hsAddress);
  }
  final YarnRPC rpc = YarnRPC.create(conf);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          hsAddress, conf);
    }
  });
}
 
Example #23
Source File: ClientCache.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected MRClientProtocol instantiateHistoryProxy()
    throws IOException {
  final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
  if (StringUtils.isEmpty(serviceAddr)) {
    return null;
  }
  LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
  final YarnRPC rpc = YarnRPC.create(conf);
  LOG.debug("Connected to HistoryServer at: " + serviceAddr);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), conf);
    }
  });
}
 
Example #24
Source File: SCMAdminProtocolService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMAdminProtocol.class, this,
          clientBindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT));

  // TODO: Enable service authorization (see YARN-2774)

  this.server.start();
  clientBindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 
Example #25
Source File: ContainerManagementProtocolProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ContainerManagementProtocolProxy(Configuration conf,
    NMTokenCache nmTokenCache) {
  this.conf = new Configuration(conf);
  this.nmTokenCache = nmTokenCache;

  maxConnectedNMs =
      conf.getInt(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
          YarnConfiguration.DEFAULT_NM_CLIENT_MAX_NM_PROXIES);
  if (maxConnectedNMs < 0) {
    throw new YarnRuntimeException(
        YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES
            + " (" + maxConnectedNMs + ") can not be less than 0.");
  }
  LOG.info(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " : "
      + maxConnectedNMs);

  if (maxConnectedNMs > 0) {
    cmProxy =
        new LinkedHashMap<String, ContainerManagementProtocolProxyData>();
  } else {
    cmProxy = Collections.emptyMap();
    // Connections are not being cached so ensure connections close quickly
    // to avoid creating thousands of RPC client threads on large clusters.
    this.conf.setInt(
        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
        0);
  }
  rpc = YarnRPC.create(conf);
}
 
Example #26
Source File: ContainerManagementProtocolProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
public ContainerManagementProtocolProxyData(YarnRPC rpc,
    String containerManagerBindAddr,
    ContainerId containerId, Token token) throws InvalidToken {
  this.containerManagerBindAddr = containerManagerBindAddr;
  ;
  this.activeCallers = 0;
  this.scheduledForClose = false;
  this.token = token;
  this.proxy = newProxy(rpc, containerManagerBindAddr, containerId, token);
}
 
Example #27
Source File: GetGroupsForTesting.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}
 
Example #28
Source File: HistoryClientService.java    From big-c with Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  initializeWebApp(conf);
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.MR_HISTORY_BIND_HOST,
      JHAdminConfig.MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
      JHAdminConfig.DEFAULT_MR_HISTORY_PORT);

  server =
      rpc.getServer(HSClientProtocol.class, protocolHandler, address,
          conf, jhsDTSecretManager,
          conf.getInt(JHAdminConfig.MR_HISTORY_CLIENT_THREAD_COUNT,
              JHAdminConfig.DEFAULT_MR_HISTORY_CLIENT_THREAD_COUNT));

  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
      false)) {
    server.refreshServiceAcl(conf, new ClientHSPolicyProvider());
  }
  
  server.start();
  this.bindAddress = conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_BIND_HOST,
                                            JHAdminConfig.MR_HISTORY_ADDRESS,
                                            JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
                                            server.getListenerAddress());
  LOG.info("Instantiated HistoryClientService at " + this.bindAddress);

  super.serviceStart();
}
 
Example #29
Source File: SCMAdmin.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected SCMAdminProtocol createSCMAdminProtocol() throws IOException {
  // Get the current configuration
  final YarnConfiguration conf = new YarnConfiguration(getConf());

  // Create the admin client
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  SCMAdminProtocol scmAdminProtocol =
      (SCMAdminProtocol) rpc.getProxy(SCMAdminProtocol.class, addr, conf);
  return scmAdminProtocol;
}
 
Example #30
Source File: MRClientService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(0);

  server =
      rpc.getServer(MRClientProtocol.class, protocolHandler, address,
          conf, appContext.getClientToAMTokenSecretManager(),
          conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
              MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
              MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new MRAMPolicyProvider());
  }

  server.start();
  this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
      server.getListenerAddress().getPort());
  LOG.info("Instantiated MRClientService at " + this.bindAddress);
  try {
    // Explicitly disabling SSL for map reduce task as we can't allow MR users
    // to gain access to keystore file for opening SSL listener. We can trust
    // RM/NM to issue SSL certificates but definitely not MR-AM as it is
    // running in user-land.
    webApp =
        WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
          .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
  super.serviceStart();
}