Java Code Examples for org.apache.hadoop.conf.Configuration#getSocketAddr()

The following examples show how to use org.apache.hadoop.conf.Configuration#getSocketAddr() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: big-c   File: WebAppUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);      
  }
  return getResolvedAddress(address);
}
 
Example 2
@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new SharedCacheUploaderService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);

  proxy =
      (SCMUploaderProtocol) rpc.getProxy(
          SCMUploaderProtocol.class, scmAddress, conf);
}
 
Example 3
Source Project: big-c   File: HSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
private int refreshLoadedJobCache() throws IOException {
  // Refresh the loaded job cache
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshLoadedJobCache();
  return 0;
}
 
Example 4
Source Project: big-c   File: HSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
private int refreshJobRetentionSettings() throws IOException {
  // Refresh job retention settings
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshJobRetentionSettings();
  return 0;
}
 
Example 5
Source Project: hadoop   File: WebAppUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  String rmId = null;
  if (HAUtil.isHAEnabled(conf)) {
    // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
    // the Active RM
    rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
  }

  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
  }
  return getResolvedAddress(address);
}
 
Example 6
Source Project: hadoop   File: MiniMRYarnCluster.java    License: Apache License 2.0 5 votes vote down vote up
public static String getResolvedMRHistoryWebAppURLWithoutScheme(
    Configuration conf, boolean isSSLEnabled) {
  InetSocketAddress address = null;
  if (isSSLEnabled) {
    address =
        conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);    }
  address = NetUtils.getConnectAddress(address);
  StringBuffer sb = new StringBuffer();
  InetAddress resolved = address.getAddress();
  if (resolved == null || resolved.isAnyLocalAddress() || 
      resolved.isLoopbackAddress()) {
    String lh = address.getHostName();
    try {
      lh = InetAddress.getLocalHost().getCanonicalHostName();
    } catch (UnknownHostException e) {
      //Ignore and fallback.
    }
    sb.append(lh);
  } else {
    sb.append(address.getHostName());
  }
  sb.append(":").append(address.getPort());
  return sb.toString();
}
 
Example 7
Source Project: hadoop   File: HSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
private int refreshLogRetentionSettings() throws IOException {
  // Refresh log retention settings
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshLogRetentionSettings();
  return 0;
}
 
Example 8
Source Project: hadoop   File: HSAdmin.java    License: Apache License 2.0 5 votes vote down vote up
private int getGroups(String[] usernames) throws IOException {
  // Get groups users belongs to
  if (usernames.length == 0) {
    usernames = new String[] { UserGroupInformation.getCurrentUser()
        .getUserName() };
  }

  // Get the current configuration
  Configuration conf = getConf();

  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  GetUserMappingsProtocol getUserMappingProtocol = HSProxies.createProxy(
      conf, address, GetUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  for (String username : usernames) {
    StringBuilder sb = new StringBuilder();
    sb.append(username + " :");
    for (String group : getUserMappingProtocol.getGroupsForUser(username)) {
      sb.append(" ");
      sb.append(group);
    }
    System.out.println(sb);
  }

  return 0;
}
 
Example 9
Source Project: hadoop   File: ClientRMService.java    License: Apache License 2.0 5 votes vote down vote up
InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(
          YarnConfiguration.RM_BIND_HOST,
          YarnConfiguration.RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_PORT);
}
 
Example 10
Source Project: big-c   File: ResourceTrackerService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}
 
Example 11
Source Project: big-c   File: GetGroupsForTesting.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}
 
Example 12
Source Project: tez   File: Master.java    License: Apache License 2.0 4 votes vote down vote up
public static InetSocketAddress getMasterAddress(Configuration conf) {
  return conf
      .getSocketAddr(YarnConfiguration.RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_PORT);
}
 
Example 13
@Test
public void testContainerLocalizer() throws Exception {
  if (!shouldRun()) {
    return;
  }
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  for (String localDir : localDirs) {
    Path userDir =
        new Path(localDir, ContainerLocalizer.USERCACHE);
    files.mkdir(userDir, new FsPermission("777"), false);
    // $local/filecache
    Path fileDir =
        new Path(localDir, ContainerLocalizer.FILECACHE);
    files.mkdir(fileDir, new FsPermission("777"), false);
  }
  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler.getLocalPathForWrite(
          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr = config.getSocketAddr(
    YarnConfiguration.NM_BIND_HOST,
    YarnConfiguration.NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user, String appId,
        String locId, InetSocketAddress nmAddr, List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
        localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
    appId, locId, dirsHandler);

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
    appId, locId2, dirsHandler);
}
 
Example 14
Source Project: hadoop   File: SCMAdminProtocolService.java    License: Apache License 2.0 4 votes vote down vote up
InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
}
 
Example 15
Source Project: big-c   File: TestAMAuthorization.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testUnauthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm = new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  RMApp app = rm.submitApp(1024);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 40) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress serviceAddr = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());

  // First try contacting NM without tokens
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
              serviceAddr, conf);
        }
      });
  
  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  try {
    client.registerApplicationMaster(request);
    Assert.fail("Should fail with authorization error");
  } catch (Exception e) {
    if (isCause(AccessControlException.class, e)) {
      // Because there are no tokens, the request should be rejected as the
      // server side will assume we are trying simple auth.
      String expectedMessage = "";
      if (UserGroupInformation.isSecurityEnabled()) {
        expectedMessage = "Client cannot authenticate via:[TOKEN]";
      } else {
        expectedMessage =
            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
      }
      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); 
    } else {
      throw e;
    }
  }

  // TODO: Add validation of invalid authorization when there's more data in
  // the AMRMToken
}
 
Example 16
Source Project: big-c   File: SCMAdminProtocolService.java    License: Apache License 2.0 4 votes vote down vote up
InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
}
 
Example 17
@Test
public void testContainerLocalizer() throws Exception {

  Assume.assumeTrue(shouldRun());

  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr =
      config.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user,
        String appId, String locId, InetSocketAddress nmAddr,
        List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId,
        nmAddr, localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(new LocalizerStartContext.Builder()
      .setNmPrivateContainerTokens(nmPrivateContainerTokensPath)
      .setNmAddr(nmAddr)
      .setUser(appSubmitter)
      .setAppId(appId)
      .setLocId(locId)
      .setDirsHandler(dirsHandler)
      .build());

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateContainerTokensPath2)
          .setNmAddr(nmAddr)
          .setUser(appSubmitter)
          .setAppId(appId)
          .setLocId(locId2)
          .setDirsHandler(dirsHandler)
          .build());


  cleanupUserAppCache(appSubmitter);
}
 
Example 18
Source Project: hadoop   File: ApplicationHistoryServer.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Retrieve the timeline server bind address from configuration
 *
 * @param conf
 * @return InetSocketAddress
 */
private static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
}
 
Example 19
Source Project: big-c   File: WebAppProxyServer.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Retrieve PROXY bind address from configuration
 *
 * @param conf
 * @return InetSocketAddress
 */
public static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS,
    YarnConfiguration.DEFAULT_PROXY_ADDRESS,
    YarnConfiguration.DEFAULT_PROXY_PORT);
}
 
Example 20
Source Project: hadoop   File: ResourceManager.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Retrieve RM bind address from configuration
 * 
 * @param conf
 * @return InetSocketAddress
 */
public static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
    YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
}