Java Code Examples for org.apache.hadoop.conf.Configuration#getSocketAddr()

The following examples show how to use org.apache.hadoop.conf.Configuration#getSocketAddr() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: WebAppUtils.java From big-c with Apache License 2.0

6 votes

public static String getResolvedRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);      
  }
  return getResolvedAddress(address);
}

Example 2

Source File: TestSharedCacheUploaderService.java From big-c with Apache License 2.0

6 votes

@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new SharedCacheUploaderService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);

  proxy =
      (SCMUploaderProtocol) rpc.getProxy(
          SCMUploaderProtocol.class, scmAddress, conf);
}

Example 3

Source File: HSAdmin.java From big-c with Apache License 2.0

5 votes

private int refreshLoadedJobCache() throws IOException {
  // Refresh the loaded job cache
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshLoadedJobCache();
  return 0;
}

Example 4

Source File: HSAdmin.java From big-c with Apache License 2.0

5 votes

private int refreshJobRetentionSettings() throws IOException {
  // Refresh job retention settings
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshJobRetentionSettings();
  return 0;
}

Example 5

Source File: WebAppUtils.java From hadoop with Apache License 2.0

5 votes

public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
    Policy httpPolicy) {
  InetSocketAddress address = null;
  String rmId = null;
  if (HAUtil.isHAEnabled(conf)) {
    // If HA enabled, pick one of the RM-IDs and rely on redirect to go to
    // the Active RM
    rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
  }

  if (httpPolicy == Policy.HTTPS_ONLY) {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(
            rmId == null
                ? YarnConfiguration.RM_WEBAPP_ADDRESS
                : HAUtil.addSuffix(
                YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
  }
  return getResolvedAddress(address);
}

Example 6

Source File: MiniMRYarnCluster.java From hadoop with Apache License 2.0

5 votes

public static String getResolvedMRHistoryWebAppURLWithoutScheme(
    Configuration conf, boolean isSSLEnabled) {
  InetSocketAddress address = null;
  if (isSSLEnabled) {
    address =
        conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_HTTPS_PORT);
  } else {
    address =
        conf.getSocketAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
            JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);    }
  address = NetUtils.getConnectAddress(address);
  StringBuffer sb = new StringBuffer();
  InetAddress resolved = address.getAddress();
  if (resolved == null || resolved.isAnyLocalAddress() || 
      resolved.isLoopbackAddress()) {
    String lh = address.getHostName();
    try {
      lh = InetAddress.getLocalHost().getCanonicalHostName();
    } catch (UnknownHostException e) {
      //Ignore and fallback.
    }
    sb.append(lh);
  } else {
    sb.append(address.getHostName());
  }
  sb.append(":").append(address.getPort());
  return sb.toString();
}

Example 7

Source File: HSAdmin.java From hadoop with Apache License 2.0

5 votes

private int refreshLogRetentionSettings() throws IOException {
  // Refresh log retention settings
  Configuration conf = getConf();
  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  HSAdminRefreshProtocol refreshProtocol = HSProxies.createProxy(conf,
      address, HSAdminRefreshProtocol.class,
      UserGroupInformation.getCurrentUser());

  refreshProtocol.refreshLogRetentionSettings();
  return 0;
}

Example 8

Source File: GetGroupsForTesting.java From big-c with Apache License 2.0

5 votes

@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}

Example 9

Source File: HSAdmin.java From hadoop with Apache License 2.0

5 votes

private int getGroups(String[] usernames) throws IOException {
  // Get groups users belongs to
  if (usernames.length == 0) {
    usernames = new String[] { UserGroupInformation.getCurrentUser()
        .getUserName() };
  }

  // Get the current configuration
  Configuration conf = getConf();

  InetSocketAddress address = conf.getSocketAddr(
      JHAdminConfig.JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_ADDRESS,
      JHAdminConfig.DEFAULT_JHS_ADMIN_PORT);

  GetUserMappingsProtocol getUserMappingProtocol = HSProxies.createProxy(
      conf, address, GetUserMappingsProtocol.class,
      UserGroupInformation.getCurrentUser());
  for (String username : usernames) {
    StringBuilder sb = new StringBuilder();
    sb.append(username + " :");
    for (String group : getUserMappingProtocol.getGroupsForUser(username)) {
      sb.append(" ");
      sb.append(group);
    }
    System.out.println(sb);
  }

  return 0;
}

Example 10

Source File: ClientRMService.java From hadoop with Apache License 2.0

5 votes

InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(
          YarnConfiguration.RM_BIND_HOST,
          YarnConfiguration.RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_PORT);
}

Example 11

Source File: ResourceTrackerService.java From big-c with Apache License 2.0

5 votes

@Override
protected void serviceInit(Configuration conf) throws Exception {
  resourceTrackerAddress = conf.getSocketAddr(
      YarnConfiguration.RM_BIND_HOST,
      YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);

  RackResolver.init(conf);
  nextHeartBeatInterval =
      conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
          YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
  if (nextHeartBeatInterval <= 0) {
    throw new YarnRuntimeException("Invalid Configuration. "
        + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
        + " should be larger than 0.");
  }

  minAllocMb = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  minAllocVcores = conf.getInt(
  	YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
  	YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);

  minimumNodeManagerVersion = conf.get(
      YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
      YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);

  super.serviceInit(conf);
}

Example 12

Source File: SCMAdminProtocolService.java From hadoop with Apache License 2.0

4 votes

InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
}

Example 13

Source File: TestLinuxContainerExecutor.java From hadoop with Apache License 2.0

4 votes

@Test
public void testContainerLocalizer() throws Exception {

  Assume.assumeTrue(shouldRun());

  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr =
      config.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user,
        String appId, String locId, InetSocketAddress nmAddr,
        List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId,
        nmAddr, localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(new LocalizerStartContext.Builder()
      .setNmPrivateContainerTokens(nmPrivateContainerTokensPath)
      .setNmAddr(nmAddr)
      .setUser(appSubmitter)
      .setAppId(appId)
      .setLocId(locId)
      .setDirsHandler(dirsHandler)
      .build());

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateContainerTokensPath2)
          .setNmAddr(nmAddr)
          .setUser(appSubmitter)
          .setAppId(appId)
          .setLocId(locId2)
          .setDirsHandler(dirsHandler)
          .build());


  cleanupUserAppCache(appSubmitter);
}

Example 14

Source File: SCMAdminProtocolService.java From big-c with Apache License 2.0

4 votes

InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
}

Example 15

Source File: TestAMAuthorization.java From big-c with Apache License 2.0

4 votes

@Test
public void testUnauthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm = new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  RMApp app = rm.submitApp(1024);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 40) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress serviceAddr = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());

  // First try contacting NM without tokens
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
              serviceAddr, conf);
        }
      });
  
  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  try {
    client.registerApplicationMaster(request);
    Assert.fail("Should fail with authorization error");
  } catch (Exception e) {
    if (isCause(AccessControlException.class, e)) {
      // Because there are no tokens, the request should be rejected as the
      // server side will assume we are trying simple auth.
      String expectedMessage = "";
      if (UserGroupInformation.isSecurityEnabled()) {
        expectedMessage = "Client cannot authenticate via:[TOKEN]";
      } else {
        expectedMessage =
            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
      }
      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); 
    } else {
      throw e;
    }
  }

  // TODO: Add validation of invalid authorization when there's more data in
  // the AMRMToken
}

Example 16

Source File: TestLinuxContainerExecutor.java From big-c with Apache License 2.0

4 votes

@Test
public void testContainerLocalizer() throws Exception {
  if (!shouldRun()) {
    return;
  }
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  for (String localDir : localDirs) {
    Path userDir =
        new Path(localDir, ContainerLocalizer.USERCACHE);
    files.mkdir(userDir, new FsPermission("777"), false);
    // $local/filecache
    Path fileDir =
        new Path(localDir, ContainerLocalizer.FILECACHE);
    files.mkdir(fileDir, new FsPermission("777"), false);
  }
  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler.getLocalPathForWrite(
          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr = config.getSocketAddr(
    YarnConfiguration.NM_BIND_HOST,
    YarnConfiguration.NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user, String appId,
        String locId, InetSocketAddress nmAddr, List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
        localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
    appId, locId, dirsHandler);

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
    appId, locId2, dirsHandler);
}

Example 17

Source File: Master.java From tez with Apache License 2.0

4 votes

public static InetSocketAddress getMasterAddress(Configuration conf) {
  return conf
      .getSocketAddr(YarnConfiguration.RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_ADDRESS,
          YarnConfiguration.DEFAULT_RM_PORT);
}

Example 18

Source File: ApplicationHistoryServer.java From hadoop with Apache License 2.0

2 votes

/**
 * Retrieve the timeline server bind address from configuration
 *
 * @param conf
 * @return InetSocketAddress
 */
private static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
}

Example 19

Source File: WebAppProxyServer.java From big-c with Apache License 2.0

2 votes

/**
 * Retrieve PROXY bind address from configuration
 *
 * @param conf
 * @return InetSocketAddress
 */
public static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.PROXY_ADDRESS,
    YarnConfiguration.DEFAULT_PROXY_ADDRESS,
    YarnConfiguration.DEFAULT_PROXY_PORT);
}

Example 20

Source File: ResourceManager.java From hadoop with Apache License 2.0

2 votes

/**
 * Retrieve RM bind address from configuration
 * 
 * @param conf
 * @return InetSocketAddress
 */
public static InetSocketAddress getBindAddress(Configuration conf) {
  return conf.getSocketAddr(YarnConfiguration.RM_ADDRESS,
    YarnConfiguration.DEFAULT_RM_ADDRESS, YarnConfiguration.DEFAULT_RM_PORT);
}