org.apache.hadoop.yarn.conf.HAUtil Java Examples
The following examples show how to use
org.apache.hadoop.yarn.conf.HAUtil.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ZKRMStateStore.java From hadoop with Apache License 2.0 | 6 votes |
/** * Given the {@link Configuration} and {@link ACL}s used (zkAcl) for * ZooKeeper access, construct the {@link ACL}s for the store's root node. * In the constructed {@link ACL}, all the users allowed by zkAcl are given * rwa access, while the current RM has exclude create-delete access. * * To be called only when HA is enabled and the configuration doesn't set ACL * for the root node. */ @VisibleForTesting @Private @Unstable protected List<ACL> constructZkRootNodeACL( Configuration conf, List<ACL> sourceACLs) throws NoSuchAlgorithmException { List<ACL> zkRootNodeAcl = new ArrayList<ACL>(); for (ACL acl : sourceACLs) { zkRootNodeAcl.add(new ACL( ZKUtil.removeSpecificPerms(acl.getPerms(), CREATE_DELETE_PERMS), acl.getId())); } zkRootNodeUsername = HAUtil.getConfValueForRMInstance( YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, conf); Id rmId = new Id(zkRootNodeAuthScheme, DigestAuthenticationProvider.generateDigest( zkRootNodeUsername + ":" + zkRootNodePassword)); zkRootNodeAcl.add(new ACL(CREATE_DELETE_PERMS, rmId)); return zkRootNodeAcl; }
Example #2
Source File: ConfiguredRMFailoverProxyProvider.java From big-c with Apache License 2.0 | 6 votes |
@Override public void init(Configuration configuration, RMProxy<T> rmProxy, Class<T> protocol) { this.rmProxy = rmProxy; this.protocol = protocol; this.rmProxy.checkAllowedProtocols(this.protocol); this.conf = new YarnConfiguration(configuration); Collection<String> rmIds = HAUtil.getRMHAIds(conf); this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]); conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES)); conf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS)); }
Example #3
Source File: RMProxy.java From big-c with Apache License 2.0 | 6 votes |
/** * Create a proxy for the specified protocol. For non-HA, * this is a direct connection to the ResourceManager address. When HA is * enabled, the proxy handles the failover between the ResourceManagers as * well. */ @Private protected static <T> T createRMProxy(final Configuration configuration, final Class<T> protocol, RMProxy instance) throws IOException { YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); RetryPolicy retryPolicy = createRetryPolicy(conf); if (HAUtil.isHAEnabled(conf)) { RMFailoverProxyProvider<T> provider = instance.createRMFailoverProxyProvider(conf, protocol); return (T) RetryProxy.create(protocol, provider, retryPolicy); } else { InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol); LOG.info("Connecting to ResourceManager at " + rmAddress); T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress); return (T) RetryProxy.create(protocol, proxy, retryPolicy); } }
Example #4
Source File: ClientRMProxy.java From big-c with Apache License 2.0 | 6 votes |
@Unstable public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) { if (HAUtil.isHAEnabled(conf)) { // Build a list of service addresses to form the service name ArrayList<String> services = new ArrayList<String>(); YarnConfiguration yarnConf = new YarnConfiguration(conf); for (String rmId : HAUtil.getRMHAIds(conf)) { // Set RM_ID to get the corresponding RM_ADDRESS yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); services.add(SecurityUtil.buildTokenService( yarnConf.getSocketAddr(address, defaultAddr, defaultPort)) .toString()); } return new Text(Joiner.on(',').join(services)); } // Non-HA case - no need to set RM_ID return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort)); }
Example #5
Source File: AmIpFilter.java From big-c with Apache License 2.0 | 6 votes |
protected String findRedirectUrl() throws ServletException { String addr; if (proxyUriBases.size() == 1) { // external proxy or not RM HA addr = proxyUriBases.values().iterator().next(); } else { // RM HA YarnConfiguration conf = new YarnConfiguration(); String activeRMId = RMHAUtils.findActiveRMHAId(conf); String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS : YarnConfiguration.RM_WEBAPP_ADDRESS; String host = conf.get( HAUtil.addSuffix(addressPropertyPrefix, activeRMId)); addr = proxyUriBases.get(host); } if (addr == null) { throw new ServletException( "Could not determine the proxy server for redirection"); } return addr; }
Example #6
Source File: RMAdminCLI.java From hadoop with Apache License 2.0 | 6 votes |
@Override protected HAServiceTarget resolveTarget(String rmId) { Collection<String> rmIds = HAUtil.getRMHAIds(getConf()); if (!rmIds.contains(rmId)) { StringBuilder msg = new StringBuilder(); msg.append(rmId + " is not a valid serviceId. It should be one of "); for (String id : rmIds) { msg.append(id + " "); } throw new IllegalArgumentException(msg.toString()); } try { YarnConfiguration conf = new YarnConfiguration(getConf()); conf.set(YarnConfiguration.RM_HA_ID, rmId); return new RMHAServiceTarget(conf); } catch (IllegalArgumentException iae) { throw new YarnRuntimeException("Could not connect to " + rmId + "; the configuration for it might be missing"); } catch (IOException ioe) { throw new YarnRuntimeException( "Could not connect to RM HA Admin for node " + rmId); } }
Example #7
Source File: TestZKRMStateStore.java From hadoop with Apache License 2.0 | 6 votes |
private Configuration createHARMConf( String rmIds, String rmId, int adminPort) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, rmIds); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(YarnConfiguration.RM_HA_ID, rmId); conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0"); for (String rpcAddress : YarnConfiguration.getServiceAddressConfKeys(conf)) { for (String id : HAUtil.getRMHAIds(conf)) { conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0"); } } conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId), "localhost:" + adminPort); return conf; }
Example #8
Source File: TestRMHA.java From hadoop with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { configuration = new Configuration(); UserGroupInformation.setConfiguration(configuration); configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); for (String confKey : YarnConfiguration .getServiceAddressConfKeys(configuration)) { configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } // Enable webapp to test web-services also configuration.setBoolean(MockRM.ENABLE_WEBAPP, true); configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); ClusterMetrics.destroy(); QueueMetrics.clearQueueMetrics(); DefaultMetricsSystem.shutdown(); }
Example #9
Source File: MiniYARNCluster.java From big-c with Apache License 2.0 | 6 votes |
private synchronized void initResourceManager(int index, Configuration conf) { if (HAUtil.isHAEnabled(conf)) { conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]); } resourceManagers[index].init(conf); resourceManagers[index].getRMContext().getDispatcher().register( RMAppAttemptEventType.class, new EventHandler<RMAppAttemptEvent>() { public void handle(RMAppAttemptEvent event) { if (event instanceof RMAppAttemptRegistrationEvent) { appMasters.put(event.getApplicationAttemptId(), event.getTimestamp()); } else if (event instanceof RMAppAttemptUnregistrationEvent) { appMasters.remove(event.getApplicationAttemptId()); } } }); }
Example #10
Source File: RMHATestBase.java From hadoop with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); configuration.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2"); configuration.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); configuration.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); configuration.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); configuration.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); configuration.set(YarnConfiguration.RM_CLUSTER_ID, "test-yarn-cluster"); int base = 100; for (String confKey : YarnConfiguration .getServiceAddressConfKeys(configuration)) { configuration.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:" + (base + 20)); configuration.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:" + (base + 40)); base = base * 2; } confForRM1 = new Configuration(configuration); confForRM1.set(YarnConfiguration.RM_HA_ID, "rm1"); confForRM2 = new Configuration(configuration); confForRM2.set(YarnConfiguration.RM_HA_ID, "rm2"); }
Example #11
Source File: AdminService.java From big-c with Apache License 2.0 | 6 votes |
@Override public void serviceInit(Configuration conf) throws Exception { if (rmContext.isHAEnabled()) { autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf); if (autoFailoverEnabled) { if (HAUtil.isAutomaticFailoverEmbedded(conf)) { embeddedElector = createEmbeddedElectorService(); addIfService(embeddedElector); } } } masterServiceBindAddress = conf.getSocketAddr( YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); daemonUser = UserGroupInformation.getCurrentUser(); authorizer = YarnAuthorizationProvider.getInstance(conf); authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation .getCurrentUser()); rmId = conf.get(YarnConfiguration.RM_HA_ID); super.serviceInit(conf); }
Example #12
Source File: ZKRMStateStore.java From hadoop with Apache License 2.0 | 6 votes |
@Override public synchronized void startInternal() throws Exception { // createConnection for future API calls createConnection(); // ensure root dirs exist createRootDirRecursively(znodeWorkingPath); createRootDir(zkRootNodePath); if (HAUtil.isHAEnabled(getConfig())){ fence(); verifyActiveStatusThread = new VerifyActiveStatusThread(); verifyActiveStatusThread.start(); } createRootDir(rmAppRoot); createRootDir(rmDTSecretManagerRoot); createRootDir(dtMasterKeysRootPath); createRootDir(delegationTokensRootPath); createRootDir(dtSequenceNumberPath); createRootDir(amrmTokenSecretManagerRoot); }
Example #13
Source File: ZKRMStateStore.java From big-c with Apache License 2.0 | 6 votes |
/** * Given the {@link Configuration} and {@link ACL}s used (zkAcl) for * ZooKeeper access, construct the {@link ACL}s for the store's root node. * In the constructed {@link ACL}, all the users allowed by zkAcl are given * rwa access, while the current RM has exclude create-delete access. * * To be called only when HA is enabled and the configuration doesn't set ACL * for the root node. */ @VisibleForTesting @Private @Unstable protected List<ACL> constructZkRootNodeACL( Configuration conf, List<ACL> sourceACLs) throws NoSuchAlgorithmException { List<ACL> zkRootNodeAcl = new ArrayList<ACL>(); for (ACL acl : sourceACLs) { zkRootNodeAcl.add(new ACL( ZKUtil.removeSpecificPerms(acl.getPerms(), CREATE_DELETE_PERMS), acl.getId())); } zkRootNodeUsername = HAUtil.getConfValueForRMInstance( YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS, conf); Id rmId = new Id(zkRootNodeAuthScheme, DigestAuthenticationProvider.generateDigest( zkRootNodeUsername + ":" + zkRootNodePassword)); zkRootNodeAcl.add(new ACL(CREATE_DELETE_PERMS, rmId)); return zkRootNodeAcl; }
Example #14
Source File: AdminService.java From hadoop with Apache License 2.0 | 6 votes |
@Override public void serviceInit(Configuration conf) throws Exception { if (rmContext.isHAEnabled()) { autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf); if (autoFailoverEnabled) { if (HAUtil.isAutomaticFailoverEmbedded(conf)) { embeddedElector = createEmbeddedElectorService(); addIfService(embeddedElector); } } } masterServiceBindAddress = conf.getSocketAddr( YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); daemonUser = UserGroupInformation.getCurrentUser(); authorizer = YarnAuthorizationProvider.getInstance(conf); authorizer.setAdmins(getAdminAclList(conf), UserGroupInformation .getCurrentUser()); rmId = conf.get(YarnConfiguration.RM_HA_ID); super.serviceInit(conf); }
Example #15
Source File: ZKRMStateStore.java From big-c with Apache License 2.0 | 6 votes |
@Override public synchronized void startInternal() throws Exception { // createConnection for future API calls createConnection(); // ensure root dirs exist createRootDirRecursively(znodeWorkingPath); createRootDir(zkRootNodePath); if (HAUtil.isHAEnabled(getConfig())){ fence(); verifyActiveStatusThread = new VerifyActiveStatusThread(); verifyActiveStatusThread.start(); } createRootDir(rmAppRoot); createRootDir(rmDTSecretManagerRoot); createRootDir(dtMasterKeysRootPath); createRootDir(delegationTokensRootPath); createRootDir(dtSequenceNumberPath); createRootDir(amrmTokenSecretManagerRoot); }
Example #16
Source File: MiniYARNCluster.java From hadoop with Apache License 2.0 | 6 votes |
private synchronized void initResourceManager(int index, Configuration conf) { if (HAUtil.isHAEnabled(conf)) { conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]); } resourceManagers[index].init(conf); resourceManagers[index].getRMContext().getDispatcher().register( RMAppAttemptEventType.class, new EventHandler<RMAppAttemptEvent>() { public void handle(RMAppAttemptEvent event) { if (event instanceof RMAppAttemptRegistrationEvent) { appMasters.put(event.getApplicationAttemptId(), event.getTimestamp()); } else if (event instanceof RMAppAttemptUnregistrationEvent) { appMasters.remove(event.getApplicationAttemptId()); } } }); }
Example #17
Source File: RMHATestBase.java From big-c with Apache License 2.0 | 6 votes |
@Before public void setup() throws Exception { configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); configuration.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2"); configuration.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); configuration.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); configuration.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); configuration.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false); configuration.set(YarnConfiguration.RM_CLUSTER_ID, "test-yarn-cluster"); int base = 100; for (String confKey : YarnConfiguration .getServiceAddressConfKeys(configuration)) { configuration.set(HAUtil.addSuffix(confKey, "rm1"), "0.0.0.0:" + (base + 20)); configuration.set(HAUtil.addSuffix(confKey, "rm2"), "0.0.0.0:" + (base + 40)); base = base * 2; } confForRM1 = new Configuration(configuration); confForRM1.set(YarnConfiguration.RM_HA_ID, "rm1"); confForRM2 = new Configuration(configuration); confForRM2.set(YarnConfiguration.RM_HA_ID, "rm2"); }
Example #18
Source File: TestRMHA.java From big-c with Apache License 2.0 | 6 votes |
@Before public void setUp() throws Exception { configuration = new Configuration(); UserGroupInformation.setConfiguration(configuration); configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); for (String confKey : YarnConfiguration .getServiceAddressConfKeys(configuration)) { configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS); configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS); } // Enable webapp to test web-services also configuration.setBoolean(MockRM.ENABLE_WEBAPP, true); configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); ClusterMetrics.destroy(); QueueMetrics.clearQueueMetrics(); DefaultMetricsSystem.shutdown(); }
Example #19
Source File: AmIpFilter.java From hadoop with Apache License 2.0 | 6 votes |
protected String findRedirectUrl() throws ServletException { String addr; if (proxyUriBases.size() == 1) { // external proxy or not RM HA addr = proxyUriBases.values().iterator().next(); } else { // RM HA YarnConfiguration conf = new YarnConfiguration(); String activeRMId = RMHAUtils.findActiveRMHAId(conf); String addressPropertyPrefix = YarnConfiguration.useHttps(conf) ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS : YarnConfiguration.RM_WEBAPP_ADDRESS; String host = conf.get( HAUtil.addSuffix(addressPropertyPrefix, activeRMId)); addr = proxyUriBases.get(host); } if (addr == null) { throw new ServletException( "Could not determine the proxy server for redirection"); } return addr; }
Example #20
Source File: TestZKRMStateStore.java From big-c with Apache License 2.0 | 6 votes |
private Configuration createHARMConf( String rmIds, String rmId, int adminPort) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, rmIds); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(YarnConfiguration.RM_HA_ID, rmId); conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0"); for (String rpcAddress : YarnConfiguration.getServiceAddressConfKeys(conf)) { for (String id : HAUtil.getRMHAIds(conf)) { conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0"); } } conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId), "localhost:" + adminPort); return conf; }
Example #21
Source File: ClientRMProxy.java From hadoop with Apache License 2.0 | 6 votes |
@Unstable public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) { if (HAUtil.isHAEnabled(conf)) { // Build a list of service addresses to form the service name ArrayList<String> services = new ArrayList<String>(); YarnConfiguration yarnConf = new YarnConfiguration(conf); for (String rmId : HAUtil.getRMHAIds(conf)) { // Set RM_ID to get the corresponding RM_ADDRESS yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); services.add(SecurityUtil.buildTokenService( yarnConf.getSocketAddr(address, defaultAddr, defaultPort)) .toString()); } return new Text(Joiner.on(',').join(services)); } // Non-HA case - no need to set RM_ID return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort)); }
Example #22
Source File: RMAdminCLI.java From big-c with Apache License 2.0 | 6 votes |
@Override protected HAServiceTarget resolveTarget(String rmId) { Collection<String> rmIds = HAUtil.getRMHAIds(getConf()); if (!rmIds.contains(rmId)) { StringBuilder msg = new StringBuilder(); msg.append(rmId + " is not a valid serviceId. It should be one of "); for (String id : rmIds) { msg.append(id + " "); } throw new IllegalArgumentException(msg.toString()); } try { YarnConfiguration conf = new YarnConfiguration(getConf()); conf.set(YarnConfiguration.RM_HA_ID, rmId); return new RMHAServiceTarget(conf); } catch (IllegalArgumentException iae) { throw new YarnRuntimeException("Could not connect to " + rmId + "; the configuration for it might be missing"); } catch (IOException ioe) { throw new YarnRuntimeException( "Could not connect to RM HA Admin for node " + rmId); } }
Example #23
Source File: RMProxy.java From hadoop with Apache License 2.0 | 6 votes |
/** * Create a proxy for the specified protocol. For non-HA, * this is a direct connection to the ResourceManager address. When HA is * enabled, the proxy handles the failover between the ResourceManagers as * well. */ @Private protected static <T> T createRMProxy(final Configuration configuration, final Class<T> protocol, RMProxy instance) throws IOException { YarnConfiguration conf = (configuration instanceof YarnConfiguration) ? (YarnConfiguration) configuration : new YarnConfiguration(configuration); RetryPolicy retryPolicy = createRetryPolicy(conf); if (HAUtil.isHAEnabled(conf)) { RMFailoverProxyProvider<T> provider = instance.createRMFailoverProxyProvider(conf, protocol); return (T) RetryProxy.create(protocol, provider, retryPolicy); } else { InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol); LOG.info("Connecting to ResourceManager at " + rmAddress); T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress); return (T) RetryProxy.create(protocol, proxy, retryPolicy); } }
Example #24
Source File: ConfiguredRMFailoverProxyProvider.java From hadoop with Apache License 2.0 | 6 votes |
@Override public void init(Configuration configuration, RMProxy<T> rmProxy, Class<T> protocol) { this.rmProxy = rmProxy; this.protocol = protocol; this.rmProxy.checkAllowedProtocols(this.protocol); this.conf = new YarnConfiguration(configuration); Collection<String> rmIds = HAUtil.getRMHAIds(conf); this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]); conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES)); conf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS)); }
Example #25
Source File: WebAppUtils.java From big-c with Apache License 2.0 | 5 votes |
public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf, Policy httpPolicy) { InetSocketAddress address = null; String rmId = null; if (HAUtil.isHAEnabled(conf)) { // If HA enabled, pick one of the RM-IDs and rely on redirect to go to // the Active RM rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0]; } if (httpPolicy == Policy.HTTPS_ONLY) { address = conf.getSocketAddr( rmId == null ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS : HAUtil.addSuffix( YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId), YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT); } else { address = conf.getSocketAddr( rmId == null ? YarnConfiguration.RM_WEBAPP_ADDRESS : HAUtil.addSuffix( YarnConfiguration.RM_WEBAPP_ADDRESS, rmId), YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_RM_WEBAPP_PORT); } return getResolvedAddress(address); }
Example #26
Source File: ZKRMStateStore.java From big-c with Apache License 2.0 | 5 votes |
T runWithRetries() throws Exception { int retry = 0; while (true) { try { return runWithCheck(); } catch (KeeperException.NoAuthException nae) { if (HAUtil.isHAEnabled(getConfig())) { // NoAuthException possibly means that this store is fenced due to // another RM becoming active. Even if not, // it is safer to assume we have been fenced throw new StoreFencedException(); } } catch (KeeperException ke) { if (ke.code() == Code.NODEEXISTS) { LOG.info("znode already exists!"); return null; } if (hasDeleteNodeOp && ke.code() == Code.NONODE) { LOG.info("znode has already been deleted!"); return null; } LOG.info("Exception while executing a ZK operation.", ke); if (shouldRetry(ke.code()) && ++retry < numRetries) { LOG.info("Retrying operation on ZK. Retry no. " + retry); Thread.sleep(zkRetryInterval); createConnection(); continue; } LOG.info("Maxed out ZK retries. Giving up!"); throw ke; } } }
Example #27
Source File: EmbeddedElectorService.java From big-c with Apache License 2.0 | 5 votes |
@Override protected void serviceInit(Configuration conf) throws Exception { conf = conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf); String zkQuorum = conf.get(YarnConfiguration.RM_ZK_ADDRESS); if (zkQuorum == null) { throw new YarnRuntimeException("Embedded automatic failover " + "is enabled, but " + YarnConfiguration.RM_ZK_ADDRESS + " is not set"); } String rmId = HAUtil.getRMHAId(conf); String clusterId = YarnConfiguration.getClusterId(conf); localActiveNodeInfo = createActiveNodeInfo(clusterId, rmId); String zkBasePath = conf.get(YarnConfiguration.AUTO_FAILOVER_ZK_BASE_PATH, YarnConfiguration.DEFAULT_AUTO_FAILOVER_ZK_BASE_PATH); String electionZNode = zkBasePath + "/" + clusterId; long zkSessionTimeout = conf.getLong(YarnConfiguration.RM_ZK_TIMEOUT_MS, YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS); List<ACL> zkAcls = RMZKUtils.getZKAcls(conf); List<ZKUtil.ZKAuthInfo> zkAuths = RMZKUtils.getZKAuths(conf); int maxRetryNum = conf.getInt( CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_KEY, CommonConfigurationKeys.HA_FC_ELECTOR_ZK_OP_RETRIES_DEFAULT); elector = new ActiveStandbyElector(zkQuorum, (int) zkSessionTimeout, electionZNode, zkAcls, zkAuths, this, maxRetryNum); elector.ensureParentZNode(); if (!isParentZnodeSafe(clusterId)) { notifyFatalError(electionZNode + " znode has invalid data! "+ "Might need formatting!"); } super.serviceInit(conf); }
Example #28
Source File: RMHAServiceTarget.java From big-c with Apache License 2.0 | 5 votes |
public RMHAServiceTarget(YarnConfiguration conf) throws IOException { autoFailoverEnabled = HAUtil.isAutomaticFailoverEnabled(conf); haAdminServiceAddress = conf.getSocketAddr( YarnConfiguration.RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS, YarnConfiguration.DEFAULT_RM_ADMIN_PORT); }
Example #29
Source File: HadoopConfExtractor.java From kylin with Apache License 2.0 | 5 votes |
public static String extractJobHistoryUrl(String yarnWebapp, Configuration conf) { Pattern pattern = Pattern.compile("(http[s]*://)([^:]*):([^/])*.*"); Matcher m = pattern.matcher(yarnWebapp); Preconditions.checkArgument(m.matches(), "Yarn master URL" + yarnWebapp + " not right."); String defaultHistoryUrl = m.group(2) + ":19888"; return m.group(1) + HAUtil.getConfValueForRMInstance(MR_JOB_HISTORY_URL_CONF_KEY, defaultHistoryUrl, conf); }
Example #30
Source File: MiniYARNClusterSplice.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private synchronized void initResourceManager(int index, Configuration conf) { if (HAUtil.isHAEnabled(conf)) { conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]); } if (conf.get(YarnConfiguration.RM_HOSTNAME) == null) { conf.set(YarnConfiguration.RM_HOSTNAME, "0.0.0.0"); } LOG.info("*** "+YarnConfiguration.RM_HOSTNAME+" is set to: "+conf.get(YarnConfiguration.RM_HOSTNAME)); if (conf.get(YarnConfiguration.RM_ADDRESS) == null) { conf.set(YarnConfiguration.RM_ADDRESS, YarnConfiguration.DEFAULT_RM_ADDRESS); } LOG.info("*** "+YarnConfiguration.RM_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_ADDRESS)); if (conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS) == null) { WebAppUtils .setNMWebAppHostNameAndPort(conf, MiniYARNClusterSplice.getHostname(), 0); } LOG.info("*** "+YarnConfiguration.RM_WEBAPP_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS)); resourceManagers[index].init(conf); resourceManagers[index].getRMContext().getDispatcher().register( RMAppAttemptEventType.class, new EventHandler<RMAppAttemptEvent>() { public void handle(RMAppAttemptEvent event) { if (event instanceof RMAppAttemptRegistrationEvent) { appMasters.put(event.getApplicationAttemptId(), event.getTimestamp()); } else if (event instanceof RMAppAttemptUnregistrationEvent) { appMasters.remove(event.getApplicationAttemptId()); } } }); }