Java Code Examples for org.apache.hadoop.net.NetUtils#createSocketAddrForHost()
The following examples show how to use
org.apache.hadoop.net.NetUtils#createSocketAddrForHost() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestSecurityUtil.java From big-c with Apache License 2.0 | 6 votes |
private void verifyServiceAddr(String host, String ip) { InetSocketAddress addr; int port = 123; // test host, port tuple //LOG.info("test tuple ("+host+","+port+")"); addr = NetUtils.createSocketAddrForHost(host, port); verifyAddress(addr, host, ip, port); // test authority with no default port //LOG.info("test authority '"+host+":"+port+"'"); addr = NetUtils.createSocketAddr(host+":"+port); verifyAddress(addr, host, ip, port); // test authority with a default port, make sure default isn't used //LOG.info("test authority '"+host+":"+port+"' with ignored default port"); addr = NetUtils.createSocketAddr(host+":"+port, port+1); verifyAddress(addr, host, ip, port); // test host-only authority, using port as default port //LOG.info("test host:"+host+" port:"+port); addr = NetUtils.createSocketAddr(host, port); verifyAddress(addr, host, ip, port); }
Example 2
Source File: TestSecurityUtil.java From hadoop with Apache License 2.0 | 6 votes |
private void verifyServiceAddr(String host, String ip) { InetSocketAddress addr; int port = 123; // test host, port tuple //LOG.info("test tuple ("+host+","+port+")"); addr = NetUtils.createSocketAddrForHost(host, port); verifyAddress(addr, host, ip, port); // test authority with no default port //LOG.info("test authority '"+host+":"+port+"'"); addr = NetUtils.createSocketAddr(host+":"+port); verifyAddress(addr, host, ip, port); // test authority with a default port, make sure default isn't used //LOG.info("test authority '"+host+":"+port+"' with ignored default port"); addr = NetUtils.createSocketAddr(host+":"+port, port+1); verifyAddress(addr, host, ip, port); // test host-only authority, using port as default port //LOG.info("test host:"+host+" port:"+port); addr = NetUtils.createSocketAddr(host, port); verifyAddress(addr, host, ip, port); }
Example 3
Source File: TestSecurityUtil.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testGoodHostsAndPorts() { InetSocketAddress compare = NetUtils.createSocketAddrForHost("localhost", 123); runGoodCases(compare, "localhost", 123); runGoodCases(compare, "localhost:", 123); runGoodCases(compare, "localhost:123", 456); }
Example 4
Source File: RecoverableRpcProxy.java From Bats with Apache License 2.0 | 5 votes |
private long connect(long timeMillis) throws IOException { String uriStr = fsRecoveryHandler.readConnectUri(); if (!uriStr.equals(lastConnectURI)) { LOG.debug("Got new RPC connect address {}", uriStr); lastConnectURI = uriStr; if (umbilical != null) { RPC.stopProxy(umbilical); } retryTimeoutMillis = Long.getLong(RETRY_TIMEOUT, RETRY_TIMEOUT_DEFAULT); retryDelayMillis = Long.getLong(RETRY_DELAY, RETRY_DELAY_DEFAULT); rpcTimeout = Integer.getInteger(RPC_TIMEOUT, RPC_TIMEOUT_DEFAULT); URI heartbeatUri = URI.create(uriStr); String queryStr = heartbeatUri.getQuery(); if (queryStr != null) { List<NameValuePair> queryList = URLEncodedUtils.parse(queryStr, Charset.defaultCharset()); if (queryList != null) { for (NameValuePair pair : queryList) { String value = pair.getValue(); String key = pair.getName(); if (QP_rpcTimeout.equals(key)) { this.rpcTimeout = Integer.parseInt(value); } else if (QP_retryTimeoutMillis.equals(key)) { this.retryTimeoutMillis = Long.parseLong(value); } else if (QP_retryDelayMillis.equals(key)) { this.retryDelayMillis = Long.parseLong(value); } } } } InetSocketAddress address = NetUtils.createSocketAddrForHost(heartbeatUri.getHost(), heartbeatUri.getPort()); umbilical = RPC.getProxy(StreamingContainerUmbilicalProtocol.class, StreamingContainerUmbilicalProtocol.versionID, address, currentUser, conf, defaultSocketFactory, rpcTimeout); // reset timeout return System.currentTimeMillis() + retryTimeoutMillis; } return timeMillis; }
Example 5
Source File: DAGClientServer.java From tez with Apache License 2.0 | 5 votes |
@Override public void serviceStart() { try { Configuration conf = getConfig(); InetSocketAddress addr = new InetSocketAddress(0); DAGClientAMProtocolBlockingPBServerImpl service = new DAGClientAMProtocolBlockingPBServerImpl(realInstance, stagingFs); BlockingService blockingService = DAGClientAMProtocol.newReflectiveBlockingService(service); int numHandlers = conf.getInt(TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT, TezConfiguration.TEZ_AM_CLIENT_THREAD_COUNT_DEFAULT); if (numHandlers < 2) { numHandlers = 2; } server = createServer(DAGClientAMProtocolBlockingPB.class, addr, conf, numHandlers, blockingService, TezConfiguration.TEZ_AM_CLIENT_AM_PORT_RANGE); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new TezAMPolicyProvider()); } server.start(); InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server); this.bindAddress = NetUtils.createSocketAddrForHost( serverBindAddress.getAddress().getCanonicalHostName(), serverBindAddress.getPort()); LOG.info("Instantiated DAGClientRPCServer at " + bindAddress); } catch (Exception e) { LOG.error("Failed to start DAGClientServer: ", e); throw new TezUncheckedException(e); } }
Example 6
Source File: MRApp.java From big-c with Apache License 2.0 | 5 votes |
public static Token newContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens InetSocketAddress addr = NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token Token containerToken = Token.newInstance(tokenIdentifier.getBytes(), ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil .buildTokenService(addr).toString()); return containerToken; }
Example 7
Source File: MRClientService.java From big-c with Apache License 2.0 | 5 votes |
protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress address = new InetSocketAddress(0); server = rpc.getServer(MRClientProtocol.class, protocolHandler, address, conf, appContext.getClientToAMTokenSecretManager(), conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT), MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new MRAMPolicyProvider()); } server.start(); this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(), server.getListenerAddress().getPort()); LOG.info("Instantiated MRClientService at " + this.bindAddress); try { // Explicitly disabling SSL for map reduce task as we can't allow MR users // to gain access to keystore file for opening SSL listener. We can trust // RM/NM to issue SSL certificates but definitely not MR-AM as it is // running in user-land. webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws") .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp()); } catch (Exception e) { LOG.error("Webapps failed to start. Ignoring for now:", e); } super.serviceStart(); }
Example 8
Source File: TaskAttemptListenerImpl.java From big-c with Apache License 2.0 | 5 votes |
protected void startRpcServer() { Configuration conf = getConfig(); try { server = new RPC.Builder(conf).setProtocol(TaskUmbilicalProtocol.class) .setInstance(this).setBindAddress("0.0.0.0") .setPort(0).setNumHandlers( conf.getInt(MRJobConfig.MR_AM_TASK_LISTENER_THREAD_COUNT, MRJobConfig.DEFAULT_MR_AM_TASK_LISTENER_THREAD_COUNT)) .setVerbose(false).setSecretManager(jobTokenSecretManager) .build(); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new MRAMPolicyProvider()); } server.start(); this.address = NetUtils.createSocketAddrForHost( context.getNMHostname(), server.getListenerAddress().getPort()); } catch (IOException e) { throw new YarnRuntimeException(e); } }
Example 9
Source File: AMLauncher.java From big-c with Apache License 2.0 | 5 votes |
protected ContainerManagementProtocol getContainerMgrProxy( final ContainerId containerId) { final NodeId node = masterContainer.getNodeId(); final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost(node.getHost(), node.getPort()); final YarnRPC rpc = YarnRPC.create(conf); // TODO: Don't create again and again. UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(containerId .getApplicationAttemptId().toString()); String user = rmContext.getRMApps() .get(containerId.getApplicationAttemptId().getApplicationId()) .getUser(); org.apache.hadoop.yarn.api.records.Token token = rmContext.getNMTokenSecretManager().createNMToken( containerId.getApplicationAttemptId(), node, user); currentUser.addToken(ConverterUtils.convertFromYarn(token, containerManagerBindAddress)); return currentUser .doAs(new PrivilegedAction<ContainerManagementProtocol>() { @Override public ContainerManagementProtocol run() { return (ContainerManagementProtocol) rpc.getProxy( ContainerManagementProtocol.class, containerManagerBindAddress, conf); } }); }
Example 10
Source File: BuilderUtils.java From big-c with Apache License 2.0 | 5 votes |
@VisibleForTesting public static Token newContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens InetSocketAddress addr = NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token Token containerToken = newToken(Token.class, tokenIdentifier.getBytes(), ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil .buildTokenService(addr).toString()); return containerToken; }
Example 11
Source File: BaseNMTokenSecretManager.java From big-c with Apache License 2.0 | 5 votes |
public static Token newInstance(byte[] password, NMTokenIdentifier identifier) { NodeId nodeId = identifier.getNodeId(); // RPC layer client expects ip:port as service for tokens InetSocketAddress addr = NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); Token nmToken = Token.newInstance(identifier.getBytes(), NMTokenIdentifier.KIND.toString(), password, SecurityUtil .buildTokenService(addr).toString()); return nmToken; }
Example 12
Source File: TestSecurityUtil.java From hadoop with Apache License 2.0 | 5 votes |
@Test public void testGoodHostsAndPorts() { InetSocketAddress compare = NetUtils.createSocketAddrForHost("localhost", 123); runGoodCases(compare, "localhost", 123); runGoodCases(compare, "localhost:", 123); runGoodCases(compare, "localhost:123", 456); }
Example 13
Source File: Client.java From hadoop with Apache License 2.0 | 5 votes |
/** * Update the server address if the address corresponding to the host * name has changed. * * @return true if an addr change was detected. * @throws IOException when the hostname cannot be resolved. */ private synchronized boolean updateAddress() throws IOException { // Do a fresh lookup with the old host name. InetSocketAddress currentAddr = NetUtils.createSocketAddrForHost( server.getHostName(), server.getPort()); if (!server.equals(currentAddr)) { LOG.warn("Address change detected. Old: " + server.toString() + " New: " + currentAddr.toString()); server = currentAddr; return true; } return false; }
Example 14
Source File: RecoverableRpcProxy.java From attic-apex-core with Apache License 2.0 | 5 votes |
private long connect(long timeMillis) throws IOException { String uriStr = fsRecoveryHandler.readConnectUri(); if (!uriStr.equals(lastConnectURI)) { LOG.debug("Got new RPC connect address {}", uriStr); lastConnectURI = uriStr; if (umbilical != null) { RPC.stopProxy(umbilical); } retryTimeoutMillis = Long.getLong(RETRY_TIMEOUT, RETRY_TIMEOUT_DEFAULT); retryDelayMillis = Long.getLong(RETRY_DELAY, RETRY_DELAY_DEFAULT); rpcTimeout = Integer.getInteger(RPC_TIMEOUT, RPC_TIMEOUT_DEFAULT); URI heartbeatUri = URI.create(uriStr); String queryStr = heartbeatUri.getQuery(); if (queryStr != null) { List<NameValuePair> queryList = URLEncodedUtils.parse(queryStr, Charset.defaultCharset()); if (queryList != null) { for (NameValuePair pair : queryList) { String value = pair.getValue(); String key = pair.getName(); if (QP_rpcTimeout.equals(key)) { this.rpcTimeout = Integer.parseInt(value); } else if (QP_retryTimeoutMillis.equals(key)) { this.retryTimeoutMillis = Long.parseLong(value); } else if (QP_retryDelayMillis.equals(key)) { this.retryDelayMillis = Long.parseLong(value); } } } } InetSocketAddress address = NetUtils.createSocketAddrForHost(heartbeatUri.getHost(), heartbeatUri.getPort()); umbilical = RPC.getProxy(StreamingContainerUmbilicalProtocol.class, StreamingContainerUmbilicalProtocol.versionID, address, currentUser, conf, defaultSocketFactory, rpcTimeout); // reset timeout return System.currentTimeMillis() + retryTimeoutMillis; } return timeMillis; }
Example 15
Source File: AMLauncher.java From hadoop with Apache License 2.0 | 5 votes |
protected ContainerManagementProtocol getContainerMgrProxy( final ContainerId containerId) { final NodeId node = masterContainer.getNodeId(); final InetSocketAddress containerManagerBindAddress = NetUtils.createSocketAddrForHost(node.getHost(), node.getPort()); final YarnRPC rpc = YarnRPC.create(conf); // TODO: Don't create again and again. UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(containerId .getApplicationAttemptId().toString()); String user = rmContext.getRMApps() .get(containerId.getApplicationAttemptId().getApplicationId()) .getUser(); org.apache.hadoop.yarn.api.records.Token token = rmContext.getNMTokenSecretManager().createNMToken( containerId.getApplicationAttemptId(), node, user); currentUser.addToken(ConverterUtils.convertFromYarn(token, containerManagerBindAddress)); return currentUser .doAs(new PrivilegedAction<ContainerManagementProtocol>() { @Override public ContainerManagementProtocol run() { return (ContainerManagementProtocol) rpc.getProxy( ContainerManagementProtocol.class, containerManagerBindAddress, conf); } }); }
Example 16
Source File: BaseNMTokenSecretManager.java From hadoop with Apache License 2.0 | 5 votes |
public static Token newInstance(byte[] password, NMTokenIdentifier identifier) { NodeId nodeId = identifier.getNodeId(); // RPC layer client expects ip:port as service for tokens InetSocketAddress addr = NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); Token nmToken = Token.newInstance(identifier.getBytes(), NMTokenIdentifier.KIND.toString(), password, SecurityUtil .buildTokenService(addr).toString()); return nmToken; }
Example 17
Source File: TestRPC.java From hadoop with Apache License 2.0 | 5 votes |
public static Token newContainerToken(NodeId nodeId, byte[] password, ContainerTokenIdentifier tokenIdentifier) { // RPC layer client expects ip:port as service for tokens InetSocketAddress addr = NetUtils.createSocketAddrForHost(nodeId.getHost(), nodeId.getPort()); // NOTE: use SecurityUtil.setTokenService if this becomes a "real" token Token containerToken = Token.newInstance(tokenIdentifier.getBytes(), ContainerTokenIdentifier.KIND.toString(), password, SecurityUtil .buildTokenService(addr).toString()); return containerToken; }
Example 18
Source File: TezTestServiceTaskCommunicatorWithErrors.java From tez with Apache License 2.0 | 4 votes |
@Override public InetSocketAddress getAddress() { return NetUtils.createSocketAddrForHost("localhost", 0); }
Example 19
Source File: TezChild.java From tez with Apache License 2.0 | 4 votes |
public TezChild(Configuration conf, String host, int port, String containerIdentifier, String tokenIdentifier, int appAttemptNumber, String workingDir, String[] localDirs, Map<String, String> serviceProviderEnvMap, ObjectRegistryImpl objectRegistry, String pid, ExecutionContext executionContext, Credentials credentials, long memAvailable, String user, TezTaskUmbilicalProtocol umbilical, boolean updateSysCounters, HadoopShim hadoopShim) throws IOException, InterruptedException { this.defaultConf = conf; this.containerIdString = containerIdentifier; this.appAttemptNumber = appAttemptNumber; this.localDirs = localDirs; this.serviceProviderEnvMap = serviceProviderEnvMap; this.workingDir = workingDir; this.pid = pid; this.executionContext = executionContext; this.credentials = credentials; this.memAvailable = memAvailable; this.user = user; this.updateSysCounters = updateSysCounters; this.hadoopShim = hadoopShim; this.sharedExecutor = new TezSharedExecutor(defaultConf); getTaskMaxSleepTime = defaultConf.getInt( TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX, TezConfiguration.TEZ_TASK_GET_TASK_SLEEP_INTERVAL_MS_MAX_DEFAULT); amHeartbeatInterval = defaultConf.getInt(TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_INTERVAL_MS_DEFAULT); sendCounterInterval = defaultConf.getLong( TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS, TezConfiguration.TEZ_TASK_AM_HEARTBEAT_COUNTER_INTERVAL_MS_DEFAULT); maxEventsToGet = defaultConf.getInt(TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT, TezConfiguration.TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT); ExecutorService executor = Executors.newFixedThreadPool(1, new ThreadFactoryBuilder() .setDaemon(true).setNameFormat("TezChild").build()); this.executor = MoreExecutors.listeningDecorator(executor); this.objectRegistry = objectRegistry; if (LOG.isDebugEnabled()) { LOG.debug("Executing with tokens:"); for (Token<?> token : credentials.getAllTokens()) { LOG.debug("",token); } } UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(tokenIdentifier); Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials); String auxiliaryService = defaultConf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT); serviceConsumerMetadata.put(auxiliaryService, TezCommonUtils.convertJobTokenToBytes(jobToken)); if (umbilical == null) { final InetSocketAddress address = NetUtils.createSocketAddrForHost(host, port); SecurityUtil.setTokenService(jobToken, address); taskOwner.addToken(jobToken); this.umbilical = taskOwner.doAs(new PrivilegedExceptionAction<TezTaskUmbilicalProtocol>() { @Override public TezTaskUmbilicalProtocol run() throws Exception { return RPC.getProxy(TezTaskUmbilicalProtocol.class, TezTaskUmbilicalProtocol.versionID, address, defaultConf); } }); ownUmbilical = true; } else { this.umbilical = umbilical; ownUmbilical = false; } TezCommonUtils.logCredentials(LOG, credentials, "tezChildInit"); }
Example 20
Source File: TaskRunner.java From incubator-tajo with Apache License 2.0 | 4 votes |
public TaskRunner(TaskRunnerManager taskRunnerManager, TajoConf conf, String[] args) { super(TaskRunner.class.getName()); this.taskRunnerManager = taskRunnerManager; this.connPool = RpcConnectionPool.getPool(conf); this.fetchLauncher = Executors.newFixedThreadPool( conf.getIntVar(ConfVars.SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM)); try { final ExecutionBlockId executionBlockId = TajoIdUtils.createExecutionBlockId(args[1]); LOG.info("Tajo Root Dir: " + conf.getVar(ConfVars.ROOT_DIR)); LOG.info("Worker Local Dir: " + conf.getVar(ConfVars.WORKER_TEMPORAL_DIR)); UserGroupInformation.setConfiguration(conf); // QueryBlockId from String // NodeId has a form of hostname:port. NodeId nodeId = ConverterUtils.toNodeId(args[2]); this.containerId = ConverterUtils.toContainerId(args[3]); // QueryMaster's address String host = args[4]; int port = Integer.parseInt(args[5]); this.qmMasterAddr = NetUtils.createSocketAddrForHost(host, port); LOG.info("QueryMaster Address:" + qmMasterAddr); // TODO - 'load credential' should be implemented // Getting taskOwner UserGroupInformation taskOwner = UserGroupInformation.createRemoteUser(conf.getVar(ConfVars.USERNAME)); //taskOwner.addToken(token); // initialize MasterWorkerProtocol as an actual task owner. // this.client = // taskOwner.doAs(new PrivilegedExceptionAction<AsyncRpcClient>() { // @Override // public AsyncRpcClient run() throws Exception { // return new AsyncRpcClient(TajoWorkerProtocol.class, masterAddr); // } // }); // this.master = client.getStub(); this.executionBlockId = executionBlockId; this.queryId = executionBlockId.getQueryId(); this.nodeId = nodeId; this.taskOwner = taskOwner; this.taskRunnerContext = new TaskRunnerContext(); } catch (Exception e) { LOG.error(e.getMessage(), e); } }