Java Code Examples for org.apache.hadoop.ipc.RPC#getProtocolVersion()
The following examples show how to use
org.apache.hadoop.ipc.RPC#getProtocolVersion() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ContainerTestUtils.java From hadoop-ozone with Apache License 2.0 | 6 votes |
/** * Creates an Endpoint class for testing purpose. * * @param conf - Conf * @param address - InetAddres * @param rpcTimeout - rpcTimeOut * @return EndPoint * @throws Exception */ public static EndpointStateMachine createEndpoint(Configuration conf, InetSocketAddress address, int rpcTimeout) throws Exception { RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class, ProtobufRpcEngine.class); long version = RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class); StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy( StorageContainerDatanodeProtocolPB.class, version, address, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), rpcTimeout, RetryPolicies.TRY_ONCE_THEN_FAIL).getProxy(); StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient = new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy); return new EndpointStateMachine(address, rpcClient, new LegacyHadoopConfigurationSource(conf)); }
Example 2
Source File: ContainerOperationClient.java From hadoop-ozone with Apache License 2.0 | 6 votes |
public static StorageContainerLocationProtocol newContainerRpcClient( ConfigurationSource configSource) throws IOException { Class<StorageContainerLocationProtocolPB> protocol = StorageContainerLocationProtocolPB.class; Configuration conf = LegacyHadoopConfigurationSource.asHadoopConfiguration(configSource); RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class); long version = RPC.getProtocolVersion(protocol); InetSocketAddress scmAddress = getScmAddressForClients(configSource); UserGroupInformation user = UserGroupInformation.getCurrentUser(); SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(conf); int rpcTimeOut = Client.getRpcTimeout(conf); StorageContainerLocationProtocolPB rpcProxy = RPC.getProxy(protocol, version, scmAddress, user, conf, socketFactory, rpcTimeOut); StorageContainerLocationProtocolClientSideTranslatorPB client = new StorageContainerLocationProtocolClientSideTranslatorPB(rpcProxy); return TracingUtil.createProxy( client, StorageContainerLocationProtocol.class, configSource); }
Example 3
Source File: DefaultCertificateClient.java From hadoop-ozone with Apache License 2.0 | 6 votes |
/** * Create a scm security client, used to get SCM signed certificate. * * @return {@link SCMSecurityProtocol} */ private static SCMSecurityProtocol getScmSecurityClient( OzoneConfiguration conf) throws IOException { RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class, ProtobufRpcEngine.class); long scmVersion = RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class); InetSocketAddress scmSecurityProtoAdd = HddsServerUtil.getScmAddressForSecurityProtocol(conf); SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient = new SCMSecurityProtocolClientSideTranslatorPB( RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion, scmSecurityProtoAdd, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf))); return scmSecurityClient; }
Example 4
Source File: HddsServerUtil.java From hadoop-ozone with Apache License 2.0 | 6 votes |
/** * Create a scm security client. * @param conf - Ozone configuration. * * @return {@link SCMSecurityProtocol} * @throws IOException */ public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient( OzoneConfiguration conf) throws IOException { RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class, ProtobufRpcEngine.class); long scmVersion = RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class); InetSocketAddress address = getScmAddressForSecurityProtocol(conf); RetryPolicy retryPolicy = RetryPolicies.retryForeverWithFixedSleep( 1000, TimeUnit.MILLISECONDS); return new SCMSecurityProtocolClientSideTranslatorPB( RPC.getProtocolProxy(SCMSecurityProtocolPB.class, scmVersion, address, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf), retryPolicy).getProxy()); }
Example 5
Source File: BaseFreonGenerator.java From hadoop-ozone with Apache License 2.0 | 6 votes |
public StorageContainerLocationProtocol createStorageContainerLocationClient( OzoneConfiguration ozoneConf) throws IOException { long version = RPC.getProtocolVersion( StorageContainerLocationProtocolPB.class); InetSocketAddress scmAddress = getScmAddressForClients(ozoneConf); RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class, ProtobufRpcEngine.class); StorageContainerLocationProtocol client = TracingUtil.createProxy( new StorageContainerLocationProtocolClientSideTranslatorPB( RPC.getProxy(StorageContainerLocationProtocolPB.class, version, scmAddress, UserGroupInformation.getCurrentUser(), ozoneConf, NetUtils.getDefaultSocketFactory(ozoneConf), Client.getRpcTimeout(ozoneConf))), StorageContainerLocationProtocol.class, ozoneConf); return client; }
Example 6
Source File: OMFailoverProxyProvider.java From hadoop-ozone with Apache License 2.0 | 6 votes |
public OMFailoverProxyProvider(ConfigurationSource configuration, UserGroupInformation ugi, String omServiceId) throws IOException { this.conf = configuration; this.omVersion = RPC.getProtocolVersion(OzoneManagerProtocolPB.class); this.ugi = ugi; this.omServiceId = omServiceId; loadOMClientConfigs(conf, this.omServiceId); this.delegationTokenService = computeDelegationTokenService(); currentProxyIndex = 0; currentProxyOMNodeId = omNodeIDList.get(currentProxyIndex); waitBetweenRetries = conf.getLong( OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY, OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT); }
Example 7
Source File: OzoneManager.java From hadoop-ozone with Apache License 2.0 | 6 votes |
/** * Create a scm block client, used by putKey() and getKey(). * * @return {@link ScmBlockLocationProtocol} * @throws IOException */ private static ScmBlockLocationProtocol getScmBlockClient( OzoneConfiguration conf) throws IOException { RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class, ProtobufRpcEngine.class); long scmVersion = RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class); InetSocketAddress scmBlockAddress = getScmAddressForBlockClients(conf); ScmBlockLocationProtocolClientSideTranslatorPB scmBlockLocationClient = new ScmBlockLocationProtocolClientSideTranslatorPB( RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion, scmBlockAddress, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf))); return TracingUtil .createProxy(scmBlockLocationClient, ScmBlockLocationProtocol.class, conf); }
Example 8
Source File: OzoneManager.java From hadoop-ozone with Apache License 2.0 | 6 votes |
/** * Returns a scm container client. * * @return {@link StorageContainerLocationProtocol} * @throws IOException */ private static StorageContainerLocationProtocol getScmContainerClient( OzoneConfiguration conf) throws IOException { RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class, ProtobufRpcEngine.class); long scmVersion = RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class); InetSocketAddress scmAddr = getScmAddressForClients( conf); StorageContainerLocationProtocol scmContainerClient = TracingUtil.createProxy( new StorageContainerLocationProtocolClientSideTranslatorPB( RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion, scmAddr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf))), StorageContainerLocationProtocol.class, conf); return scmContainerClient; }
Example 9
Source File: SCMConnectionManager.java From hadoop-ozone with Apache License 2.0 | 5 votes |
/** * Adds a new Recon server to the set of endpoints. * @param address Recon address. * @throws IOException */ public void addReconServer(InetSocketAddress address) throws IOException { LOG.info("Adding Recon Server : {}", address.toString()); writeLock(); try { if (scmMachines.containsKey(address)) { LOG.warn("Trying to add an existing SCM Machine to Machines group. " + "Ignoring the request."); return; } Configuration hadoopConfig = LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf); RPC.setProtocolEngine(hadoopConfig, ReconDatanodeProtocolPB.class, ProtobufRpcEngine.class); long version = RPC.getProtocolVersion(ReconDatanodeProtocolPB.class); RetryPolicy retryPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(10, 60000, TimeUnit.MILLISECONDS); ReconDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy( ReconDatanodeProtocolPB.class, version, address, UserGroupInformation.getCurrentUser(), hadoopConfig, NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(), retryPolicy).getProxy(); StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient = new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy); EndpointStateMachine endPoint = new EndpointStateMachine(address, rpcClient, conf); endPoint.setPassive(true); scmMachines.put(address, endPoint); } finally { writeUnlock(); } }
Example 10
Source File: HddsServerUtil.java From hadoop-ozone with Apache License 2.0 | 5 votes |
/** * Create a scm block client, used by putKey() and getKey(). * * @return {@link ScmBlockLocationProtocol} * @throws IOException */ public static SCMSecurityProtocol getScmSecurityClient( OzoneConfiguration conf, UserGroupInformation ugi) throws IOException { RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class, ProtobufRpcEngine.class); long scmVersion = RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class); InetSocketAddress scmSecurityProtoAdd = getScmAddressForSecurityProtocol(conf); return new SCMSecurityProtocolClientSideTranslatorPB( RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion, scmSecurityProtoAdd, ugi, conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf))); }
Example 11
Source File: Hadoop27RpcTransport.java From hadoop-ozone with Apache License 2.0 | 5 votes |
public Hadoop27RpcTransport( ConfigurationSource conf) throws IOException { InetSocketAddress socket = OmUtils.getOmAddressForClients(conf); long version = RPC.getProtocolVersion(OzoneManagerProtocolPB.class); OzoneConfiguration ozoneConfiguration = OzoneConfiguration.of(conf); RPC.setProtocolEngine(ozoneConfiguration, OzoneManagerProtocolPB.class, ProtobufRpcEngine.class); proxy = RPC.getProtocolProxy(OzoneManagerProtocolPB.class, version, socket, UserGroupInformation.getCurrentUser(), ozoneConfiguration, NetUtils.getDefaultSocketFactory(ozoneConfiguration), getRpcTimeout(ozoneConfiguration), null).getProxy(); }
Example 12
Source File: NameNodeProxies.java From big-c with Apache License 2.0 | 5 votes |
private static ClientProtocol createNNProxyWithClientProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries, AtomicBoolean fallbackToSimpleAuth) throws IOException { RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy( conf, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT, SafeModeException.class); final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class); ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy( ClientNamenodeProtocolPB.class, version, address, ugi, conf, NetUtils.getDefaultSocketFactory(conf), org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy, fallbackToSimpleAuth).getProxy(); if (withRetries) { // create the proxy with retries Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); ClientProtocol translatorProxy = new ClientNamenodeProtocolTranslatorPB(proxy); return (ClientProtocol) RetryProxy.create( ClientProtocol.class, new DefaultFailoverProxyProvider<ClientProtocol>( ClientProtocol.class, translatorProxy), methodNameToPolicyMap, defaultPolicy); } else { return new ClientNamenodeProtocolTranslatorPB(proxy); } }
Example 13
Source File: NameNodeProxies.java From hadoop with Apache License 2.0 | 5 votes |
private static ClientProtocol createNNProxyWithClientProtocol( InetSocketAddress address, Configuration conf, UserGroupInformation ugi, boolean withRetries, AtomicBoolean fallbackToSimpleAuth) throws IOException { RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class); final RetryPolicy defaultPolicy = RetryUtils.getDefaultRetryPolicy( conf, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY, DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT, SafeModeException.class); final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class); ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy( ClientNamenodeProtocolPB.class, version, address, ugi, conf, NetUtils.getDefaultSocketFactory(conf), org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy, fallbackToSimpleAuth).getProxy(); if (withRetries) { // create the proxy with retries Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>(); ClientProtocol translatorProxy = new ClientNamenodeProtocolTranslatorPB(proxy); return (ClientProtocol) RetryProxy.create( ClientProtocol.class, new DefaultFailoverProxyProvider<ClientProtocol>( ClientProtocol.class, translatorProxy), methodNameToPolicyMap, defaultPolicy); } else { return new ClientNamenodeProtocolTranslatorPB(proxy); } }
Example 14
Source File: HAServiceProtocolServerSideTranslatorPB.java From big-c with Apache License 2.0 | 4 votes |
@Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { return RPC.getProtocolVersion(HAServiceProtocolPB.class); }
Example 15
Source File: SCMConnectionManager.java From hadoop-ozone with Apache License 2.0 | 4 votes |
/** * adds a new SCM machine to the target set. * * @param address - Address of the SCM machine to send heatbeat to. * @throws IOException */ public void addSCMServer(InetSocketAddress address) throws IOException { writeLock(); try { if (scmMachines.containsKey(address)) { LOG.warn("Trying to add an existing SCM Machine to Machines group. " + "Ignoring the request."); return; } Configuration hadoopConfig = LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf); RPC.setProtocolEngine( hadoopConfig, StorageContainerDatanodeProtocolPB.class, ProtobufRpcEngine.class); long version = RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class); RetryPolicy retryPolicy = RetryPolicies.retryForeverWithFixedSleep( 1000, TimeUnit.MILLISECONDS); StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy( StorageContainerDatanodeProtocolPB.class, version, address, UserGroupInformation.getCurrentUser(), hadoopConfig, NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(), retryPolicy).getProxy(); StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient = new StorageContainerDatanodeProtocolClientSideTranslatorPB( rpcProxy); EndpointStateMachine endPoint = new EndpointStateMachine(address, rpcClient, this.conf); endPoint.setPassive(false); scmMachines.put(address, endPoint); } finally { writeUnlock(); } }
Example 16
Source File: ZKFCProtocolServerSideTranslatorPB.java From hadoop with Apache License 2.0 | 4 votes |
@Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { return RPC.getProtocolVersion(ZKFCProtocolPB.class); }
Example 17
Source File: HAServiceProtocolServerSideTranslatorPB.java From hadoop with Apache License 2.0 | 4 votes |
@Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { return RPC.getProtocolVersion(HAServiceProtocolPB.class); }
Example 18
Source File: TestSecureOzoneCluster.java From hadoop-ozone with Apache License 2.0 | 4 votes |
@Test public void testGetS3Secret() throws Exception { // Setup secure OM for start setupOm(conf); long omVersion = RPC.getProtocolVersion(OzoneManagerProtocolPB.class); try { // Start OM om.setCertClient(new CertificateClientTestImpl(conf)); om.start(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); String username = ugi.getUserName(); // Get first OM client which will authenticate via Kerberos omClient = new OzoneManagerProtocolClientSideTranslatorPB( OmTransportFactory.create(conf, ugi, null), RandomStringUtils.randomAscii(5)); //Creates a secret since it does not exist S3SecretValue attempt1 = omClient.getS3Secret(username); //Fetches the secret from db since it was created in previous step S3SecretValue attempt2 = omClient.getS3Secret(username); //secret fetched on both attempts must be same assertEquals(attempt1.getAwsSecret(), attempt2.getAwsSecret()); //access key fetched on both attempts must be same assertEquals(attempt1.getAwsAccessKey(), attempt2.getAwsAccessKey()); try { omClient.getS3Secret("HADOOP/JOHNDOE"); fail("testGetS3Secret failed"); } catch (IOException ex) { GenericTestUtils.assertExceptionContains("USER_MISMATCH", ex); } } finally { IOUtils.closeQuietly(om); } }
Example 19
Source File: TestSecureOzoneCluster.java From hadoop-ozone with Apache License 2.0 | 4 votes |
/** * Tests delegation token renewal. */ @Test public void testDelegationTokenRenewal() throws Exception { GenericTestUtils .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO); LogCapturer omLogs = LogCapturer.captureLogs(OzoneManager.getLogger()); // Setup secure OM for start. OzoneConfiguration newConf = new OzoneConfiguration(conf); int tokenMaxLifetime = 1000; newConf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, tokenMaxLifetime); setupOm(newConf); long omVersion = RPC.getProtocolVersion(OzoneManagerProtocolPB.class); OzoneManager.setTestSecureOmFlag(true); // Start OM try { om.setCertClient(new CertificateClientTestImpl(conf)); om.start(); UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); // Get first OM client which will authenticate via Kerberos omClient = new OzoneManagerProtocolClientSideTranslatorPB( OmTransportFactory.create(conf, ugi, null), RandomStringUtils.randomAscii(5)); // Since client is already connected get a delegation token Token<OzoneTokenIdentifier> token = omClient.getDelegationToken( new Text("om")); // Check if token is of right kind and renewer is running om instance assertNotNull(token); assertEquals("OzoneToken", token.getKind().toString()); assertEquals(OmUtils.getOmRpcAddress(conf), token.getService().toString()); // Renew delegation token long expiryTime = omClient.renewDelegationToken(token); assertTrue(expiryTime > 0); omLogs.clearOutput(); // Test failure of delegation renewal // 1. When token maxExpiryTime exceeds Thread.sleep(tokenMaxLifetime); OMException ex = LambdaTestUtils.intercept(OMException.class, "TOKEN_EXPIRED", () -> omClient.renewDelegationToken(token)); assertEquals(TOKEN_EXPIRED, ex.getResult()); omLogs.clearOutput(); // 2. When renewer doesn't match (implicitly covers when renewer is // null or empty ) Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken( new Text("randomService")); assertNotNull(token2); LambdaTestUtils.intercept(OMException.class, "Delegation token renewal failed", () -> omClient.renewDelegationToken(token2)); assertTrue(omLogs.getOutput().contains(" with non-matching " + "renewer randomService")); omLogs.clearOutput(); // 3. Test tampered token OzoneTokenIdentifier tokenId = OzoneTokenIdentifier.readProtoBuf( token.getIdentifier()); tokenId.setRenewer(new Text("om")); tokenId.setMaxDate(System.currentTimeMillis() * 2); Token<OzoneTokenIdentifier> tamperedToken = new Token<>( tokenId.getBytes(), token2.getPassword(), token2.getKind(), token2.getService()); LambdaTestUtils.intercept(OMException.class, "Delegation token renewal failed", () -> omClient.renewDelegationToken(tamperedToken)); assertTrue(omLogs.getOutput().contains("can't be found in " + "cache")); omLogs.clearOutput(); } finally { om.stop(); om.join(); } }
Example 20
Source File: ZKFCProtocolServerSideTranslatorPB.java From big-c with Apache License 2.0 | 4 votes |
@Override public long getProtocolVersion(String protocol, long clientVersion) throws IOException { return RPC.getProtocolVersion(ZKFCProtocolPB.class); }