Java Code Examples for org.apache.hadoop.ipc.RPC#getProtocolVersion()

/**
 * Creates an Endpoint class for testing purpose.
 *
 * @param conf - Conf
 * @param address - InetAddres
 * @param rpcTimeout - rpcTimeOut
 * @return EndPoint
 * @throws Exception
 */
public static EndpointStateMachine createEndpoint(Configuration conf,
    InetSocketAddress address, int rpcTimeout) throws Exception {
  RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
      ProtobufRpcEngine.class);
  long version =
      RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class);

  StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
      StorageContainerDatanodeProtocolPB.class, version,
      address, UserGroupInformation.getCurrentUser(), conf,
      NetUtils.getDefaultSocketFactory(conf), rpcTimeout,
      RetryPolicies.TRY_ONCE_THEN_FAIL).getProxy();

  StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
      new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy);
  return new EndpointStateMachine(address, rpcClient,
      new LegacyHadoopConfigurationSource(conf));
}
 

public static StorageContainerLocationProtocol newContainerRpcClient(
    ConfigurationSource configSource) throws IOException {

  Class<StorageContainerLocationProtocolPB> protocol =
      StorageContainerLocationProtocolPB.class;
  Configuration conf =
      LegacyHadoopConfigurationSource.asHadoopConfiguration(configSource);
  RPC.setProtocolEngine(conf, protocol, ProtobufRpcEngine.class);
  long version = RPC.getProtocolVersion(protocol);
  InetSocketAddress scmAddress = getScmAddressForClients(configSource);
  UserGroupInformation user = UserGroupInformation.getCurrentUser();
  SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(conf);
  int rpcTimeOut = Client.getRpcTimeout(conf);

  StorageContainerLocationProtocolPB rpcProxy =
      RPC.getProxy(protocol, version, scmAddress, user, conf,
          socketFactory, rpcTimeOut);

  StorageContainerLocationProtocolClientSideTranslatorPB client =
      new StorageContainerLocationProtocolClientSideTranslatorPB(rpcProxy);
  return TracingUtil.createProxy(
      client, StorageContainerLocationProtocol.class, configSource);
}
 

/**
 * Create a scm security client, used to get SCM signed certificate.
 *
 * @return {@link SCMSecurityProtocol}
 */
private static SCMSecurityProtocol getScmSecurityClient(
    OzoneConfiguration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
      ProtobufRpcEngine.class);
  long scmVersion =
      RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
  InetSocketAddress scmSecurityProtoAdd =
      HddsServerUtil.getScmAddressForSecurityProtocol(conf);
  SCMSecurityProtocolClientSideTranslatorPB scmSecurityClient =
      new SCMSecurityProtocolClientSideTranslatorPB(
          RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion,
              scmSecurityProtoAdd, UserGroupInformation.getCurrentUser(),
              conf, NetUtils.getDefaultSocketFactory(conf),
              Client.getRpcTimeout(conf)));
  return scmSecurityClient;
}
 

/**
 * Create a scm security client.
 * @param conf    - Ozone configuration.
 *
 * @return {@link SCMSecurityProtocol}
 * @throws IOException
 */
public static SCMSecurityProtocolClientSideTranslatorPB getScmSecurityClient(
    OzoneConfiguration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
      ProtobufRpcEngine.class);
  long scmVersion =
      RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
  InetSocketAddress address =
      getScmAddressForSecurityProtocol(conf);
  RetryPolicy retryPolicy =
      RetryPolicies.retryForeverWithFixedSleep(
          1000, TimeUnit.MILLISECONDS);
  return new SCMSecurityProtocolClientSideTranslatorPB(
      RPC.getProtocolProxy(SCMSecurityProtocolPB.class, scmVersion,
          address, UserGroupInformation.getCurrentUser(),
          conf, NetUtils.getDefaultSocketFactory(conf),
          Client.getRpcTimeout(conf), retryPolicy).getProxy());
}
 

public StorageContainerLocationProtocol createStorageContainerLocationClient(
    OzoneConfiguration ozoneConf)
    throws IOException {

  long version = RPC.getProtocolVersion(
      StorageContainerLocationProtocolPB.class);
  InetSocketAddress scmAddress =
      getScmAddressForClients(ozoneConf);

  RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
      ProtobufRpcEngine.class);
  StorageContainerLocationProtocol client =
      TracingUtil.createProxy(
          new StorageContainerLocationProtocolClientSideTranslatorPB(
              RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
                  scmAddress, UserGroupInformation.getCurrentUser(),
                  ozoneConf,
                  NetUtils.getDefaultSocketFactory(ozoneConf),
                  Client.getRpcTimeout(ozoneConf))),
          StorageContainerLocationProtocol.class, ozoneConf);
  return client;
}
 

public OMFailoverProxyProvider(ConfigurationSource configuration,
    UserGroupInformation ugi, String omServiceId) throws IOException {
  this.conf = configuration;
  this.omVersion = RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
  this.ugi = ugi;
  this.omServiceId = omServiceId;
  loadOMClientConfigs(conf, this.omServiceId);
  this.delegationTokenService = computeDelegationTokenService();

  currentProxyIndex = 0;
  currentProxyOMNodeId = omNodeIDList.get(currentProxyIndex);

  waitBetweenRetries = conf.getLong(
      OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY,
      OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
}
 

/**
 * Create a scm block client, used by putKey() and getKey().
 *
 * @return {@link ScmBlockLocationProtocol}
 * @throws IOException
 */
private static ScmBlockLocationProtocol getScmBlockClient(
    OzoneConfiguration conf) throws IOException {
  RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
      ProtobufRpcEngine.class);
  long scmVersion =
      RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
  InetSocketAddress scmBlockAddress =
      getScmAddressForBlockClients(conf);
  ScmBlockLocationProtocolClientSideTranslatorPB scmBlockLocationClient =
      new ScmBlockLocationProtocolClientSideTranslatorPB(
          RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion,
              scmBlockAddress, UserGroupInformation.getCurrentUser(), conf,
              NetUtils.getDefaultSocketFactory(conf),
              Client.getRpcTimeout(conf)));
  return TracingUtil
      .createProxy(scmBlockLocationClient, ScmBlockLocationProtocol.class,
          conf);
}
 

/**
 * Returns a scm container client.
 *
 * @return {@link StorageContainerLocationProtocol}
 * @throws IOException
 */
private static StorageContainerLocationProtocol getScmContainerClient(
    OzoneConfiguration conf) throws IOException {
  RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
      ProtobufRpcEngine.class);
  long scmVersion =
      RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
  InetSocketAddress scmAddr = getScmAddressForClients(
      conf);
  StorageContainerLocationProtocol scmContainerClient =
      TracingUtil.createProxy(
          new StorageContainerLocationProtocolClientSideTranslatorPB(
              RPC.getProxy(StorageContainerLocationProtocolPB.class,
                  scmVersion,
                  scmAddr, UserGroupInformation.getCurrentUser(), conf,
                  NetUtils.getDefaultSocketFactory(conf),
                  Client.getRpcTimeout(conf))),
          StorageContainerLocationProtocol.class, conf);
  return scmContainerClient;
}
 

/**
 * Adds a new Recon server to the set of endpoints.
 * @param address Recon address.
 * @throws IOException
 */
public void addReconServer(InetSocketAddress address) throws IOException {
  LOG.info("Adding Recon Server : {}", address.toString());
  writeLock();
  try {
    if (scmMachines.containsKey(address)) {
      LOG.warn("Trying to add an existing SCM Machine to Machines group. " +
          "Ignoring the request.");
      return;
    }
    Configuration hadoopConfig =
        LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf);
    RPC.setProtocolEngine(hadoopConfig, ReconDatanodeProtocolPB.class,
        ProtobufRpcEngine.class);
    long version =
        RPC.getProtocolVersion(ReconDatanodeProtocolPB.class);

    RetryPolicy retryPolicy =
        RetryPolicies.retryUpToMaximumCountWithFixedSleep(10,
            60000, TimeUnit.MILLISECONDS);
    ReconDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
        ReconDatanodeProtocolPB.class, version,
        address, UserGroupInformation.getCurrentUser(), hadoopConfig,
        NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(),
        retryPolicy).getProxy();

    StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
        new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy);

    EndpointStateMachine endPoint =
        new EndpointStateMachine(address, rpcClient, conf);
    endPoint.setPassive(true);
    scmMachines.put(address, endPoint);
  } finally {
    writeUnlock();
  }
}
 

/**
 * Create a scm block client, used by putKey() and getKey().
 *
 * @return {@link ScmBlockLocationProtocol}
 * @throws IOException
 */
public static SCMSecurityProtocol getScmSecurityClient(
    OzoneConfiguration conf, UserGroupInformation ugi) throws IOException {
  RPC.setProtocolEngine(conf, SCMSecurityProtocolPB.class,
      ProtobufRpcEngine.class);
  long scmVersion =
      RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
  InetSocketAddress scmSecurityProtoAdd =
      getScmAddressForSecurityProtocol(conf);
  return new SCMSecurityProtocolClientSideTranslatorPB(
      RPC.getProxy(SCMSecurityProtocolPB.class, scmVersion,
          scmSecurityProtoAdd, ugi, conf,
          NetUtils.getDefaultSocketFactory(conf),
          Client.getRpcTimeout(conf)));
}
 

public Hadoop27RpcTransport(
    ConfigurationSource conf) throws IOException {
  InetSocketAddress socket = OmUtils.getOmAddressForClients(conf);
  long version = RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
  OzoneConfiguration ozoneConfiguration = OzoneConfiguration.of(conf);

  RPC.setProtocolEngine(ozoneConfiguration,
      OzoneManagerProtocolPB.class,
      ProtobufRpcEngine.class);
  proxy = RPC.getProtocolProxy(OzoneManagerProtocolPB.class, version,
      socket, UserGroupInformation.getCurrentUser(),
      ozoneConfiguration,
      NetUtils.getDefaultSocketFactory(ozoneConfiguration),
      getRpcTimeout(ozoneConfiguration), null).getProxy();
}
 

private static ClientProtocol createNNProxyWithClientProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries, AtomicBoolean fallbackToSimpleAuth)
    throws IOException {
  RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);

  final RetryPolicy defaultPolicy = 
      RetryUtils.getDefaultRetryPolicy(
          conf, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
          SafeModeException.class);
  
  final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
  ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
      ClientNamenodeProtocolPB.class, version, address, ugi, conf,
      NetUtils.getDefaultSocketFactory(conf),
      org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy,
      fallbackToSimpleAuth).getProxy();

  if (withRetries) { // create the proxy with retries

    Map<String, RetryPolicy> methodNameToPolicyMap 
               = new HashMap<String, RetryPolicy>();
  
    ClientProtocol translatorProxy =
      new ClientNamenodeProtocolTranslatorPB(proxy);
    return (ClientProtocol) RetryProxy.create(
        ClientProtocol.class,
        new DefaultFailoverProxyProvider<ClientProtocol>(
            ClientProtocol.class, translatorProxy),
        methodNameToPolicyMap,
        defaultPolicy);
  } else {
    return new ClientNamenodeProtocolTranslatorPB(proxy);
  }
}
 

private static ClientProtocol createNNProxyWithClientProtocol(
    InetSocketAddress address, Configuration conf, UserGroupInformation ugi,
    boolean withRetries, AtomicBoolean fallbackToSimpleAuth)
    throws IOException {
  RPC.setProtocolEngine(conf, ClientNamenodeProtocolPB.class, ProtobufRpcEngine.class);

  final RetryPolicy defaultPolicy = 
      RetryUtils.getDefaultRetryPolicy(
          conf, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT, 
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
          DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
          SafeModeException.class);
  
  final long version = RPC.getProtocolVersion(ClientNamenodeProtocolPB.class);
  ClientNamenodeProtocolPB proxy = RPC.getProtocolProxy(
      ClientNamenodeProtocolPB.class, version, address, ugi, conf,
      NetUtils.getDefaultSocketFactory(conf),
      org.apache.hadoop.ipc.Client.getTimeout(conf), defaultPolicy,
      fallbackToSimpleAuth).getProxy();

  if (withRetries) { // create the proxy with retries

    Map<String, RetryPolicy> methodNameToPolicyMap 
               = new HashMap<String, RetryPolicy>();
  
    ClientProtocol translatorProxy =
      new ClientNamenodeProtocolTranslatorPB(proxy);
    return (ClientProtocol) RetryProxy.create(
        ClientProtocol.class,
        new DefaultFailoverProxyProvider<ClientProtocol>(
            ClientProtocol.class, translatorProxy),
        methodNameToPolicyMap,
        defaultPolicy);
  } else {
    return new ClientNamenodeProtocolTranslatorPB(proxy);
  }
}
 

@Override
public long getProtocolVersion(String protocol, long clientVersion)
    throws IOException {
  return RPC.getProtocolVersion(HAServiceProtocolPB.class);
}
 

/**
 * adds a new SCM machine to the target set.
 *
 * @param address - Address of the SCM machine to send heatbeat to.
 * @throws IOException
 */
public void addSCMServer(InetSocketAddress address) throws IOException {
  writeLock();
  try {
    if (scmMachines.containsKey(address)) {
      LOG.warn("Trying to add an existing SCM Machine to Machines group. " +
          "Ignoring the request.");
      return;
    }

    Configuration hadoopConfig =
        LegacyHadoopConfigurationSource.asHadoopConfiguration(this.conf);
    RPC.setProtocolEngine(
        hadoopConfig,
        StorageContainerDatanodeProtocolPB.class,
        ProtobufRpcEngine.class);
    long version =
        RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class);

    RetryPolicy retryPolicy =
        RetryPolicies.retryForeverWithFixedSleep(
            1000, TimeUnit.MILLISECONDS);

    StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProtocolProxy(
        StorageContainerDatanodeProtocolPB.class, version,
        address, UserGroupInformation.getCurrentUser(), hadoopConfig,
        NetUtils.getDefaultSocketFactory(hadoopConfig), getRpcTimeout(),
        retryPolicy).getProxy();

    StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
        new StorageContainerDatanodeProtocolClientSideTranslatorPB(
        rpcProxy);

    EndpointStateMachine endPoint =
        new EndpointStateMachine(address, rpcClient, this.conf);
    endPoint.setPassive(false);
    scmMachines.put(address, endPoint);
  } finally {
    writeUnlock();
  }
}
 

@Override
public long getProtocolVersion(String protocol, long clientVersion)
    throws IOException {
  return RPC.getProtocolVersion(ZKFCProtocolPB.class);
}
 

@Override
public long getProtocolVersion(String protocol, long clientVersion)
    throws IOException {
  return RPC.getProtocolVersion(HAServiceProtocolPB.class);
}
 

@Test
public void testGetS3Secret() throws Exception {

  // Setup secure OM for start
  setupOm(conf);
  long omVersion =
      RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
  try {
    // Start OM
    om.setCertClient(new CertificateClientTestImpl(conf));
    om.start();
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    String username = ugi.getUserName();

    // Get first OM client which will authenticate via Kerberos
    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
        OmTransportFactory.create(conf, ugi, null),
        RandomStringUtils.randomAscii(5));

    //Creates a secret since it does not exist
    S3SecretValue attempt1 = omClient.getS3Secret(username);

    //Fetches the secret from db since it was created in previous step
    S3SecretValue attempt2 = omClient.getS3Secret(username);

    //secret fetched on both attempts must be same
    assertEquals(attempt1.getAwsSecret(), attempt2.getAwsSecret());

    //access key fetched on both attempts must be same
    assertEquals(attempt1.getAwsAccessKey(), attempt2.getAwsAccessKey());


    try {
      omClient.getS3Secret("HADOOP/JOHNDOE");
      fail("testGetS3Secret failed");
    } catch (IOException ex) {
      GenericTestUtils.assertExceptionContains("USER_MISMATCH", ex);
    }
  } finally {
    IOUtils.closeQuietly(om);
  }
}
 

/**
 * Tests delegation token renewal.
 */
@Test
public void testDelegationTokenRenewal() throws Exception {
  GenericTestUtils
      .setLogLevel(LoggerFactory.getLogger(Server.class.getName()), INFO);
  LogCapturer omLogs = LogCapturer.captureLogs(OzoneManager.getLogger());

  // Setup secure OM for start.
  OzoneConfiguration newConf = new OzoneConfiguration(conf);
  int tokenMaxLifetime = 1000;
  newConf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, tokenMaxLifetime);
  setupOm(newConf);
  long omVersion =
      RPC.getProtocolVersion(OzoneManagerProtocolPB.class);
  OzoneManager.setTestSecureOmFlag(true);
  // Start OM

  try {
    om.setCertClient(new CertificateClientTestImpl(conf));
    om.start();

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();

    // Get first OM client which will authenticate via Kerberos
    omClient = new OzoneManagerProtocolClientSideTranslatorPB(
        OmTransportFactory.create(conf, ugi, null),
        RandomStringUtils.randomAscii(5));

    // Since client is already connected get a delegation token
    Token<OzoneTokenIdentifier> token = omClient.getDelegationToken(
        new Text("om"));

    // Check if token is of right kind and renewer is running om instance
    assertNotNull(token);
    assertEquals("OzoneToken", token.getKind().toString());
    assertEquals(OmUtils.getOmRpcAddress(conf),
        token.getService().toString());

    // Renew delegation token
    long expiryTime = omClient.renewDelegationToken(token);
    assertTrue(expiryTime > 0);
    omLogs.clearOutput();

    // Test failure of delegation renewal
    // 1. When token maxExpiryTime exceeds
    Thread.sleep(tokenMaxLifetime);
    OMException ex = LambdaTestUtils.intercept(OMException.class,
        "TOKEN_EXPIRED",
        () -> omClient.renewDelegationToken(token));
    assertEquals(TOKEN_EXPIRED, ex.getResult());
    omLogs.clearOutput();

    // 2. When renewer doesn't match (implicitly covers when renewer is
    // null or empty )
    Token<OzoneTokenIdentifier> token2 = omClient.getDelegationToken(
        new Text("randomService"));
    assertNotNull(token2);
    LambdaTestUtils.intercept(OMException.class,
        "Delegation token renewal failed",
        () -> omClient.renewDelegationToken(token2));
    assertTrue(omLogs.getOutput().contains(" with non-matching " +
        "renewer randomService"));
    omLogs.clearOutput();

    // 3. Test tampered token
    OzoneTokenIdentifier tokenId = OzoneTokenIdentifier.readProtoBuf(
        token.getIdentifier());
    tokenId.setRenewer(new Text("om"));
    tokenId.setMaxDate(System.currentTimeMillis() * 2);
    Token<OzoneTokenIdentifier> tamperedToken = new Token<>(
        tokenId.getBytes(), token2.getPassword(), token2.getKind(),
        token2.getService());
    LambdaTestUtils.intercept(OMException.class,
        "Delegation token renewal failed",
        () -> omClient.renewDelegationToken(tamperedToken));
    assertTrue(omLogs.getOutput().contains("can't be found in " +
        "cache"));
    omLogs.clearOutput();

  } finally {
    om.stop();
    om.join();
  }
}
 

@Override
public long getProtocolVersion(String protocol, long clientVersion)
    throws IOException {
  return RPC.getProtocolVersion(ZKFCProtocolPB.class);
}