Java Code Examples for org.apache.hadoop.ipc.RPC#getProxy()

public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
     DatanodeID datanodeid, Configuration conf, final int socketTimeout)
   throws IOException {
   InetSocketAddress addr = NetUtils.createSocketAddr(
datanodeid.getHost() + ":" + datanodeid.getIpcPort());
   if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
     InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
   }
   UserGroupInformation ugi;
   try {
     ugi = UserGroupInformation.login(conf);
   } catch (LoginException le) {
     throw new RuntimeException("Couldn't login!");
   }
   return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
       InterDatanodeProtocol.versionID, addr,
       ugi, conf,
       NetUtils.getDefaultSocketFactory(conf), socketTimeout);
 }
 

/**
 * Command to ask the jobtracker to reread the hosts and excluded hosts 
 * file.
 * Usage: java MRAdmin -refreshNodes
 * @exception IOException 
 */
private int refreshNodes() throws IOException {
  // Get the current configuration
  Configuration conf = getConf();
  
  // Create the client
  AdminOperationsProtocol adminOperationsProtocol = 
    (AdminOperationsProtocol) 
    RPC.getProxy(AdminOperationsProtocol.class, 
                 AdminOperationsProtocol.versionID, 
                 JobTracker.getAddress(conf), getUGI(conf), conf,
                 NetUtils.getSocketFactory(conf, 
                                           AdminOperationsProtocol.class));
  
  // Refresh the queue properties
  adminOperationsProtocol.refreshNodes();
  
  return 0;
}
 

/**
 * Refresh the authorization policy on the {@link NameNode}.
 * @return exitcode 0 on success, non-zero on failure
 * @throws IOException
 */
public int refreshServiceAcl() throws IOException {
  // Get the current configuration
  Configuration conf = getConf();
  
  // Create the client
  RefreshAuthorizationPolicyProtocol refreshProtocol = 
    (RefreshAuthorizationPolicyProtocol) 
    RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, 
                 RefreshAuthorizationPolicyProtocol.versionID, 
                 NameNode.getAddress(conf), getUGI(conf), conf,
                 NetUtils.getSocketFactory(conf, 
                                           RefreshAuthorizationPolicyProtocol.class));
  
  // Refresh the authorization policy in-effect
  refreshProtocol.refreshServiceAcl();
  
  return 0;
}
 

public SCMUploaderProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMUploaderProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy =
      RPC.getProxy(SCMUploaderProtocolPB.class, clientVersion, addr, conf);
}
 

public static <PROTOCOL> PROTOCOL getProxy(
    Class<PROTOCOL> clazz, String addressStr, Configuration conf)
    throws IOException {
  RPC.setProtocolEngine(conf, clazz, ProtobufRpcEngineShaded.class);
  return RPC.getProxy(clazz, RPC.getProtocolVersion(clazz),
      org.apache.ratis.util.NetUtils.createSocketAddr(addressStr),
      UserGroupInformation.getCurrentUser(),
      conf, NetUtils.getSocketFactory(conf, clazz));
}
 

public HSClientProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  super();
  RPC.setProtocolEngine(conf, HSClientProtocolPB.class,
      ProtobufRpcEngine.class);
  proxy = (HSClientProtocolPB)RPC.getProxy(
      HSClientProtocolPB.class, clientVersion, addr, conf);
}
 

public ApplicationHistoryProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, ApplicationHistoryProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy =
      RPC.getProxy(ApplicationHistoryProtocolPB.class, clientVersion, addr,
        conf);
}
 

public HAServiceProtocolClientSideTranslatorPB(
    InetSocketAddress addr, Configuration conf,
    SocketFactory socketFactory, int timeout) throws IOException {
  RPC.setProtocolEngine(conf, HAServiceProtocolPB.class,
      ProtobufRpcEngine.class);
  rpcProxy = RPC.getProxy(HAServiceProtocolPB.class,
      RPC.getProtocolVersion(HAServiceProtocolPB.class), addr,
      UserGroupInformation.getCurrentUser(), conf, socketFactory, timeout);
}
 

private static Object createNameNodeProxy(InetSocketAddress address,
    Configuration conf, UserGroupInformation ugi, Class<?> xface)
    throws IOException {
  RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
  Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
      ugi, conf, NetUtils.getDefaultSocketFactory(conf));
  return proxy;
}
 

private static AvatarProtocol createRPCAvatarnode(
    InetSocketAddress avatarNodeAddr, Configuration conf,
    UnixUserGroupInformation ugi) throws IOException {
  LOG.info("AvatarShell connecting to " + avatarNodeAddr);
  return (AvatarProtocol) RPC.getProxy(AvatarProtocol.class,
      AvatarProtocol.versionID, avatarNodeAddr, ugi, conf,
      NetUtils.getSocketFactory(conf, AvatarProtocol.class));
}
 

/**
 * Get a client of the {@link GetUserMappingsProtocol}.
 * @return A {@link GetUserMappingsProtocol} client proxy.
 * @throws IOException
 */
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  GetUserMappingsProtocol userGroupMappingProtocol =
    RPC.getProxy(GetUserMappingsProtocol.class, 
        GetUserMappingsProtocol.versionID,
        getProtocolAddress(getConf()), UserGroupInformation.getCurrentUser(),
        getConf(), NetUtils.getSocketFactory(getConf(),
            GetUserMappingsProtocol.class));
  return userGroupMappingProtocol;
}
 

private long connect(long timeMillis) throws IOException
{
  String uriStr = fsRecoveryHandler.readConnectUri();
  if (!uriStr.equals(lastConnectURI)) {
    LOG.debug("Got new RPC connect address {}", uriStr);
    lastConnectURI = uriStr;
    if (umbilical != null) {
      RPC.stopProxy(umbilical);
    }

    retryTimeoutMillis = Long.getLong(RETRY_TIMEOUT, RETRY_TIMEOUT_DEFAULT);
    retryDelayMillis = Long.getLong(RETRY_DELAY, RETRY_DELAY_DEFAULT);
    rpcTimeout = Integer.getInteger(RPC_TIMEOUT, RPC_TIMEOUT_DEFAULT);

    URI heartbeatUri = URI.create(uriStr);

    String queryStr = heartbeatUri.getQuery();
    if (queryStr != null) {
      List<NameValuePair> queryList = URLEncodedUtils.parse(queryStr, Charset.defaultCharset());
      if (queryList != null) {
        for (NameValuePair pair : queryList) {
          String value = pair.getValue();
          String key = pair.getName();
          if (QP_rpcTimeout.equals(key)) {
            this.rpcTimeout = Integer.parseInt(value);
          } else if (QP_retryTimeoutMillis.equals(key)) {
            this.retryTimeoutMillis = Long.parseLong(value);
          } else if (QP_retryDelayMillis.equals(key)) {
            this.retryDelayMillis = Long.parseLong(value);
          }
        }
      }
    }
    InetSocketAddress address = NetUtils.createSocketAddrForHost(heartbeatUri.getHost(), heartbeatUri.getPort());
    umbilical = RPC.getProxy(StreamingContainerUmbilicalProtocol.class, StreamingContainerUmbilicalProtocol.versionID, address, currentUser, conf, defaultSocketFactory, rpcTimeout);
    // reset timeout
    return System.currentTimeMillis() + retryTimeoutMillis;
  }
  return timeMillis;
}
 

private static AvatarProtocol createRPCAvatarnode(
    InetSocketAddress avatarNodeAddr, Configuration conf,
    UnixUserGroupInformation ugi) throws IOException {
  LOG.info("AvatarShell connecting to " + avatarNodeAddr);
  return (AvatarProtocol) RPC.getProxy(AvatarProtocol.class,
      AvatarProtocol.versionID, avatarNodeAddr, ugi, conf, NetUtils
          .getSocketFactory(conf, AvatarProtocol.class));
}
 

public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, Configuration conf) throws IOException {
  InetSocketAddress addr = NetUtils.createSocketAddr(
      datanodeid.getHost() + ":" + datanodeid.getIpcPort());
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
  }
  return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
      InterDatanodeProtocol.versionID, addr, conf);
}
 

public SCMUploaderProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMUploaderProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy =
      RPC.getProxy(SCMUploaderProtocolPB.class, clientVersion, addr, conf);
}
 

/**
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test
public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(conf);
  
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm = new BlockTokenSecretManager(
      blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
  Token<BlockTokenIdentifier> token = sm.generateToken(block3,
      EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));

  final Server server = createMockDatanode(sm, token, conf);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());

  ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
  LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);

  // Create another RPC proxy with the same configuration - this will never
  // attempt to connect anywhere -- but it causes the refcount on the
  // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
  // actually close the TCP connections to the real target DN.
  ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
      ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
      new InetSocketAddress("1.1.1.1", 1),
      UserGroupInformation.createRemoteUser("junk"), conf,
      NetUtils.getDefaultSocketFactory(conf));

  ClientDatanodeProtocol proxy = null;

  int fdsAtStart = countOpenFileDescriptors();
  try {
    long endTime = Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
          false, fakeBlock);
      assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }

    int fdsAtEnd = countOpenFileDescriptors();

    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  } finally {
    server.stop();
  }

  RPC.stopProxy(proxyToNoWhere);
}
 

/**
 * Process communication constructor, every process have just only one ProcessCommSlave instance.
 * @param loginName if you use ssh to execute command, you must provide login name, e.g. ssh loginName@host "your command"
 * @param masterHost master host name
 * @param masterPort master host port
 * @throws Mp4jException
 */
public ProcessCommSlave(String loginName,
                        String masterHost,
                        int masterPort) throws Mp4jException {

    try {

        LOG.info("master host:" + masterHost + ", master port:" + masterPort);
        InetSocketAddress address = new InetSocketAddress(masterHost, masterPort);
        server = (IServer) RPC.getProxy(IServer.class, IServer.versionID, address, new Configuration());

        String host = InetAddress.getLocalHost().getHostName();
        recvDataSock = new ServerSocket(0);
        int port = recvDataSock.getLocalPort();

        Text addresses = server.getAllSlavesInfo(new Text(host + HOST_PORT_DELIM + port));

        if (address == null) {
            throw new Mp4jException("slaves connecting master failed, may be this slave restarted or master port is occupied, task failed!");
        }

        String[] slavesAddresses = addresses.toString().split(Server.ADDRESS_DELIM);
        slaveNum = slavesAddresses.length - 1;
        rank = Integer.parseInt(slavesAddresses[slaveNum]);
        rankMsgPrefix = "[rank=" + rank + "] ";

                LOG.info("this slave recv data port:" + port);
        LOG.info("slave num:" + slaveNum);
        LOG.info("slave rank:" + rank);

        // get pid
        // get name representing the running Java virtual machine.
        String name = ManagementFactory.getRuntimeMXBean().getName();
        String pid = name.split("@")[0];
        LOG.info("Pid is:" + pid);

        if (slaveNum > 1) {
            server.killMe(rank, new Text("ssh " + loginName + "@" + host +
                    " \"kill -9 " + pid + "\""));
        } else {
            server.killMe(rank, new Text("kill -9 " + pid));
        }


        slaveHosts = new String[slaveNum];
        slavePorts = new int[slaveNum];
        for (int i = 0; i < slaveNum; i++) {
            String[] addr = slavesAddresses[i].split(HOST_PORT_DELIM);
            slaveHosts[i] = addr[0];
            slavePorts[i] = Integer.parseInt(addr[1]);
        }

        LOG.info("slaves addresses:");
        for (int i = 0; i < slaveNum; i++) {
            LOG.info(slaveHosts[i] + ":" + slavePorts[i]);
        }

        this.scheduledThreadPool = Executors.newScheduledThreadPool(1);
        scheduledThreadPool.scheduleAtFixedRate(sendHeartBeatTask, 5, 15, TimeUnit.SECONDS);
        sendDataTask.start();
        recvDataTask.start();

    } catch (Exception e) {
        LOG.error("exception!", e);
        throw new Mp4jException(e.getCause());
    }

    info("this slave init finished!");
}
 

public ClientSCMProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, ClientSCMProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy = RPC.getProxy(ClientSCMProtocolPB.class, clientVersion, addr, conf);
}
 

public ApplicationClientProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, ApplicationClientProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy = RPC.getProxy(ApplicationClientProtocolPB.class, clientVersion, addr, conf);
}
 

public SCMAdminProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMAdminProtocolPB.class,
      ProtobufRpcEngine.class);
  proxy = RPC.getProxy(SCMAdminProtocolPB.class, clientVersion, addr, conf);
}