Java Code Examples for org.apache.hadoop.ipc.RPC#getProxy()

The following examples show how to use org.apache.hadoop.ipc.RPC#getProxy() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DataNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
     DatanodeID datanodeid, Configuration conf, final int socketTimeout)
   throws IOException {
   InetSocketAddress addr = NetUtils.createSocketAddr(
datanodeid.getHost() + ":" + datanodeid.getIpcPort());
   if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
     InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
   }
   UserGroupInformation ugi;
   try {
     ugi = UserGroupInformation.login(conf);
   } catch (LoginException le) {
     throw new RuntimeException("Couldn't login!");
   }
   return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
       InterDatanodeProtocol.versionID, addr,
       ugi, conf,
       NetUtils.getDefaultSocketFactory(conf), socketTimeout);
 }
 
Example 2
Source File: MRAdmin.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Command to ask the jobtracker to reread the hosts and excluded hosts 
 * file.
 * Usage: java MRAdmin -refreshNodes
 * @exception IOException 
 */
private int refreshNodes() throws IOException {
  // Get the current configuration
  Configuration conf = getConf();
  
  // Create the client
  AdminOperationsProtocol adminOperationsProtocol = 
    (AdminOperationsProtocol) 
    RPC.getProxy(AdminOperationsProtocol.class, 
                 AdminOperationsProtocol.versionID, 
                 JobTracker.getAddress(conf), getUGI(conf), conf,
                 NetUtils.getSocketFactory(conf, 
                                           AdminOperationsProtocol.class));
  
  // Refresh the queue properties
  adminOperationsProtocol.refreshNodes();
  
  return 0;
}
 
Example 3
Source File: DFSAdmin.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Refresh the authorization policy on the {@link NameNode}.
 * @return exitcode 0 on success, non-zero on failure
 * @throws IOException
 */
public int refreshServiceAcl() throws IOException {
  // Get the current configuration
  Configuration conf = getConf();
  
  // Create the client
  RefreshAuthorizationPolicyProtocol refreshProtocol = 
    (RefreshAuthorizationPolicyProtocol) 
    RPC.getProxy(RefreshAuthorizationPolicyProtocol.class, 
                 RefreshAuthorizationPolicyProtocol.versionID, 
                 NameNode.getAddress(conf), getUGI(conf), conf,
                 NetUtils.getSocketFactory(conf, 
                                           RefreshAuthorizationPolicyProtocol.class));
  
  // Refresh the authorization policy in-effect
  refreshProtocol.refreshServiceAcl();
  
  return 0;
}
 
Example 4
Source File: SCMUploaderProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public SCMUploaderProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMUploaderProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy =
      RPC.getProxy(SCMUploaderProtocolPB.class, clientVersion, addr, conf);
}
 
Example 5
Source File: Proxy.java    From ratis with Apache License 2.0 5 votes vote down vote up
public static <PROTOCOL> PROTOCOL getProxy(
    Class<PROTOCOL> clazz, String addressStr, Configuration conf)
    throws IOException {
  RPC.setProtocolEngine(conf, clazz, ProtobufRpcEngineShaded.class);
  return RPC.getProxy(clazz, RPC.getProtocolVersion(clazz),
      org.apache.ratis.util.NetUtils.createSocketAddr(addressStr),
      UserGroupInformation.getCurrentUser(),
      conf, NetUtils.getSocketFactory(conf, clazz));
}
 
Example 6
Source File: HSClientProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public HSClientProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  super();
  RPC.setProtocolEngine(conf, HSClientProtocolPB.class,
      ProtobufRpcEngine.class);
  proxy = (HSClientProtocolPB)RPC.getProxy(
      HSClientProtocolPB.class, clientVersion, addr, conf);
}
 
Example 7
Source File: ApplicationHistoryProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ApplicationHistoryProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, ApplicationHistoryProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy =
      RPC.getProxy(ApplicationHistoryProtocolPB.class, clientVersion, addr,
        conf);
}
 
Example 8
Source File: HAServiceProtocolClientSideTranslatorPB.java    From big-c with Apache License 2.0 5 votes vote down vote up
public HAServiceProtocolClientSideTranslatorPB(
    InetSocketAddress addr, Configuration conf,
    SocketFactory socketFactory, int timeout) throws IOException {
  RPC.setProtocolEngine(conf, HAServiceProtocolPB.class,
      ProtobufRpcEngine.class);
  rpcProxy = RPC.getProxy(HAServiceProtocolPB.class,
      RPC.getProtocolVersion(HAServiceProtocolPB.class), addr,
      UserGroupInformation.getCurrentUser(), conf, socketFactory, timeout);
}
 
Example 9
Source File: NameNodeProxies.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static Object createNameNodeProxy(InetSocketAddress address,
    Configuration conf, UserGroupInformation ugi, Class<?> xface)
    throws IOException {
  RPC.setProtocolEngine(conf, xface, ProtobufRpcEngine.class);
  Object proxy = RPC.getProxy(xface, RPC.getProtocolVersion(xface), address,
      ugi, conf, NetUtils.getDefaultSocketFactory(conf));
  return proxy;
}
 
Example 10
Source File: AvatarZKShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private static AvatarProtocol createRPCAvatarnode(
    InetSocketAddress avatarNodeAddr, Configuration conf,
    UnixUserGroupInformation ugi) throws IOException {
  LOG.info("AvatarShell connecting to " + avatarNodeAddr);
  return (AvatarProtocol) RPC.getProxy(AvatarProtocol.class,
      AvatarProtocol.versionID, avatarNodeAddr, ugi, conf,
      NetUtils.getSocketFactory(conf, AvatarProtocol.class));
}
 
Example 11
Source File: GetGroupsBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Get a client of the {@link GetUserMappingsProtocol}.
 * @return A {@link GetUserMappingsProtocol} client proxy.
 * @throws IOException
 */
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  GetUserMappingsProtocol userGroupMappingProtocol =
    RPC.getProxy(GetUserMappingsProtocol.class, 
        GetUserMappingsProtocol.versionID,
        getProtocolAddress(getConf()), UserGroupInformation.getCurrentUser(),
        getConf(), NetUtils.getSocketFactory(getConf(),
            GetUserMappingsProtocol.class));
  return userGroupMappingProtocol;
}
 
Example 12
Source File: RecoverableRpcProxy.java    From Bats with Apache License 2.0 5 votes vote down vote up
private long connect(long timeMillis) throws IOException
{
  String uriStr = fsRecoveryHandler.readConnectUri();
  if (!uriStr.equals(lastConnectURI)) {
    LOG.debug("Got new RPC connect address {}", uriStr);
    lastConnectURI = uriStr;
    if (umbilical != null) {
      RPC.stopProxy(umbilical);
    }

    retryTimeoutMillis = Long.getLong(RETRY_TIMEOUT, RETRY_TIMEOUT_DEFAULT);
    retryDelayMillis = Long.getLong(RETRY_DELAY, RETRY_DELAY_DEFAULT);
    rpcTimeout = Integer.getInteger(RPC_TIMEOUT, RPC_TIMEOUT_DEFAULT);

    URI heartbeatUri = URI.create(uriStr);

    String queryStr = heartbeatUri.getQuery();
    if (queryStr != null) {
      List<NameValuePair> queryList = URLEncodedUtils.parse(queryStr, Charset.defaultCharset());
      if (queryList != null) {
        for (NameValuePair pair : queryList) {
          String value = pair.getValue();
          String key = pair.getName();
          if (QP_rpcTimeout.equals(key)) {
            this.rpcTimeout = Integer.parseInt(value);
          } else if (QP_retryTimeoutMillis.equals(key)) {
            this.retryTimeoutMillis = Long.parseLong(value);
          } else if (QP_retryDelayMillis.equals(key)) {
            this.retryDelayMillis = Long.parseLong(value);
          }
        }
      }
    }
    InetSocketAddress address = NetUtils.createSocketAddrForHost(heartbeatUri.getHost(), heartbeatUri.getPort());
    umbilical = RPC.getProxy(StreamingContainerUmbilicalProtocol.class, StreamingContainerUmbilicalProtocol.versionID, address, currentUser, conf, defaultSocketFactory, rpcTimeout);
    // reset timeout
    return System.currentTimeMillis() + retryTimeoutMillis;
  }
  return timeMillis;
}
 
Example 13
Source File: AvatarShell.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private static AvatarProtocol createRPCAvatarnode(
    InetSocketAddress avatarNodeAddr, Configuration conf,
    UnixUserGroupInformation ugi) throws IOException {
  LOG.info("AvatarShell connecting to " + avatarNodeAddr);
  return (AvatarProtocol) RPC.getProxy(AvatarProtocol.class,
      AvatarProtocol.versionID, avatarNodeAddr, ugi, conf, NetUtils
          .getSocketFactory(conf, AvatarProtocol.class));
}
 
Example 14
Source File: DataNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, Configuration conf) throws IOException {
  InetSocketAddress addr = NetUtils.createSocketAddr(
      datanodeid.getHost() + ":" + datanodeid.getIpcPort());
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
  }
  return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
      InterDatanodeProtocol.versionID, addr, conf);
}
 
Example 15
Source File: SCMUploaderProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public SCMUploaderProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMUploaderProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy =
      RPC.getProxy(SCMUploaderProtocolPB.class, clientVersion, addr, conf);
}
 
Example 16
Source File: TestBlockToken.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test
public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(conf);
  
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm = new BlockTokenSecretManager(
      blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
  Token<BlockTokenIdentifier> token = sm.generateToken(block3,
      EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));

  final Server server = createMockDatanode(sm, token, conf);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());

  ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
  LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);

  // Create another RPC proxy with the same configuration - this will never
  // attempt to connect anywhere -- but it causes the refcount on the
  // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
  // actually close the TCP connections to the real target DN.
  ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
      ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
      new InetSocketAddress("1.1.1.1", 1),
      UserGroupInformation.createRemoteUser("junk"), conf,
      NetUtils.getDefaultSocketFactory(conf));

  ClientDatanodeProtocol proxy = null;

  int fdsAtStart = countOpenFileDescriptors();
  try {
    long endTime = Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
          false, fakeBlock);
      assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }

    int fdsAtEnd = countOpenFileDescriptors();

    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  } finally {
    server.stop();
  }

  RPC.stopProxy(proxyToNoWhere);
}
 
Example 17
Source File: ProcessCommSlave.java    From ytk-mp4j with MIT License 4 votes vote down vote up
/**
 * Process communication constructor, every process have just only one ProcessCommSlave instance.
 * @param loginName if you use ssh to execute command, you must provide login name, e.g. ssh loginName@host "your command"
 * @param masterHost master host name
 * @param masterPort master host port
 * @throws Mp4jException
 */
public ProcessCommSlave(String loginName,
                        String masterHost,
                        int masterPort) throws Mp4jException {

    try {

        LOG.info("master host:" + masterHost + ", master port:" + masterPort);
        InetSocketAddress address = new InetSocketAddress(masterHost, masterPort);
        server = (IServer) RPC.getProxy(IServer.class, IServer.versionID, address, new Configuration());

        String host = InetAddress.getLocalHost().getHostName();
        recvDataSock = new ServerSocket(0);
        int port = recvDataSock.getLocalPort();

        Text addresses = server.getAllSlavesInfo(new Text(host + HOST_PORT_DELIM + port));

        if (address == null) {
            throw new Mp4jException("slaves connecting master failed, may be this slave restarted or master port is occupied, task failed!");
        }

        String[] slavesAddresses = addresses.toString().split(Server.ADDRESS_DELIM);
        slaveNum = slavesAddresses.length - 1;
        rank = Integer.parseInt(slavesAddresses[slaveNum]);
        rankMsgPrefix = "[rank=" + rank + "] ";

                LOG.info("this slave recv data port:" + port);
        LOG.info("slave num:" + slaveNum);
        LOG.info("slave rank:" + rank);

        // get pid
        // get name representing the running Java virtual machine.
        String name = ManagementFactory.getRuntimeMXBean().getName();
        String pid = name.split("@")[0];
        LOG.info("Pid is:" + pid);

        if (slaveNum > 1) {
            server.killMe(rank, new Text("ssh " + loginName + "@" + host +
                    " \"kill -9 " + pid + "\""));
        } else {
            server.killMe(rank, new Text("kill -9 " + pid));
        }


        slaveHosts = new String[slaveNum];
        slavePorts = new int[slaveNum];
        for (int i = 0; i < slaveNum; i++) {
            String[] addr = slavesAddresses[i].split(HOST_PORT_DELIM);
            slaveHosts[i] = addr[0];
            slavePorts[i] = Integer.parseInt(addr[1]);
        }

        LOG.info("slaves addresses:");
        for (int i = 0; i < slaveNum; i++) {
            LOG.info(slaveHosts[i] + ":" + slavePorts[i]);
        }

        this.scheduledThreadPool = Executors.newScheduledThreadPool(1);
        scheduledThreadPool.scheduleAtFixedRate(sendHeartBeatTask, 5, 15, TimeUnit.SECONDS);
        sendDataTask.start();
        recvDataTask.start();

    } catch (Exception e) {
        LOG.error("exception!", e);
        throw new Mp4jException(e.getCause());
    }

    info("this slave init finished!");
}
 
Example 18
Source File: ClientSCMProtocolPBClientImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public ClientSCMProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, ClientSCMProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy = RPC.getProxy(ClientSCMProtocolPB.class, clientVersion, addr, conf);
}
 
Example 19
Source File: ApplicationClientProtocolPBClientImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public ApplicationClientProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, ApplicationClientProtocolPB.class,
    ProtobufRpcEngine.class);
  proxy = RPC.getProxy(ApplicationClientProtocolPB.class, clientVersion, addr, conf);
}
 
Example 20
Source File: SCMAdminProtocolPBClientImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public SCMAdminProtocolPBClientImpl(long clientVersion,
    InetSocketAddress addr, Configuration conf) throws IOException {
  RPC.setProtocolEngine(conf, SCMAdminProtocolPB.class,
      ProtobufRpcEngine.class);
  proxy = RPC.getProxy(SCMAdminProtocolPB.class, clientVersion, addr, conf);
}