Java Code Examples for org.apache.hadoop.net.NetUtils#connect()

/**
 * Create a socket for a write pipeline
 * @param first the first datanode 
 * @param length the pipeline length
 * @param client client
 * @return the socket connected to the first datanode
 */
static Socket createSocketForPipeline(final DatanodeInfo first,
    final int length, final DFSClient client) throws IOException {
  final String dnAddr = first.getXferAddr(
      client.getConf().connectToDnViaHostname);
  if (DFSClient.LOG.isDebugEnabled()) {
    DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
  }
  final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
  final Socket sock = client.socketFactory.createSocket();
  final int timeout = client.getDatanodeReadTimeout(length);
  NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().socketTimeout);
  sock.setSoTimeout(timeout);
  sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
  if(DFSClient.LOG.isDebugEnabled()) {
    DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize());
  }
  return sock;
}
 

protected void checkHdfsUriForTimeout(Configuration config) throws IOException {
    URI hdfsUri = FileSystem.getDefaultUri(config);
    String address = hdfsUri.getAuthority();
    int port = hdfsUri.getPort();
    if (address == null || address.isEmpty() || port < 0) {
        return;
    }
    InetSocketAddress namenode = NetUtils.createSocketAddr(address, port);
    SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config);
    Socket socket = null;
    try {
        socket = socketFactory.createSocket();
        NetUtils.connect(socket, namenode, 1000); // 1 second timeout
    } finally {
        IOUtils.closeQuietly(socket);
    }
}
 

protected void checkHdfsUriForTimeout(Configuration config) throws IOException {
    URI hdfsUri = FileSystem.getDefaultUri(config);
    String address = hdfsUri.getAuthority();
    int port = hdfsUri.getPort();
    if (address == null || address.isEmpty() || port < 0) {
        return;
    }
    InetSocketAddress namenode = NetUtils.createSocketAddr(address, port);
    SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config);
    Socket socket = null;
    try {
        socket = socketFactory.createSocket();
        NetUtils.connect(socket, namenode, 1000); // 1 second timeout
    } finally {
        IOUtils.closeQuietly(socket);
    }
}
 

protected void checkHdfsUriForTimeout(Configuration config) throws IOException {
    URI hdfsUri = FileSystem.getDefaultUri(config);
    String address = hdfsUri.getAuthority();
    int port = hdfsUri.getPort();
    if (address == null || address.isEmpty() || port < 0) {
        return;
    }
    InetSocketAddress namenode = NetUtils.createSocketAddr(address, port);
    SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config);
    Socket socket = null;
    try {
        socket = socketFactory.createSocket();
        NetUtils.connect(socket, namenode, 1000); // 1 second timeout
    } finally {
        IOUtils.closeQuietly(socket);
    }
}
 

@Override // RemotePeerFactory
public Peer newConnectedPeer(InetSocketAddress addr,
    Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
    throws IOException {
  Peer peer = null;
  boolean success = false;
  Socket sock = null;
  try {
    sock = socketFactory.createSocket();
    NetUtils.connect(sock, addr,
      getRandomLocalInterfaceAddr(),
      dfsClientConf.socketTimeout);
    peer = TcpPeerServer.peerFromSocketAndKey(saslClient, sock, this,
        blockToken, datanodeId);
    peer.setReadTimeout(dfsClientConf.socketTimeout);
    success = true;
    return peer;
  } finally {
    if (!success) {
      IOUtils.cleanup(LOG, peer);
      IOUtils.closeSocket(sock);
    }
  }
}
 

/**
 * Connect to the given datanode's datantrasfer port, and return
 * the resulting IOStreamPair. This includes encryption wrapping, etc.
 */
private IOStreamPair connectToDN(DatanodeInfo dn, int timeout,
    LocatedBlock lb) throws IOException {
  boolean success = false;
  Socket sock = null;
  try {
    sock = socketFactory.createSocket();
    String dnAddr = dn.getXferAddr(getConf().connectToDnViaHostname);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Connecting to datanode " + dnAddr);
    }
    NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
    sock.setSoTimeout(timeout);

    OutputStream unbufOut = NetUtils.getOutputStream(sock);
    InputStream unbufIn = NetUtils.getInputStream(sock);
    IOStreamPair ret = saslClient.newSocketSend(sock, unbufOut, unbufIn, this,
      lb.getBlockToken(), dn);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeSocket(sock);
    }
  }
}
 

/**
 * Create a socket for a write pipeline
 * @param first the first datanode 
 * @param length the pipeline length
 * @param client client
 * @return the socket connected to the first datanode
 */
static Socket createSocketForPipeline(final DatanodeInfo first,
    final int length, final DFSClient client) throws IOException {
  final String dnAddr = first.getXferAddr(
      client.getConf().connectToDnViaHostname);
  if (DFSClient.LOG.isDebugEnabled()) {
    DFSClient.LOG.debug("Connecting to datanode " + dnAddr);
  }
  final InetSocketAddress isa = NetUtils.createSocketAddr(dnAddr);
  final Socket sock = client.socketFactory.createSocket();
  final int timeout = client.getDatanodeReadTimeout(length);
  NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(), client.getConf().socketTimeout);
  sock.setSoTimeout(timeout);
  sock.setSendBufferSize(HdfsConstants.DEFAULT_DATA_SOCKET_SIZE);
  if(DFSClient.LOG.isDebugEnabled()) {
    DFSClient.LOG.debug("Send buf size " + sock.getSendBufferSize());
  }
  return sock;
}
 

@Override // RemotePeerFactory
public Peer newConnectedPeer(InetSocketAddress addr,
    Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
    throws IOException {
  Peer peer = null;
  boolean success = false;
  Socket sock = null;
  try {
    sock = socketFactory.createSocket();
    NetUtils.connect(sock, addr,
      getRandomLocalInterfaceAddr(),
      dfsClientConf.socketTimeout);
    peer = TcpPeerServer.peerFromSocketAndKey(saslClient, sock, this,
        blockToken, datanodeId);
    peer.setReadTimeout(dfsClientConf.socketTimeout);
    success = true;
    return peer;
  } finally {
    if (!success) {
      IOUtils.cleanup(LOG, peer);
      IOUtils.closeSocket(sock);
    }
  }
}
 

/**
 * Connect to the given datanode's datantrasfer port, and return
 * the resulting IOStreamPair. This includes encryption wrapping, etc.
 */
private IOStreamPair connectToDN(DatanodeInfo dn, int timeout,
    LocatedBlock lb) throws IOException {
  boolean success = false;
  Socket sock = null;
  try {
    sock = socketFactory.createSocket();
    String dnAddr = dn.getXferAddr(getConf().connectToDnViaHostname);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Connecting to datanode " + dnAddr);
    }
    NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
    sock.setSoTimeout(timeout);

    OutputStream unbufOut = NetUtils.getOutputStream(sock);
    InputStream unbufIn = NetUtils.getInputStream(sock);
    IOStreamPair ret = saslClient.newSocketSend(sock, unbufOut, unbufIn, this,
      lb.getBlockToken(), dn);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeSocket(sock);
    }
  }
}
 

protected void checkHdfsUriForTimeout(Configuration config) throws IOException {
    URI hdfsUri = FileSystem.getDefaultUri(config);
    String address = hdfsUri.getAuthority();
    int port = hdfsUri.getPort();
    if (address == null || address.isEmpty() || port < 0) {
        return;
    }
    InetSocketAddress namenode = NetUtils.createSocketAddr(address, port);
    SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config);
    Socket socket = null;
    try {
        socket = socketFactory.createSocket();
        NetUtils.connect(socket, namenode, 1000); // 1 second timeout
    } finally {
        IOUtils.closeQuietly(socket);
    }
}
 

private void doIpcVersionTest(
    byte[] requestData,
    byte[] expectedResponse) throws IOException {
  Server server = new TestServer(1, true);
  InetSocketAddress addr = NetUtils.getConnectAddress(server);
  server.start();
  Socket socket = new Socket();

  try {
    NetUtils.connect(socket, addr, 5000);
    
    OutputStream out = socket.getOutputStream();
    InputStream in = socket.getInputStream();
    out.write(requestData, 0, requestData.length);
    out.flush();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    IOUtils.copyBytes(in, baos, 256);
    
    byte[] responseData = baos.toByteArray();
    
    assertEquals(
        StringUtils.byteToHexString(expectedResponse),
        StringUtils.byteToHexString(responseData));
  } finally {
    IOUtils.closeSocket(socket);
    server.stop();
  }
}
 

protected void setupConnection() throws IOException {
  short ioFailures = 0;
  short timeoutFailures = 0;
  while (true) {
    try {
      this.socket = this.rpcClient.socketFactory.createSocket();
      this.socket.setTcpNoDelay(this.rpcClient.isTcpNoDelay());
      this.socket.setKeepAlive(this.rpcClient.tcpKeepAlive);
      if (this.rpcClient.localAddr != null) {
        this.socket.bind(this.rpcClient.localAddr);
      }
      NetUtils.connect(this.socket, remoteId.getAddress(), this.rpcClient.connectTO);
      this.socket.setSoTimeout(this.rpcClient.readTO);
      return;
    } catch (SocketTimeoutException toe) {
      /*
       * The max number of retries is 45, which amounts to 20s*45 = 15 minutes retries.
       */
      if (LOG.isDebugEnabled()) {
        LOG.debug("Received exception in connection setup.\n" +
            StringUtils.stringifyException(toe));
      }
      handleConnectionFailure(timeoutFailures++, this.rpcClient.maxRetries, toe);
    } catch (IOException ie) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Received exception in connection setup.\n" +
            StringUtils.stringifyException(ie));
      }
      handleConnectionFailure(ioFailures++, this.rpcClient.maxRetries, ie);
    }
  }
}
 

private void doTestConnection(URI host, Configuration hadoopConfiguration) throws IOException {
	SocketFactory socketFactory = NetUtils.getSocketFactory(hadoopConfiguration, ClientProtocol.class);
	Socket socket = socketFactory.createSocket();
       socket.setTcpNoDelay(false);
       SocketAddress address = new InetSocketAddress(host.getHost(), host.getPort());
       try {
       	NetUtils.connect(socket, address, CONNECTION_TEST_TIMEOUT);
	} finally {
		try {
			socket.close();
		} catch (IOException e) {}
	}
}
 

@Test
public void testMaxConnections() throws Exception {
  conf.setInt("ipc.server.max.connections", 5);
  Server server = null;
  Thread connectors[] = new Thread[10];

  try {
    server = new TestServer(3, false);
    final InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();
    assertEquals(0, server.getNumOpenConnections());

    for (int i = 0; i < 10; i++) {
      connectors[i] = new Thread() {
        @Override
        public void run() {
          Socket sock = null;
          try {
            sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
            NetUtils.connect(sock, addr, 3000);
            try {
              Thread.sleep(4000);
            } catch (InterruptedException ie) { }
          } catch (IOException ioe) {
          } finally {
            if (sock != null) {
              try {
                sock.close();
              } catch (IOException ioe) { }
            }
          }
        }
      };
      connectors[i].start();
    }

    Thread.sleep(1000);
    // server should only accept up to 5 connections
    assertEquals(5, server.getNumOpenConnections());

    for (int i = 0; i < 10; i++) {
      connectors[i].join();
    }
  } finally {
    if (server != null) {
      server.stop();
    }
    conf.setInt("ipc.server.max.connections", 0);
  }
}
 

private boolean createBlockOutputStream(DatanodeInfo[] nodes, String client,
                boolean recoveryFlag) {
  String firstBadLink = "";
  if (LOG.isDebugEnabled()) {
    for (int i = 0; i < nodes.length; i++) {
      LOG.debug("pipeline = " + nodes[i].getName());
    }
  }

  // persist blocks on namenode on next flush
  persistBlocks = true;

  try {
    LOG.debug("Connecting to " + nodes[0].getName());
    InetSocketAddress target = NetUtils.createSocketAddr(nodes[0].getName());
    s = socketFactory.createSocket();
    int timeoutValue = 3000 * nodes.length + socketTimeout;
    NetUtils.connect(s, target, timeoutValue);
    s.setSoTimeout(timeoutValue);
    s.setSendBufferSize(DEFAULT_DATA_SOCKET_SIZE);
    LOG.debug("Send buf size " + s.getSendBufferSize());
    long writeTimeout = HdfsConstants.WRITE_TIMEOUT_EXTENSION * nodes.length +
                        datanodeWriteTimeout;

    //
    // Xmit header info to datanode
    //
    DataOutputStream out = new DataOutputStream(
        new BufferedOutputStream(NetUtils.getOutputStream(s, writeTimeout), 
                                 DataNode.SMALL_BUFFER_SIZE));
    blockReplyStream = new DataInputStream(NetUtils.getInputStream(s));

    out.writeShort( DataTransferProtocol.DATA_TRANSFER_VERSION );
    out.write( DataTransferProtocol.OP_WRITE_BLOCK );
    out.writeLong( block.getBlockId() );
    out.writeLong( block.getGenerationStamp() );
    out.writeInt( nodes.length );
    out.writeBoolean( recoveryFlag );       // recovery flag
    Text.writeString( out, client );
    out.writeBoolean(false); // Not sending src node information
    out.writeInt( nodes.length - 1 );
    for (int i = 1; i < nodes.length; i++) {
      nodes[i].write(out);
    }
    checksum.writeHeader( out );
    out.flush();

    // receive ack for connect
    firstBadLink = Text.readString(blockReplyStream);
    if (firstBadLink.length() != 0) {
      throw new IOException("Bad connect ack with firstBadLink " + firstBadLink);
    }

    blockStream = out;
    return true;     // success

  } catch (IOException ie) {

    LOG.info("Exception in createBlockOutputStream " + ie);

    // find the datanode that matches
    if (firstBadLink.length() != 0) {
      for (int i = 0; i < nodes.length; i++) {
        if (nodes[i].getName().equals(firstBadLink)) {
          errorIndex = i;
          break;
        }
      }
    }
    hasError = true;
    setLastException(ie);
    blockReplyStream = null;
    return false;  // error
  }
}
 

private synchronized void setupConnection() throws IOException {
  short ioFailures = 0;
  short timeoutFailures = 0;
  while (true) {
    try {
      this.socket = socketFactory.createSocket();
      this.socket.setTcpNoDelay(tcpNoDelay);
      this.socket.setKeepAlive(true);
      
      /*
       * Bind the socket to the host specified in the principal name of the
       * client, to ensure Server matching address of the client connection
       * to host name in principal passed.
       */
      UserGroupInformation ticket = remoteId.getTicket();
      if (ticket != null && ticket.hasKerberosCredentials()) {
        KerberosInfo krbInfo = 
          remoteId.getProtocol().getAnnotation(KerberosInfo.class);
        if (krbInfo != null && krbInfo.clientPrincipal() != null) {
          String host = 
            SecurityUtil.getHostFromPrincipal(remoteId.getTicket().getUserName());
          
          // If host name is a valid local address then bind socket to it
          InetAddress localAddr = NetUtils.getLocalInetAddress(host);
          if (localAddr != null) {
            this.socket.bind(new InetSocketAddress(localAddr, 0));
          }
        }
      }
      
      NetUtils.connect(this.socket, server, connectionTimeout);
      if (rpcTimeout > 0) {
        pingInterval = rpcTimeout;  // rpcTimeout overwrites pingInterval
      }
      this.socket.setSoTimeout(pingInterval);
      return;
    } catch (ConnectTimeoutException toe) {
      /* Check for an address change and update the local reference.
       * Reset the failure counter if the address was changed
       */
      if (updateAddress()) {
        timeoutFailures = ioFailures = 0;
      }
      handleConnectionTimeout(timeoutFailures++,
          maxRetriesOnSocketTimeouts, toe);
    } catch (IOException ie) {
      if (updateAddress()) {
        timeoutFailures = ioFailures = 0;
      }
      handleConnectionFailure(ioFailures++, ie);
    }
  }
}
 

@Test
public void testMaxConnections() throws Exception {
  conf.setInt("ipc.server.max.connections", 5);
  Server server = null;
  Thread connectors[] = new Thread[10];

  try {
    server = new TestServer(3, false);
    final InetSocketAddress addr = NetUtils.getConnectAddress(server);
    server.start();
    assertEquals(0, server.getNumOpenConnections());

    for (int i = 0; i < 10; i++) {
      connectors[i] = new Thread() {
        @Override
        public void run() {
          Socket sock = null;
          try {
            sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
            NetUtils.connect(sock, addr, 3000);
            try {
              Thread.sleep(4000);
            } catch (InterruptedException ie) { }
          } catch (IOException ioe) {
          } finally {
            if (sock != null) {
              try {
                sock.close();
              } catch (IOException ioe) { }
            }
          }
        }
      };
      connectors[i].start();
    }

    Thread.sleep(1000);
    // server should only accept up to 5 connections
    assertEquals(5, server.getNumOpenConnections());

    for (int i = 0; i < 10; i++) {
      connectors[i].join();
    }
  } finally {
    if (server != null) {
      server.stop();
    }
    conf.setInt("ipc.server.max.connections", 0);
  }
}
 

/** Connect to the server and set up the I/O streams. It then sends
 * a header to the server and starts
 * the connection thread that waits for responses.
 */
private synchronized void setupIOstreams() {
  if (socket != null || shouldCloseConnection.get()) {
    return;
  }
  
  short ioFailures = 0;
  short timeoutFailures = 0;
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Connecting to "+server);
    }
    while (true) {
      try {
        this.socket = socketFactory.createSocket();
        this.socket.setTcpNoDelay(tcpNoDelay);
        // connection time out is 20s
        NetUtils.connect(this.socket, remoteId.getAddress(), 20000);
        this.socket.setSoTimeout(pingInterval);
        break;
      } catch (SocketTimeoutException toe) {
        /* The max number of retries is 45,
         * which amounts to 20s*45 = 15 minutes retries.
         */
        handleConnectionFailure(timeoutFailures++, 45, toe);
      } catch (IOException ie) {
        handleConnectionFailure(ioFailures++, maxRetries, ie);
      }
    }
    this.in = new DataInputStream(new BufferedInputStream
        (new PingInputStream(NetUtils.getInputStream(socket))));
    this.out = new DataOutputStream
        (new BufferedOutputStream(NetUtils.getOutputStream(socket)));
    writeHeader();

    // update last activity time
    touch();

    // start the receiver thread after the socket connection has been set up
    start();
  } catch (IOException e) {
    markClosed(e);
    close();
  }
}
 

/**
 * Send a generated block to a datanode.
 * 
 * @param datanode
 *            Chosen datanode name in host:port form.
 * @param blockContents
 *            Stream with the block contents.
 * @param block
 *            Block object identifying the block to be sent.
 * @param blockSize
 *            size of the block.
 * @param dataTransferVersion
 *            the data transfer version
 * @param namespaceId
 *            namespace id the block belongs to
 * @throws IOException
 */
private void sendReconstructedBlock(String datanode,
		final InputStream blockContents, DataInputStream metadataIn,
		Block block, long blockSize, int dataTransferVersion,
		int namespaceId, Progressable progress) throws IOException {
	InetSocketAddress target = NetUtils.createSocketAddr(datanode);
	Socket sock = SocketChannel.open().socket();

	int readTimeout = getConf().getInt(
			BlockIntegrityMonitor.BLOCKFIX_READ_TIMEOUT,
			HdfsConstants.READ_TIMEOUT);
	NetUtils.connect(sock, target, readTimeout);
	sock.setSoTimeout(readTimeout);

	int writeTimeout = getConf().getInt(
			BlockIntegrityMonitor.BLOCKFIX_WRITE_TIMEOUT,
			HdfsConstants.WRITE_TIMEOUT);

	OutputStream baseStream = NetUtils.getOutputStream(sock, writeTimeout);
	DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
			baseStream, FSConstants.SMALL_BUFFER_SIZE));

	boolean corruptChecksumOk = false;
	boolean chunkOffsetOK = false;
	boolean verifyChecksum = true;
	boolean transferToAllowed = false;

	try {
		LOG.info("Sending block " + block + " from "
				+ sock.getLocalSocketAddress().toString() + " to "
				+ sock.getRemoteSocketAddress().toString());
		BlockSender blockSender = new BlockSender(namespaceId, block,
				blockSize, 0, blockSize, corruptChecksumOk, chunkOffsetOK,
				verifyChecksum, transferToAllowed, metadataIn,
				new BlockSender.InputStreamFactory() {
					@Override
					public InputStream createStream(long offset)
							throws IOException {
						// we are passing 0 as the offset above,
						// so we can safely ignore
						// the offset passed
						return blockContents;
					}
				});

		// Header info
		out.writeShort(dataTransferVersion);
		out.writeByte(DataTransferProtocol.OP_WRITE_BLOCK);
		if (dataTransferVersion >= DataTransferProtocol.FEDERATION_VERSION) {
			out.writeInt(namespaceId);
		}
		out.writeLong(block.getBlockId());
		out.writeLong(block.getGenerationStamp());
		out.writeInt(0); // no pipelining
		out.writeBoolean(false); // not part of recovery
		Text.writeString(out, ""); // client
		out.writeBoolean(true); // sending src node information
		DatanodeInfo srcNode = new DatanodeInfo();
		srcNode.write(out); // Write src node DatanodeInfo
		// write targets
		out.writeInt(0); // num targets
		// send data & checksum
		blockSender.sendBlock(out, baseStream, null, progress);

		LOG.info("Sent block " + block + " to " + datanode);
	} finally {
		sock.close();
		out.close();
	}
}
 

/** Connect to the server and set up the I/O streams. It then sends
 * a header to the server and starts
 * the connection thread that waits for responses.
 */
private synchronized void setupIOstreamsWithInternal() {
  if (socket != null || shouldCloseConnection.get()) {
    return;
  }
  short ioFailures = 0;
  short timeoutFailures = 0;
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Connecting to "+server);
    }
    while (true) {
      try {
        this.socket = socketFactory.createSocket();
        this.socket.setTcpNoDelay(tcpNoDelay);
        // connection time out is 20s by default
        NetUtils.connect(this.socket, remoteId.getAddress(), connectTimeout);
        if (rpcTimeout > 0) {
          pingInterval = rpcTimeout;  // rpcTimeout overwrites pingInterval
        }
        this.socket.setSoTimeout(pingInterval);
        break;
      } catch (SocketTimeoutException toe) {
        /* The max number of retries is 45,
         * which amounts to 20s*45 = 15 minutes retries.
         */
        handleConnectionFailure(timeoutFailures++, maxRetries, toe);
      } catch (IOException ie) {
        handleConnectionFailure(ioFailures++, maxRetries, ie);
      }
    }
    this.in = new DataInputStream(new BufferedInputStream
        (new PingInputStream(NetUtils.getInputStream(socket))));
    this.out = new DataOutputStream
        (new BufferedOutputStream(NetUtils.getOutputStream(socket)));
    writeHeader();

    // update last activity time
    touch();

    // start the receiver thread after the socket connection has been set up
    start();
  } catch (IOException e) {
    markClosed(e);
    close();
  }
}