Java Code Examples for org.apache.hadoop.hdfs.DFSTestUtil#getLocalDatanodeID()

The following examples show how to use org.apache.hadoop.hdfs.DFSTestUtil#getLocalDatanodeID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestInterDatanodeProtocol.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
Example 2
Source File: TestInterDatanodeProtocol.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
Example 3
Source File: BlockManagerTestUtil.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static DatanodeDescriptor getLocalDatanodeDescriptor(
    boolean initializeStorage) {
  DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.getLocalDatanodeID());
  if (initializeStorage) {
    dn.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid()));
  }
  return dn;
}
Example 4
Source File: TestPBHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConvertDatanodeID() {
  DatanodeID dn = DFSTestUtil.getLocalDatanodeID();
  DatanodeIDProto dnProto = PBHelper.convert(dn);
  DatanodeID dn2 = PBHelper.convert(dnProto);
  compare(dn, dn2);
}
Example 5
Source File: TestPBHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConvertDatanodeRegistration() {
  DatanodeID dnId = DFSTestUtil.getLocalDatanodeID();
  BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) };
  ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10,
      getBlockKey(1), keys);
  DatanodeRegistration reg = new DatanodeRegistration(dnId,
      new StorageInfo(NodeType.DATA_NODE), expKeys, "3.0.0");
  DatanodeRegistrationProto proto = PBHelper.convert(reg);
  DatanodeRegistration reg2 = PBHelper.convert(proto);
  compare(reg.getStorageInfo(), reg2.getStorageInfo());
  compare(reg.getExportedKeys(), reg2.getExportedKeys());
  compare(reg, reg2);
  assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion());
}
Example 6
Source File: BlockManagerTestUtil.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static DatanodeDescriptor getLocalDatanodeDescriptor(
    boolean initializeStorage) {
  DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.getLocalDatanodeID());
  if (initializeStorage) {
    dn.updateStorage(new DatanodeStorage(DatanodeStorage.generateUuid()));
  }
  return dn;
}
Example 7
Source File: TestPBHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConvertDatanodeID() {
  DatanodeID dn = DFSTestUtil.getLocalDatanodeID();
  DatanodeIDProto dnProto = PBHelper.convert(dn);
  DatanodeID dn2 = PBHelper.convert(dnProto);
  compare(dn, dn2);
}
Example 8
Source File: TestPBHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testConvertDatanodeRegistration() {
  DatanodeID dnId = DFSTestUtil.getLocalDatanodeID();
  BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) };
  ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10,
      getBlockKey(1), keys);
  DatanodeRegistration reg = new DatanodeRegistration(dnId,
      new StorageInfo(NodeType.DATA_NODE), expKeys, "3.0.0");
  DatanodeRegistrationProto proto = PBHelper.convert(reg);
  DatanodeRegistration reg2 = PBHelper.convert(proto);
  compare(reg.getStorageInfo(), reg2.getStorageInfo());
  compare(reg.getExportedKeys(), reg2.getExportedKeys());
  compare(reg, reg2);
  assertEquals(reg.getSoftwareVersion(), reg2.getSoftwareVersion());
}
Example 9
Source File: TestBlockToken.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test
public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(conf);
  
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm = new BlockTokenSecretManager(
      blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
  Token<BlockTokenIdentifier> token = sm.generateToken(block3,
      EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));

  final Server server = createMockDatanode(sm, token, conf);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());

  ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
  LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);

  // Create another RPC proxy with the same configuration - this will never
  // attempt to connect anywhere -- but it causes the refcount on the
  // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
  // actually close the TCP connections to the real target DN.
  ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
      ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
      new InetSocketAddress("1.1.1.1", 1),
      UserGroupInformation.createRemoteUser("junk"), conf,
      NetUtils.getDefaultSocketFactory(conf));

  ClientDatanodeProtocol proxy = null;

  int fdsAtStart = countOpenFileDescriptors();
  try {
    long endTime = Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
          false, fakeBlock);
      assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }

    int fdsAtEnd = countOpenFileDescriptors();

    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  } finally {
    server.stop();
  }

  RPC.stopProxy(proxyToNoWhere);
}
Example 10
Source File: TestBlockToken.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Test that fast repeated invocations of createClientDatanodeProtocolProxy
 * will not end up using up thousands of sockets. This is a regression test
 * for HDFS-1965.
 */
@Test
public void testBlockTokenRpcLeak() throws Exception {
  Configuration conf = new Configuration();
  conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
  UserGroupInformation.setConfiguration(conf);
  
  Assume.assumeTrue(FD_DIR.exists());
  BlockTokenSecretManager sm = new BlockTokenSecretManager(
      blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null);
  Token<BlockTokenIdentifier> token = sm.generateToken(block3,
      EnumSet.allOf(BlockTokenSecretManager.AccessMode.class));

  final Server server = createMockDatanode(sm, token, conf);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());

  ExtendedBlock b = new ExtendedBlock("fake-pool", new Block(12345L));
  LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]);
  fakeBlock.setBlockToken(token);

  // Create another RPC proxy with the same configuration - this will never
  // attempt to connect anywhere -- but it causes the refcount on the
  // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't
  // actually close the TCP connections to the real target DN.
  ClientDatanodeProtocol proxyToNoWhere = RPC.getProxy(
      ClientDatanodeProtocol.class, ClientDatanodeProtocol.versionID,
      new InetSocketAddress("1.1.1.1", 1),
      UserGroupInformation.createRemoteUser("junk"), conf,
      NetUtils.getDefaultSocketFactory(conf));

  ClientDatanodeProtocol proxy = null;

  int fdsAtStart = countOpenFileDescriptors();
  try {
    long endTime = Time.now() + 3000;
    while (Time.now() < endTime) {
      proxy = DFSUtil.createClientDatanodeProtocolProxy(fakeDnId, conf, 1000,
          false, fakeBlock);
      assertEquals(block3.getBlockId(), proxy.getReplicaVisibleLength(block3));
      if (proxy != null) {
        RPC.stopProxy(proxy);
      }
      LOG.info("Num open fds:" + countOpenFileDescriptors());
    }

    int fdsAtEnd = countOpenFileDescriptors();

    if (fdsAtEnd - fdsAtStart > 50) {
      fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!");
    }
  } finally {
    server.stop();
  }

  RPC.stopProxy(proxyToNoWhere);
}