org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.12.
 * One replica is RBW and another is RWR. 
 * @throws IOException in case of an error
 */
@Test
public void testRBW_RWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 
Example #2
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.11.
 * Two replicas are RBW.
 * @throws IOException in case of an error
 */
@Test
public void testRBWReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 
Example #3
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.12.
 * One replica is RBW and another is RWR. 
 * @throws IOException in case of an error
 */
@Test
public void testRBW_RWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 
Example #4
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.13. 
 * Two replicas are RWR.
 * @throws IOException in case of an error
 */
@Test
public void testRWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RWR);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 
Example #5
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/** Sync two replicas */
private void testSyncReplicas(ReplicaRecoveryInfo replica1, 
    ReplicaRecoveryInfo replica2,
    InterDatanodeProtocol dn1,
    InterDatanodeProtocol dn2,
    long expectLen) throws IOException {
  
  DatanodeInfo[] locs = new DatanodeInfo[]{
      mock(DatanodeInfo.class), mock(DatanodeInfo.class)};
  RecoveringBlock rBlock = new RecoveringBlock(block, 
      locs, RECOVERY_ID);
  ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2);
  BlockRecord record1 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn1, replica1);
  BlockRecord record2 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn2, replica2);
  syncList.add(record1);
  syncList.add(record2);
  
  when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage1");
  when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage2");
  dn.syncBlock(rBlock, syncList);
}
 
Example #6
Source File: TestInterDatanodeProtocol.java    From big-c with Apache License 2.0 6 votes vote down vote up
/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
 
Example #7
Source File: FSDataset.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Find the file corresponding to the block and return it if it exists.
 */
File getValidateBlockFile(int namespaceId, Block b, boolean checkSize)
    throws IOException {
  //Should we check for metadata file too?
  DatanodeBlockInfo blockInfo = getDatanodeBlockInfo(namespaceId, b);
  File f = null;
  if (blockInfo != null) {
    if (checkSize) {
      blockInfo.verifyFinalizedSize();
    }
    f = blockInfo.getFile();
    if(f.exists())
      return f;
 
    // if file is not null, but doesn't exist - possibly disk failed
    datanode.checkDiskError();
  }
  
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.debug("b=" + b + ", f=" + ((f == null) ? "null"
        : f));
  }
  return null;
}
 
Example #8
Source File: DataNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
     DatanodeID datanodeid, Configuration conf, final int socketTimeout)
   throws IOException {
   InetSocketAddress addr = NetUtils.createSocketAddr(
datanodeid.getHost() + ":" + datanodeid.getIpcPort());
   if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
     InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
   }
   UserGroupInformation ugi;
   try {
     ugi = UserGroupInformation.login(conf);
   } catch (LoginException le) {
     throw new RuntimeException("Couldn't login!");
   }
   return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
       InterDatanodeProtocol.versionID, addr,
       ugi, conf,
       NetUtils.getDefaultSocketFactory(conf), socketTimeout);
 }
 
Example #9
Source File: DataNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, final Configuration conf, final int socketTimeout,
    final boolean connectToDnViaHostname) throws IOException {
  final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
  final InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
  }
  final UserGroupInformation loginUgi = UserGroupInformation.getLoginUser();
  try {
    return loginUgi
        .doAs(new PrivilegedExceptionAction<InterDatanodeProtocol>() {
          @Override
          public InterDatanodeProtocol run() throws IOException {
            return new InterDatanodeProtocolTranslatorPB(addr, loginUgi,
                conf, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
          }
        });
  } catch (InterruptedException ie) {
    throw new IOException(ie.getMessage());
  }
}
 
Example #10
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.13. 
 * Two replicas are RWR.
 * @throws IOException in case of an error
 */
@Test
public void testRWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RWR);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 
Example #11
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.11.
 * Two replicas are RBW.
 * @throws IOException in case of an error
 */
@Test
public void testRBWReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 
Example #12
Source File: DataNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public Boolean call() throws Exception {
  InterDatanodeProtocol remoteDatanode = null;
  try {
    File srcBlockFile = data.getBlockFile(srcNamespaceId, srcBlock);
    remoteDatanode = DataNode
        .createInterDataNodeProtocolProxy(target, getConf(), socketTimeout);
    remoteDatanode.copyBlockLocal(srcFileSystem, srcNamespaceId, srcBlock,
        dstNamespaceId, dstBlock,
        srcBlockFile.getAbsolutePath());
  } catch (IOException e) {
    LOG.warn("Cross datanode local block copy failed", e);
    throw e;
  } finally {
    if (remoteDatanode != null) {
      stopDatanodeProxy(remoteDatanode);
    }
  }
  return true;
}
 
Example #13
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/** Sync two replicas */
private void testSyncReplicas(ReplicaRecoveryInfo replica1, 
    ReplicaRecoveryInfo replica2,
    InterDatanodeProtocol dn1,
    InterDatanodeProtocol dn2,
    long expectLen) throws IOException {
  
  DatanodeInfo[] locs = new DatanodeInfo[]{
      mock(DatanodeInfo.class), mock(DatanodeInfo.class)};
  RecoveringBlock rBlock = new RecoveringBlock(block, 
      locs, RECOVERY_ID);
  ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2);
  BlockRecord record1 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn1, replica1);
  BlockRecord record2 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn2, replica2);
  syncList.add(record1);
  syncList.add(record2);
  
  when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage1");
  when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage2");
  dn.syncBlock(rBlock, syncList);
}
 
Example #14
Source File: TestInterDatanodeProtocol.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
 
Example #15
Source File: DataNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, final Configuration conf, final int socketTimeout,
    final boolean connectToDnViaHostname) throws IOException {
  final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
  final InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
  }
  final UserGroupInformation loginUgi = UserGroupInformation.getLoginUser();
  try {
    return loginUgi
        .doAs(new PrivilegedExceptionAction<InterDatanodeProtocol>() {
          @Override
          public InterDatanodeProtocol run() throws IOException {
            return new InterDatanodeProtocolTranslatorPB(addr, loginUgi,
                conf, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
          }
        });
  } catch (InterruptedException ie) {
    throw new IOException(ie.getMessage());
  }
}
 
Example #16
Source File: DataNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
public long getProtocolVersion(String protocol, long clientVersion
    ) throws IOException {
  if (protocol.equals(InterDatanodeProtocol.class.getName())) {
    return InterDatanodeProtocol.versionID;
  } else if (protocol.equals(ClientDatanodeProtocol.class.getName())) {
    checkVersion(protocol, clientVersion, ClientDatanodeProtocol.versionID);
    return ClientDatanodeProtocol.versionID;
  }
  throw new IOException("Unknown protocol to " + getClass().getSimpleName()
      + ": " + protocol);
}
 
Example #17
Source File: DataNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** Block synchronization */
private LocatedBlock syncBlock(int namespaceId, Block block,
    List<BlockRecord> syncList, boolean closeFile,
    List<InterDatanodeProtocol> datanodeProxies, long deadline)
    throws IOException {
  return namespaceManager.get(namespaceId).syncBlock(block, syncList,
      closeFile, datanodeProxies, deadline);
}
 
Example #18
Source File: DataNode.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Convenience method, which unwraps RemoteException.
 * @throws IOException not a RemoteException.
 */
private static ReplicaRecoveryInfo callInitReplicaRecovery(
    InterDatanodeProtocol datanode,
    RecoveringBlock rBlock) throws IOException {
  try {
    return datanode.initReplicaRecovery(rBlock);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException();
  }
}
 
Example #19
Source File: TestInterDatanodeProtocol.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public static void checkMetaInfo(Block b, InterDatanodeProtocol idp,
    DataBlockScanner scanner) throws IOException {
  BlockMetaDataInfo metainfo = idp.getBlockMetaDataInfo(b);
  assertEquals(b.getBlockId(), metainfo.getBlockId());
  assertEquals(b.getNumBytes(), metainfo.getNumBytes());
  if (scanner != null) {
    assertEquals(scanner.getLastScanTime(b),
        metainfo.getLastScanTime());
  }
}
 
Example #20
Source File: FSDataset.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Get File name for a given block.
 */
public File getBlockFile(int namespaceId, Block b) throws IOException {
  File f = validateBlockFile(namespaceId, b);
  if (f == null) {
    if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
      InterDatanodeProtocol.LOG
          .debug("b=" + b + ", volumeMap=" + volumeMap);
    }
    throw new IOException("Block " + b + ", namespace= " + namespaceId
        + " is not valid.");
  }
  return f;
}
 
Example #21
Source File: AvatarDataNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
@Override
public LocatedBlock syncBlock(Block block, List<BlockRecord> syncList,
    boolean closeFile, List<InterDatanodeProtocol> datanodeProxies,
    long deadline) throws IOException {
  if (offerService1 != null && isPrimaryOfferService(offerService1))
    return offerService1.syncBlock(block, syncList, closeFile,
        datanodeProxies, deadline);
  if (offerService2 != null && isPrimaryOfferService(offerService2))
    return offerService2.syncBlock(block, syncList, closeFile,
        datanodeProxies, deadline);
  return null;
}
 
Example #22
Source File: TestInterDatanodeProtocol.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public static void checkMetaInfo(int namespaceId, Block b, InterDatanodeProtocol idp,
    DataBlockScannerSet scanner) throws IOException {
  BlockMetaDataInfo metainfo = idp.getBlockMetaDataInfo(namespaceId, b);
  assertEquals(b.getBlockId(), metainfo.getBlockId());
  assertEquals(b.getNumBytes(), metainfo.getNumBytes());    
  if (scanner != null) {
    assertEquals(scanner.getLastScanTime(namespaceId, b),
        metainfo.getLastScanTime());
  }
}
 
Example #23
Source File: FSDataset.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 * Get File name for a given block.
 */
public synchronized File getBlockFile(Block b) throws IOException {
  File f = validateBlockFile(b);
  if(f == null) {
    if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
      InterDatanodeProtocol.LOG.debug("b=" + b + ", volumeMap=" + volumeMap);
    }
    throw new IOException("Block " + b + " is not valid.");
  }
  return f;
}
 
Example #24
Source File: TestLeaseRecovery.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * test the recoverBlock does not leak clients when creating 
 * InterDatanodeProtocol RPC instances
 */
public void testForClientLeak() throws Exception {
    Client client = ClientAdapter.getClient(
      conf, NetUtils.getSocketFactory(conf, InterDatanodeProtocol.class)
    );

    DistributedFileSystem fileSystem = (DistributedFileSystem) cluster.getFileSystem();
    int initialRefCount = ClientAdapter.getRefCount(client);
    String filename = "/file1";
    DFSOutputStream out = (DFSOutputStream)
      ((DistributedFileSystem) fileSystem).getClient().create(
        filename, FsPermission.getDefault(), true, (short) 5, 1024,
        new Progressable() {
          @Override
          public void progress() {
          }
        },
        64 * 1024
      );
    out.write(DFSTestUtil.generateSequentialBytes(0, 512));
    out.sync();

    DatanodeInfo[] dataNodeInfos =
      ((DFSOutputStream)out).getPipeline();

    // killing one DN in the pipe and doing a write triggers lease recovery
    // and will result in the refcount being adjusted; if there's a lease
    // in Datanode.recoverBlock(), this will trigger it
    cluster.stopDataNode(dataNodeInfos[0].getName());
    out.write(DFSTestUtil.generateSequentialBytes(0, 512));
    assertEquals(
      "Client refcount leak!",
      initialRefCount - 1,  //-1 since we stop a DN above
      ClientAdapter.getRefCount(client)
    );

    out.close();
}
 
Example #25
Source File: FSDataset.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 * Find the file corresponding to the block and return it if it exists.
 */
File validateBlockFile(Block b) {
  //Should we check for metadata file too?
  File f = getFile(b);
  if(f != null && f.exists())
    return f;
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.debug("b=" + b + ", f=" + f);
  }
  return null;
}
 
Example #26
Source File: DataNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, Configuration conf) throws IOException {
  InetSocketAddress addr = NetUtils.createSocketAddr(
      datanodeid.getHost() + ":" + datanodeid.getIpcPort());
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
  }
  return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
      InterDatanodeProtocol.versionID, addr, conf);
}
 
Example #27
Source File: DataNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
public long getProtocolVersion(String protocol, long clientVersion
    ) throws IOException {
  if (protocol.equals(InterDatanodeProtocol.class.getName())) {
    return InterDatanodeProtocol.versionID; 
  } else if (protocol.equals(ClientDatanodeProtocol.class.getName())) {
    return ClientDatanodeProtocol.versionID; 
  }
  throw new IOException("Unknown protocol to " + getClass().getSimpleName()
      + ": " + protocol);
}
 
Example #28
Source File: DataNodeTestUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static InterDatanodeProtocol createInterDatanodeProtocolProxy(
    DataNode dn, DatanodeID datanodeid, final Configuration conf,
    boolean connectToDnViaHostname) throws IOException {
  if (connectToDnViaHostname != dn.getDnConf().connectToDnViaHostname) {
    throw new AssertionError("Unexpected DN hostname configuration");
  }
  return DataNode.createInterDataNodeProtocolProxy(datanodeid, conf,
      dn.getDnConf().socketTimeout, dn.getDnConf().connectToDnViaHostname);
}
 
Example #29
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * BlockRecovery_02.8.
 * Two replicas are in Finalized state
 * @throws IOException in case of an error
 */
@Test
public void testFinalizedReplicas () throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.FINALIZED);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);

  // two finalized replicas have different length
  replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.FINALIZED);

  try {
    testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
    Assert.fail("Two finalized replicas should not have different lengthes!");
  } catch (IOException e) {
    Assert.assertTrue(e.getMessage().startsWith(
        "Inconsistent size of finalized replicas. "));
  }
}
 
Example #30
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * BlockRecovery_02.9.
 * One replica is Finalized and another is RBW. 
 * @throws IOException in case of an error
 */
@Test
public void testFinalizedRbwReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  
  // rbw and finalized replicas have the same length
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  
  // rbw replica has a different length from the finalized one
  replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  dn1 = mock(InterDatanodeProtocol.class);
  dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}