org.apache.hadoop.hdfs.server.protocol.InterDatanodeProtocol Java Examples

/**
 * BlockRecovery_02.12.
 * One replica is RBW and another is RWR. 
 * @throws IOException in case of an error
 */
@Test
public void testRBW_RWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 

/**
 * BlockRecovery_02.11.
 * Two replicas are RBW.
 * @throws IOException in case of an error
 */
@Test
public void testRBWReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 

/**
 * BlockRecovery_02.12.
 * One replica is RBW and another is RWR. 
 * @throws IOException in case of an error
 */
@Test
public void testRBW_RWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 

/**
 * BlockRecovery_02.13. 
 * Two replicas are RWR.
 * @throws IOException in case of an error
 */
@Test
public void testRWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RWR);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 

/** Sync two replicas */
private void testSyncReplicas(ReplicaRecoveryInfo replica1, 
    ReplicaRecoveryInfo replica2,
    InterDatanodeProtocol dn1,
    InterDatanodeProtocol dn2,
    long expectLen) throws IOException {
  
  DatanodeInfo[] locs = new DatanodeInfo[]{
      mock(DatanodeInfo.class), mock(DatanodeInfo.class)};
  RecoveringBlock rBlock = new RecoveringBlock(block, 
      locs, RECOVERY_ID);
  ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2);
  BlockRecord record1 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn1, replica1);
  BlockRecord record2 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn2, replica2);
  syncList.add(record1);
  syncList.add(record2);
  
  when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage1");
  when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage2");
  dn.syncBlock(rBlock, syncList);
}
 

/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
 

/**
 * Find the file corresponding to the block and return it if it exists.
 */
File getValidateBlockFile(int namespaceId, Block b, boolean checkSize)
    throws IOException {
  //Should we check for metadata file too?
  DatanodeBlockInfo blockInfo = getDatanodeBlockInfo(namespaceId, b);
  File f = null;
  if (blockInfo != null) {
    if (checkSize) {
      blockInfo.verifyFinalizedSize();
    }
    f = blockInfo.getFile();
    if(f.exists())
      return f;
 
    // if file is not null, but doesn't exist - possibly disk failed
    datanode.checkDiskError();
  }
  
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.debug("b=" + b + ", f=" + ((f == null) ? "null"
        : f));
  }
  return null;
}
 

public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
     DatanodeID datanodeid, Configuration conf, final int socketTimeout)
   throws IOException {
   InetSocketAddress addr = NetUtils.createSocketAddr(
datanodeid.getHost() + ":" + datanodeid.getIpcPort());
   if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
     InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
   }
   UserGroupInformation ugi;
   try {
     ugi = UserGroupInformation.login(conf);
   } catch (LoginException le) {
     throw new RuntimeException("Couldn't login!");
   }
   return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
       InterDatanodeProtocol.versionID, addr,
       ugi, conf,
       NetUtils.getDefaultSocketFactory(conf), socketTimeout);
 }
 

public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, final Configuration conf, final int socketTimeout,
    final boolean connectToDnViaHostname) throws IOException {
  final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
  final InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
  }
  final UserGroupInformation loginUgi = UserGroupInformation.getLoginUser();
  try {
    return loginUgi
        .doAs(new PrivilegedExceptionAction<InterDatanodeProtocol>() {
          @Override
          public InterDatanodeProtocol run() throws IOException {
            return new InterDatanodeProtocolTranslatorPB(addr, loginUgi,
                conf, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
          }
        });
  } catch (InterruptedException ie) {
    throw new IOException(ie.getMessage());
  }
}
 

/**
 * BlockRecovery_02.13. 
 * Two replicas are RWR.
 * @throws IOException in case of an error
 */
@Test
public void testRWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RWR);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 

/**
 * BlockRecovery_02.11.
 * Two replicas are RBW.
 * @throws IOException in case of an error
 */
@Test
public void testRBWReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 

public Boolean call() throws Exception {
  InterDatanodeProtocol remoteDatanode = null;
  try {
    File srcBlockFile = data.getBlockFile(srcNamespaceId, srcBlock);
    remoteDatanode = DataNode
        .createInterDataNodeProtocolProxy(target, getConf(), socketTimeout);
    remoteDatanode.copyBlockLocal(srcFileSystem, srcNamespaceId, srcBlock,
        dstNamespaceId, dstBlock,
        srcBlockFile.getAbsolutePath());
  } catch (IOException e) {
    LOG.warn("Cross datanode local block copy failed", e);
    throw e;
  } finally {
    if (remoteDatanode != null) {
      stopDatanodeProxy(remoteDatanode);
    }
  }
  return true;
}
 

/** Sync two replicas */
private void testSyncReplicas(ReplicaRecoveryInfo replica1, 
    ReplicaRecoveryInfo replica2,
    InterDatanodeProtocol dn1,
    InterDatanodeProtocol dn2,
    long expectLen) throws IOException {
  
  DatanodeInfo[] locs = new DatanodeInfo[]{
      mock(DatanodeInfo.class), mock(DatanodeInfo.class)};
  RecoveringBlock rBlock = new RecoveringBlock(block, 
      locs, RECOVERY_ID);
  ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2);
  BlockRecord record1 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn1, replica1);
  BlockRecord record2 = new BlockRecord(
      DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn2, replica2);
  syncList.add(record1);
  syncList.add(record2);
  
  when(dn1.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage1");
  when(dn2.updateReplicaUnderRecovery((ExtendedBlock)anyObject(), anyLong(),
      anyLong(), anyLong())).thenReturn("storage2");
  dn.syncBlock(rBlock, syncList);
}
 

/** Test to verify that InterDatanode RPC timesout as expected when
 *  the server DN does not respond.
 */
@Test(expected=SocketTimeoutException.class)
public void testInterDNProtocolTimeout() throws Throwable {
  final Server server = new TestServer(1, true);
  server.start();

  final InetSocketAddress addr = NetUtils.getConnectAddress(server);
  DatanodeID fakeDnId = DFSTestUtil.getLocalDatanodeID(addr.getPort());
  DatanodeInfo dInfo = new DatanodeInfo(fakeDnId);
  InterDatanodeProtocol proxy = null;

  try {
    proxy = DataNode.createInterDataNodeProtocolProxy(
        dInfo, conf, 500, false);
    proxy.initReplicaRecovery(new RecoveringBlock(
        new ExtendedBlock("bpid", 1), null, 100));
    fail ("Expected SocketTimeoutException exception, but did not get.");
  } finally {
    if (proxy != null) {
      RPC.stopProxy(proxy);
    }
    server.stop();
  }
}
 

public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, final Configuration conf, final int socketTimeout,
    final boolean connectToDnViaHostname) throws IOException {
  final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
  final InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
  }
  final UserGroupInformation loginUgi = UserGroupInformation.getLoginUser();
  try {
    return loginUgi
        .doAs(new PrivilegedExceptionAction<InterDatanodeProtocol>() {
          @Override
          public InterDatanodeProtocol run() throws IOException {
            return new InterDatanodeProtocolTranslatorPB(addr, loginUgi,
                conf, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
          }
        });
  } catch (InterruptedException ie) {
    throw new IOException(ie.getMessage());
  }
}
 

/** {@inheritDoc} */
public long getProtocolVersion(String protocol, long clientVersion
    ) throws IOException {
  if (protocol.equals(InterDatanodeProtocol.class.getName())) {
    return InterDatanodeProtocol.versionID;
  } else if (protocol.equals(ClientDatanodeProtocol.class.getName())) {
    checkVersion(protocol, clientVersion, ClientDatanodeProtocol.versionID);
    return ClientDatanodeProtocol.versionID;
  }
  throw new IOException("Unknown protocol to " + getClass().getSimpleName()
      + ": " + protocol);
}
 

/** Block synchronization */
private LocatedBlock syncBlock(int namespaceId, Block block,
    List<BlockRecord> syncList, boolean closeFile,
    List<InterDatanodeProtocol> datanodeProxies, long deadline)
    throws IOException {
  return namespaceManager.get(namespaceId).syncBlock(block, syncList,
      closeFile, datanodeProxies, deadline);
}
 

/**
 * Convenience method, which unwraps RemoteException.
 * @throws IOException not a RemoteException.
 */
private static ReplicaRecoveryInfo callInitReplicaRecovery(
    InterDatanodeProtocol datanode,
    RecoveringBlock rBlock) throws IOException {
  try {
    return datanode.initReplicaRecovery(rBlock);
  } catch(RemoteException re) {
    throw re.unwrapRemoteException();
  }
}
 

public static void checkMetaInfo(Block b, InterDatanodeProtocol idp,
    DataBlockScanner scanner) throws IOException {
  BlockMetaDataInfo metainfo = idp.getBlockMetaDataInfo(b);
  assertEquals(b.getBlockId(), metainfo.getBlockId());
  assertEquals(b.getNumBytes(), metainfo.getNumBytes());
  if (scanner != null) {
    assertEquals(scanner.getLastScanTime(b),
        metainfo.getLastScanTime());
  }
}
 

/**
 * Get File name for a given block.
 */
public File getBlockFile(int namespaceId, Block b) throws IOException {
  File f = validateBlockFile(namespaceId, b);
  if (f == null) {
    if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
      InterDatanodeProtocol.LOG
          .debug("b=" + b + ", volumeMap=" + volumeMap);
    }
    throw new IOException("Block " + b + ", namespace= " + namespaceId
        + " is not valid.");
  }
  return f;
}
 

@Override
public LocatedBlock syncBlock(Block block, List<BlockRecord> syncList,
    boolean closeFile, List<InterDatanodeProtocol> datanodeProxies,
    long deadline) throws IOException {
  if (offerService1 != null && isPrimaryOfferService(offerService1))
    return offerService1.syncBlock(block, syncList, closeFile,
        datanodeProxies, deadline);
  if (offerService2 != null && isPrimaryOfferService(offerService2))
    return offerService2.syncBlock(block, syncList, closeFile,
        datanodeProxies, deadline);
  return null;
}
 

public static void checkMetaInfo(int namespaceId, Block b, InterDatanodeProtocol idp,
    DataBlockScannerSet scanner) throws IOException {
  BlockMetaDataInfo metainfo = idp.getBlockMetaDataInfo(namespaceId, b);
  assertEquals(b.getBlockId(), metainfo.getBlockId());
  assertEquals(b.getNumBytes(), metainfo.getNumBytes());    
  if (scanner != null) {
    assertEquals(scanner.getLastScanTime(namespaceId, b),
        metainfo.getLastScanTime());
  }
}
 

/**
 * Get File name for a given block.
 */
public synchronized File getBlockFile(Block b) throws IOException {
  File f = validateBlockFile(b);
  if(f == null) {
    if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
      InterDatanodeProtocol.LOG.debug("b=" + b + ", volumeMap=" + volumeMap);
    }
    throw new IOException("Block " + b + " is not valid.");
  }
  return f;
}
 

/**
 * test the recoverBlock does not leak clients when creating 
 * InterDatanodeProtocol RPC instances
 */
public void testForClientLeak() throws Exception {
    Client client = ClientAdapter.getClient(
      conf, NetUtils.getSocketFactory(conf, InterDatanodeProtocol.class)
    );

    DistributedFileSystem fileSystem = (DistributedFileSystem) cluster.getFileSystem();
    int initialRefCount = ClientAdapter.getRefCount(client);
    String filename = "/file1";
    DFSOutputStream out = (DFSOutputStream)
      ((DistributedFileSystem) fileSystem).getClient().create(
        filename, FsPermission.getDefault(), true, (short) 5, 1024,
        new Progressable() {
          @Override
          public void progress() {
          }
        },
        64 * 1024
      );
    out.write(DFSTestUtil.generateSequentialBytes(0, 512));
    out.sync();

    DatanodeInfo[] dataNodeInfos =
      ((DFSOutputStream)out).getPipeline();

    // killing one DN in the pipe and doing a write triggers lease recovery
    // and will result in the refcount being adjusted; if there's a lease
    // in Datanode.recoverBlock(), this will trigger it
    cluster.stopDataNode(dataNodeInfos[0].getName());
    out.write(DFSTestUtil.generateSequentialBytes(0, 512));
    assertEquals(
      "Client refcount leak!",
      initialRefCount - 1,  //-1 since we stop a DN above
      ClientAdapter.getRefCount(client)
    );

    out.close();
}
 

/**
 * Find the file corresponding to the block and return it if it exists.
 */
File validateBlockFile(Block b) {
  //Should we check for metadata file too?
  File f = getFile(b);
  if(f != null && f.exists())
    return f;
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.debug("b=" + b + ", f=" + f);
  }
  return null;
}
 

public static InterDatanodeProtocol createInterDataNodeProtocolProxy(
    DatanodeID datanodeid, Configuration conf) throws IOException {
  InetSocketAddress addr = NetUtils.createSocketAddr(
      datanodeid.getHost() + ":" + datanodeid.getIpcPort());
  if (InterDatanodeProtocol.LOG.isDebugEnabled()) {
    InterDatanodeProtocol.LOG.info("InterDatanodeProtocol addr=" + addr);
  }
  return (InterDatanodeProtocol)RPC.getProxy(InterDatanodeProtocol.class,
      InterDatanodeProtocol.versionID, addr, conf);
}
 

/** {@inheritDoc} */
public long getProtocolVersion(String protocol, long clientVersion
    ) throws IOException {
  if (protocol.equals(InterDatanodeProtocol.class.getName())) {
    return InterDatanodeProtocol.versionID; 
  } else if (protocol.equals(ClientDatanodeProtocol.class.getName())) {
    return ClientDatanodeProtocol.versionID; 
  }
  throw new IOException("Unknown protocol to " + getClass().getSimpleName()
      + ": " + protocol);
}
 

public static InterDatanodeProtocol createInterDatanodeProtocolProxy(
    DataNode dn, DatanodeID datanodeid, final Configuration conf,
    boolean connectToDnViaHostname) throws IOException {
  if (connectToDnViaHostname != dn.getDnConf().connectToDnViaHostname) {
    throw new AssertionError("Unexpected DN hostname configuration");
  }
  return DataNode.createInterDataNodeProtocolProxy(datanodeid, conf,
      dn.getDnConf().socketTimeout, dn.getDnConf().connectToDnViaHostname);
}
 

/**
 * BlockRecovery_02.8.
 * Two replicas are in Finalized state
 * @throws IOException in case of an error
 */
@Test
public void testFinalizedReplicas () throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.FINALIZED);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);

  // two finalized replicas have different length
  replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.FINALIZED);

  try {
    testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
    Assert.fail("Two finalized replicas should not have different lengthes!");
  } catch (IOException e) {
    Assert.assertTrue(e.getMessage().startsWith(
        "Inconsistent size of finalized replicas. "));
  }
}
 

/**
 * BlockRecovery_02.9.
 * One replica is Finalized and another is RBW. 
 * @throws IOException in case of an error
 */
@Test
public void testFinalizedRbwReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  
  // rbw and finalized replicas have the same length
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  
  // rbw replica has a different length from the finalized one
  replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  dn1 = mock(InterDatanodeProtocol.class);
  dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}