Java Code Examples for org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState#RBW

The following examples show how to use org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState#RBW . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.12.
 * One replica is RBW and another is RWR. 
 * @throws IOException in case of an error
 */
@Test
public void testRBW_RWRReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RWR);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 
Example 2
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.11.
 * Two replicas are RBW.
 * @throws IOException in case of an error
 */
@Test
public void testRBWReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 
Example 3
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecovery_02.11.
 * Two replicas are RBW.
 * @throws IOException in case of an error
 */
@Test
public void testRBWReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.RBW);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  long minLen = Math.min(REPLICA_LEN1, REPLICA_LEN2);
  testSyncReplicas(replica1, replica2, dn1, dn2, minLen);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, minLen);
}
 
Example 4
Source File: SimulatedFSDataset.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
throws IOException {
  ExtendedBlock b = rBlock.getBlock();
  final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
  BInfo binfo = map.get(b.getLocalBlock());
  if (binfo == null) {
    throw new IOException("No such Block " + b );  
  }

  return new ReplicaRecoveryInfo(binfo.getBlockId(), binfo.getBytesOnDisk(), 
      binfo.getGenerationStamp(), 
      binfo.isFinalized()?ReplicaState.FINALIZED : ReplicaState.RBW);
}
 
Example 5
Source File: PBHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ReplicaState convert(ReplicaStateProto state) {
  switch (state) {
  case RBW:
    return ReplicaState.RBW;
  case RUR:
    return ReplicaState.RUR;
  case RWR:
    return ReplicaState.RWR;
  case TEMPORARY:
    return ReplicaState.TEMPORARY;
  case FINALIZED:
  default:
    return ReplicaState.FINALIZED;
  }
}
 
Example 6
Source File: ReplicaUnderRecovery.java    From big-c with Apache License 2.0 5 votes vote down vote up
public ReplicaUnderRecovery(ReplicaInfo replica, long recoveryId) {
  super(replica, replica.getVolume(), replica.getDir());
  if ( replica.getState() != ReplicaState.FINALIZED &&
       replica.getState() != ReplicaState.RBW &&
       replica.getState() != ReplicaState.RWR ) {
    throw new IllegalArgumentException("Cannot recover replica: " + replica);
  }
  this.original = replica;
  this.recoveryId = recoveryId;
}
 
Example 7
Source File: FsDatasetImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override // FsDatasetSpi
public synchronized String recoverClose(ExtendedBlock b, long newGS,
    long expectedBlockLen) throws IOException {
  LOG.info("Recover failed close " + b);
  // check replica's state
  ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
  // bump the replica's GS
  bumpReplicaGS(replicaInfo, newGS);
  // finalize the replica if RBW
  if (replicaInfo.getState() == ReplicaState.RBW) {
    finalizeReplica(b.getBlockPoolId(), replicaInfo);
  }
  return replicaInfo.getStorageUuid();
}
 
Example 8
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * BlockRecovery_02.9.
 * One replica is Finalized and another is RBW. 
 * @throws IOException in case of an error
 */
@Test
public void testFinalizedRbwReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  
  // rbw and finalized replicas have the same length
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  
  // rbw replica has a different length from the finalized one
  replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  dn1 = mock(InterDatanodeProtocol.class);
  dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 
Example 9
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * BlockRecovery_02.9.
 * One replica is Finalized and another is RBW. 
 * @throws IOException in case of an error
 */
@Test
public void testFinalizedRbwReplicas() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  
  // rbw and finalized replicas have the same length
  ReplicaRecoveryInfo replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  ReplicaRecoveryInfo replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-2, ReplicaState.RBW);

  InterDatanodeProtocol dn1 = mock(InterDatanodeProtocol.class);
  InterDatanodeProtocol dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  verify(dn2).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID,
      REPLICA_LEN1);
  
  // rbw replica has a different length from the finalized one
  replica1 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN1, GEN_STAMP-1, ReplicaState.FINALIZED);
  replica2 = new ReplicaRecoveryInfo(BLOCK_ID, 
      REPLICA_LEN2, GEN_STAMP-2, ReplicaState.RBW);

  dn1 = mock(InterDatanodeProtocol.class);
  dn2 = mock(InterDatanodeProtocol.class);

  testSyncReplicas(replica1, replica2, dn1, dn2, REPLICA_LEN1);
  verify(dn1).updateReplicaUnderRecovery(block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
  verify(dn2, never()).updateReplicaUnderRecovery(
      block, RECOVERY_ID, BLOCK_ID, REPLICA_LEN1);
}
 
Example 10
Source File: SimulatedFSDataset.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public ReplicaRecoveryInfo initReplicaRecovery(RecoveringBlock rBlock)
throws IOException {
  ExtendedBlock b = rBlock.getBlock();
  final Map<Block, BInfo> map = getMap(b.getBlockPoolId());
  BInfo binfo = map.get(b.getLocalBlock());
  if (binfo == null) {
    throw new IOException("No such Block " + b );  
  }

  return new ReplicaRecoveryInfo(binfo.getBlockId(), binfo.getBytesOnDisk(), 
      binfo.getGenerationStamp(), 
      binfo.isFinalized()?ReplicaState.FINALIZED : ReplicaState.RBW);
}
 
Example 11
Source File: PBHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ReplicaState convert(ReplicaStateProto state) {
  switch (state) {
  case RBW:
    return ReplicaState.RBW;
  case RUR:
    return ReplicaState.RUR;
  case RWR:
    return ReplicaState.RWR;
  case TEMPORARY:
    return ReplicaState.TEMPORARY;
  case FINALIZED:
  default:
    return ReplicaState.FINALIZED;
  }
}
 
Example 12
Source File: ReplicaUnderRecovery.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ReplicaUnderRecovery(ReplicaInfo replica, long recoveryId) {
  super(replica, replica.getVolume(), replica.getDir());
  if ( replica.getState() != ReplicaState.FINALIZED &&
       replica.getState() != ReplicaState.RBW &&
       replica.getState() != ReplicaState.RWR ) {
    throw new IllegalArgumentException("Cannot recover replica: " + replica);
  }
  this.original = replica;
  this.recoveryId = recoveryId;
}
 
Example 13
Source File: FsDatasetImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override // FsDatasetSpi
public synchronized String recoverClose(ExtendedBlock b, long newGS,
    long expectedBlockLen) throws IOException {
  LOG.info("Recover failed close " + b);
  // check replica's state
  ReplicaInfo replicaInfo = recoverCheck(b, newGS, expectedBlockLen);
  // bump the replica's GS
  bumpReplicaGS(replicaInfo, newGS);
  // finalize the replica if RBW
  if (replicaInfo.getState() == ReplicaState.RBW) {
    finalizeReplica(b.getBlockPoolId(), replicaInfo);
  }
  return replicaInfo.getStorageUuid();
}
 
Example 14
Source File: SimulatedFSDataset.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public ReplicaState getState() {
  return finalized ? ReplicaState.FINALIZED : ReplicaState.RBW;
}
 
Example 15
Source File: FsDatasetImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
private ReplicaInfo recoverCheck(ExtendedBlock b, long newGS, 
    long expectedBlockLen) throws IOException {
  ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
  
  // check state
  if (replicaInfo.getState() != ReplicaState.FINALIZED &&
      replicaInfo.getState() != ReplicaState.RBW) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.UNFINALIZED_AND_NONRBW_REPLICA + replicaInfo);
  }

  // check generation stamp
  long replicaGenerationStamp = replicaInfo.getGenerationStamp();
  if (replicaGenerationStamp < b.getGenerationStamp() ||
      replicaGenerationStamp > newGS) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.UNEXPECTED_GS_REPLICA + replicaGenerationStamp
        + ". Expected GS range is [" + b.getGenerationStamp() + ", " + 
        newGS + "].");
  }
  
  // stop the previous writer before check a replica's length
  long replicaLen = replicaInfo.getNumBytes();
  if (replicaInfo.getState() == ReplicaState.RBW) {
    ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo;
    // kill the previous writer
    rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout());
    rbw.setWriter(Thread.currentThread());
    // check length: bytesRcvd, bytesOnDisk, and bytesAcked should be the same
    if (replicaLen != rbw.getBytesOnDisk() 
        || replicaLen != rbw.getBytesAcked()) {
      throw new ReplicaAlreadyExistsException("RBW replica " + replicaInfo + 
          "bytesRcvd(" + rbw.getNumBytes() + "), bytesOnDisk(" + 
          rbw.getBytesOnDisk() + "), and bytesAcked(" + rbw.getBytesAcked() +
          ") are not the same.");
    }
  }
  
  // check block length
  if (replicaLen != expectedBlockLen) {
    throw new IOException("Corrupted replica " + replicaInfo + 
        " with a length of " + replicaLen + 
        " expected length is " + expectedBlockLen);
  }
  
  return replicaInfo;
}
 
Example 16
Source File: FsDatasetImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override // FsDatasetSpi
public synchronized ReplicaHandler recoverRbw(
    ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
    throws IOException {
  LOG.info("Recover RBW replica " + b);

  ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
  
  // check the replica's state
  if (replicaInfo.getState() != ReplicaState.RBW) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.NON_RBW_REPLICA + replicaInfo);
  }
  ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo;
  
  LOG.info("Recovering " + rbw);

  // Stop the previous writer
  rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout());
  rbw.setWriter(Thread.currentThread());

  // check generation stamp
  long replicaGenerationStamp = rbw.getGenerationStamp();
  if (replicaGenerationStamp < b.getGenerationStamp() ||
      replicaGenerationStamp > newGS) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.UNEXPECTED_GS_REPLICA + b +
        ". Expected GS range is [" + b.getGenerationStamp() + ", " + 
        newGS + "].");
  }
  
  // check replica length
  long bytesAcked = rbw.getBytesAcked();
  long numBytes = rbw.getNumBytes();
  if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd){
    throw new ReplicaNotFoundException("Unmatched length replica " + 
        replicaInfo + ": BytesAcked = " + bytesAcked + 
        " BytesRcvd = " + numBytes + " are not in the range of [" + 
        minBytesRcvd + ", " + maxBytesRcvd + "].");
  }

  FsVolumeReference ref = rbw.getVolume().obtainReference();
  try {
    // Truncate the potentially corrupt portion.
    // If the source was client and the last node in the pipeline was lost,
    // any corrupt data written after the acked length can go unnoticed.
    if (numBytes > bytesAcked) {
      final File replicafile = rbw.getBlockFile();
      truncateBlock(replicafile, rbw.getMetaFile(), numBytes, bytesAcked);
      rbw.setNumBytes(bytesAcked);
      rbw.setLastChecksumAndDataLen(bytesAcked, null);
    }

    // bump the replica's generation stamp to newGS
    bumpReplicaGS(rbw, newGS);
  } catch (IOException e) {
    IOUtils.cleanup(null, ref);
    throw e;
  }
  return new ReplicaHandler(rbw, ref);
}
 
Example 17
Source File: ReplicaBeingWritten.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override   //ReplicaInfo
public ReplicaState getState() {
  return ReplicaState.RBW;
}
 
Example 18
Source File: ReplicaBeingWritten.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override   //ReplicaInfo
public ReplicaState getState() {
  return ReplicaState.RBW;
}
 
Example 19
Source File: FsDatasetImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override // FsDatasetSpi
public synchronized ReplicaHandler recoverRbw(
    ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd)
    throws IOException {
  LOG.info("Recover RBW replica " + b);

  ReplicaInfo replicaInfo = getReplicaInfo(b.getBlockPoolId(), b.getBlockId());
  
  // check the replica's state
  if (replicaInfo.getState() != ReplicaState.RBW) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.NON_RBW_REPLICA + replicaInfo);
  }
  ReplicaBeingWritten rbw = (ReplicaBeingWritten)replicaInfo;
  
  LOG.info("Recovering " + rbw);

  // Stop the previous writer
  rbw.stopWriter(datanode.getDnConf().getXceiverStopTimeout());
  rbw.setWriter(Thread.currentThread());

  // check generation stamp
  long replicaGenerationStamp = rbw.getGenerationStamp();
  if (replicaGenerationStamp < b.getGenerationStamp() ||
      replicaGenerationStamp > newGS) {
    throw new ReplicaNotFoundException(
        ReplicaNotFoundException.UNEXPECTED_GS_REPLICA + b +
        ". Expected GS range is [" + b.getGenerationStamp() + ", " + 
        newGS + "].");
  }
  
  // check replica length
  long bytesAcked = rbw.getBytesAcked();
  long numBytes = rbw.getNumBytes();
  if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd){
    throw new ReplicaNotFoundException("Unmatched length replica " + 
        replicaInfo + ": BytesAcked = " + bytesAcked + 
        " BytesRcvd = " + numBytes + " are not in the range of [" + 
        minBytesRcvd + ", " + maxBytesRcvd + "].");
  }

  FsVolumeReference ref = rbw.getVolume().obtainReference();
  try {
    // Truncate the potentially corrupt portion.
    // If the source was client and the last node in the pipeline was lost,
    // any corrupt data written after the acked length can go unnoticed.
    if (numBytes > bytesAcked) {
      final File replicafile = rbw.getBlockFile();
      truncateBlock(replicafile, rbw.getMetaFile(), numBytes, bytesAcked);
      rbw.setNumBytes(bytesAcked);
      rbw.setLastChecksumAndDataLen(bytesAcked, null);
    }

    // bump the replica's generation stamp to newGS
    bumpReplicaGS(rbw, newGS);
  } catch (IOException e) {
    IOUtils.cleanup(null, ref);
    throw e;
  }
  return new ReplicaHandler(rbw, ref);
}
 
Example 20
Source File: SimulatedFSDataset.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public ReplicaState getState() {
  return finalized ? ReplicaState.FINALIZED : ReplicaState.RBW;
}