org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecoveryFI_10. DN has no ReplicaUnderRecovery.
 *
 * @throws IOException in case of an error
 */
@Test
public void testNoReplicaUnderRecovery() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  dn.data.createRbw(StorageType.DEFAULT, block, false);
  try {
    dn.syncBlock(rBlock, initBlockRecords(dn));
    fail("Sync should fail");
  } catch (IOException e) {
    e.getMessage().startsWith("Cannot recover ");
  }
  DatanodeProtocol namenode = dn.getActiveNamenodeForBP(POOL_ID);
  verify(namenode, never()).commitBlockSynchronization(
      any(ExtendedBlock.class), anyLong(), anyLong(), anyBoolean(),
      anyBoolean(), any(DatanodeID[].class), any(String[].class));
}
 
Example #2
Source File: DataNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
void setupNS(Configuration conf, AbstractList<File> dataDirs) 
throws IOException {
  // get NN proxy
  DatanodeProtocol dnp = 
    (DatanodeProtocol)RPC.waitForProxy(DatanodeProtocol.class,
        DatanodeProtocol.versionID, nnAddr, conf);
  setNameNode(dnp);

  // handshake with NN
  NamespaceInfo nsInfo = handshake();
  setNamespaceInfo(nsInfo);
  synchronized(DataNode.this){
    setupNSStorage();
  }
  
  nsRegistration.setIpcPort(ipcServer.getListenerAddress().getPort());
  nsRegistration.setInfoPort(infoServer.getPort());
}
 
Example #3
Source File: NNThroughputBenchmark.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
int replicateBlocks() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, 0, 0);
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
      }
    }
  }
  return 0;
}
 
Example #4
Source File: PBHelper.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static BlockIdCommand convert(BlockIdCommandProto blkIdCmd) {
  int numBlockIds = blkIdCmd.getBlockIdsCount();
  long blockIds[] = new long[numBlockIds];
  for (int i = 0; i < numBlockIds; i++) {
    blockIds[i] = blkIdCmd.getBlockIds(i);
  }
  int action = DatanodeProtocol.DNA_UNKNOWN;
  switch (blkIdCmd.getAction()) {
  case CACHE:
    action = DatanodeProtocol.DNA_CACHE;
    break;
  case UNCACHE:
    action = DatanodeProtocol.DNA_UNCACHE;
    break;
  default:
    throw new AssertionError("Unknown action type: " + blkIdCmd.getAction());
  }
  return new BlockIdCommand(action, blkIdCmd.getBlockPoolId(), blockIds);
}
 
Example #5
Source File: PBHelper.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static BlockIdCommandProto convert(BlockIdCommand cmd) {
  BlockIdCommandProto.Builder builder = BlockIdCommandProto.newBuilder()
      .setBlockPoolId(cmd.getBlockPoolId());
  switch (cmd.getAction()) {
  case DatanodeProtocol.DNA_CACHE:
    builder.setAction(BlockIdCommandProto.Action.CACHE);
    break;
  case DatanodeProtocol.DNA_UNCACHE:
    builder.setAction(BlockIdCommandProto.Action.UNCACHE);
    break;
  default:
    throw new AssertionError("Invalid action");
  }
  long[] blockIds = cmd.getBlockIds();
  for (int i = 0; i < blockIds.length; i++) {
    builder.addBlockIds(blockIds[i]);
  }
  return builder.build();
}
 
Example #6
Source File: DataNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void handleDiskError(String errMsgr) {
  final boolean hasEnoughResources = data.hasEnoughResource();
  LOG.warn("DataNode.handleDiskError: Keep Running: " + hasEnoughResources);
  
  // If we have enough active valid volumes then we do not want to 
  // shutdown the DN completely.
  int dpError = hasEnoughResources ? DatanodeProtocol.DISK_ERROR  
                                   : DatanodeProtocol.FATAL_DISK_ERROR;  
  metrics.incrVolumeFailures();

  //inform NameNodes
  for(BPOfferService bpos: blockPoolManager.getAllNamenodeThreads()) {
    bpos.trySendErrorReport(dpError, errMsgr);
  }
  
  if(hasEnoughResources) {
    scheduleAllBlockReport(0);
    return; // do not shutdown
  }
  
  LOG.warn("DataNode is shutting down: " + errMsgr);
  shouldRun = false;
}
 
Example #7
Source File: NameNodeRpcServer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override // DatanodeProtocol
public void errorReport(DatanodeRegistration nodeReg,
                        int errorCode, String msg) throws IOException { 
  checkNNStartup();
  String dnName = 
     (nodeReg == null) ? "Unknown DataNode" : nodeReg.toString();

  if (errorCode == DatanodeProtocol.NOTIFY) {
    LOG.info("Error report from " + dnName + ": " + msg);
    return;
  }
  verifyRequest(nodeReg);

  if (errorCode == DatanodeProtocol.DISK_ERROR) {
    LOG.warn("Disk error on " + dnName + ": " + msg);
  } else if (errorCode == DatanodeProtocol.FATAL_DISK_ERROR) {
    LOG.warn("Fatal disk error on " + dnName + ": " + msg);
    namesystem.getBlockManager().getDatanodeManager().removeDatanode(nodeReg);            
  } else {
    LOG.info("Error report from " + dnName + ": " + msg);
  }
}
 
Example #8
Source File: NNThroughputBenchmark.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
  // register datanode
  StorageReport[] rep = { new StorageReport(storage,
      false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
      rep, 0L, 0L, 0, 0, 0, null).getCommands();
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(),
                              bcmd.getTargetStorageIDs());
      }
    }
  }
  return 0;
}
 
Example #9
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecoveryFI_07. max replica length from all DNs is zero.
 *
 * @throws IOException in case of an error
 */
@Test
public void testZeroLenReplicas() throws IOException, InterruptedException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  DataNode spyDN = spy(dn);
  doReturn(new ReplicaRecoveryInfo(block.getBlockId(), 0,
      block.getGenerationStamp(), ReplicaState.FINALIZED)).when(spyDN).
      initReplicaRecovery(any(RecoveringBlock.class));
  Daemon d = spyDN.recoverBlocks("fake NN", initRecoveringBlocks());
  d.join();
  DatanodeProtocol dnP = dn.getActiveNamenodeForBP(POOL_ID);
  verify(dnP).commitBlockSynchronization(
      block, RECOVERY_ID, 0, true, true, DatanodeID.EMPTY_ARRAY, null);
}
 
Example #10
Source File: TestBlockRecovery.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecoveryFI_10. DN has no ReplicaUnderRecovery.
 *
 * @throws IOException in case of an error
 */
@Test
public void testNoReplicaUnderRecovery() throws IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  dn.data.createRbw(StorageType.DEFAULT, block, false);
  try {
    dn.syncBlock(rBlock, initBlockRecords(dn));
    fail("Sync should fail");
  } catch (IOException e) {
    e.getMessage().startsWith("Cannot recover ");
  }
  DatanodeProtocol namenode = dn.getActiveNamenodeForBP(POOL_ID);
  verify(namenode, never()).commitBlockSynchronization(
      any(ExtendedBlock.class), anyLong(), anyLong(), anyBoolean(),
      anyBoolean(), any(DatanodeID[].class), any(String[].class));
}
 
Example #11
Source File: NameNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException {
  if (protocol.equals(ClientProtocol.class.getName())) {
    return ClientProtocol.versionID; 
  } else if (protocol.equals(DatanodeProtocol.class.getName())){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName())){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())){
    return RefreshUserMappingsProtocol.versionID;
  } else if (protocol.equals(RefreshCallQueueProtocol.class.getName())) {
    return RefreshCallQueueProtocol.versionID;
  } else if (protocol.equals(GetUserMappingsProtocol.class.getName())){
    return GetUserMappingsProtocol.versionID;
  } else if (protocol.equals(TraceAdminProtocol.class.getName())){
    return TraceAdminProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #12
Source File: NameNodeRpcServer.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override // DatanodeProtocol
public void errorReport(DatanodeRegistration nodeReg,
                        int errorCode, String msg) throws IOException { 
  checkNNStartup();
  String dnName = 
     (nodeReg == null) ? "Unknown DataNode" : nodeReg.toString();

  if (errorCode == DatanodeProtocol.NOTIFY) {
    LOG.info("Error report from " + dnName + ": " + msg);
    return;
  }
  verifyRequest(nodeReg);

  if (errorCode == DatanodeProtocol.DISK_ERROR) {
    LOG.warn("Disk error on " + dnName + ": " + msg);
  } else if (errorCode == DatanodeProtocol.FATAL_DISK_ERROR) {
    LOG.warn("Fatal disk error on " + dnName + ": " + msg);
    namesystem.getBlockManager().getDatanodeManager().removeDatanode(nodeReg);            
  } else {
    LOG.info("Error report from " + dnName + ": " + msg);
  }
}
 
Example #13
Source File: PBHelper.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static BlockIdCommandProto convert(BlockIdCommand cmd) {
  BlockIdCommandProto.Builder builder = BlockIdCommandProto.newBuilder()
      .setBlockPoolId(cmd.getBlockPoolId());
  switch (cmd.getAction()) {
  case DatanodeProtocol.DNA_CACHE:
    builder.setAction(BlockIdCommandProto.Action.CACHE);
    break;
  case DatanodeProtocol.DNA_UNCACHE:
    builder.setAction(BlockIdCommandProto.Action.UNCACHE);
    break;
  default:
    throw new AssertionError("Invalid action");
  }
  long[] blockIds = cmd.getBlockIds();
  for (int i = 0; i < blockIds.length; i++) {
    builder.addBlockIds(blockIds[i]);
  }
  return builder.build();
}
 
Example #14
Source File: NameNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 */
public void errorReport(DatanodeRegistration nodeReg,
                        int errorCode,
                        String msg) throws IOException {
  // Log error message from datanode
  String dnName = (nodeReg == null ? "unknown DataNode" : nodeReg.getName());
  LOG.info("Error report from " + dnName + ": " + msg);
  if (errorCode == DatanodeProtocol.NOTIFY) {
    return;
  }
  verifyRequest(nodeReg);
  if (errorCode == DatanodeProtocol.DISK_ERROR) {
    LOG.warn("Volume failed on " + dnName); 
  } else if (errorCode == DatanodeProtocol.FATAL_DISK_ERROR) {
    namesystem.removeDatanode(nodeReg);            
  }
}
 
Example #15
Source File: TestBlockRecovery.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * BlockRecoveryFI_07. max replica length from all DNs is zero.
 *
 * @throws IOException in case of an error
 */
@Test
public void testZeroLenReplicas() throws IOException, InterruptedException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("Running " + GenericTestUtils.getMethodName());
  }
  DataNode spyDN = spy(dn);
  doReturn(new ReplicaRecoveryInfo(block.getBlockId(), 0,
      block.getGenerationStamp(), ReplicaState.FINALIZED)).when(spyDN).
      initReplicaRecovery(any(RecoveringBlock.class));
  Daemon d = spyDN.recoverBlocks("fake NN", initRecoveringBlocks());
  d.join();
  DatanodeProtocol dnP = dn.getActiveNamenodeForBP(POOL_ID);
  verify(dnP).commitBlockSynchronization(
      block, RECOVERY_ID, 0, true, true, DatanodeID.EMPTY_ARRAY, null);
}
 
Example #16
Source File: NameNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException {
  if (protocol.equals(ClientProtocol.class.getName())) {
    return ClientProtocol.versionID; 
  } else if (protocol.equals(DatanodeProtocol.class.getName())){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName())){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else if (protocol.equals(RefreshUserMappingsProtocol.class.getName())){
    return RefreshUserMappingsProtocol.versionID;
  } else if (protocol.equals(RefreshCallQueueProtocol.class.getName())) {
    return RefreshCallQueueProtocol.versionID;
  } else if (protocol.equals(GetUserMappingsProtocol.class.getName())){
    return GetUserMappingsProtocol.versionID;
  } else if (protocol.equals(TraceAdminProtocol.class.getName())){
    return TraceAdminProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #17
Source File: TestNameNodePorts.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public void testSinglePortStartup() throws IOException {
  Configuration conf = new Configuration();
  MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
  NameNode nn = cluster.getNameNode();
  InetSocketAddress dnAddress = nn.getNameNodeDNAddress();
  InetSocketAddress clientAddress = nn.getNameNodeAddress();

  assertEquals(clientAddress, dnAddress);

  DatanodeProtocol dnProtocol = (DatanodeProtocol) RPC.waitForProxy(
      DatanodeProtocol.class, DatanodeProtocol.versionID, dnAddress, conf);
  // perform a dummy call
  dnProtocol.getProtocolVersion(DatanodeProtocol.class.getName(),
      DatanodeProtocol.versionID);
  ClientProtocol client = (ClientProtocol) RPC.waitForProxy(
      ClientProtocol.class, ClientProtocol.versionID, dnAddress, conf);
  client.getProtocolVersion(ClientProtocol.class.getName(),
      ClientProtocol.versionID);

  cluster.shutdown();
}
 
Example #18
Source File: NNThroughputBenchmark.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
int replicateBlocks() throws IOException {
  // register datanode
  DatanodeCommand[] cmds = nameNode.sendHeartbeat(
      dnRegistration, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets());
      }
    }
  }
  return 0;
}
 
Example #19
Source File: NNThroughputBenchmark.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused")
int replicateBlocks() throws IOException {
	// register datanode
	DatanodeCommand[] cmds = nameNode.sendHeartbeat(dnRegistration,
			DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0, 0);
	if (cmds != null) {
		for (DatanodeCommand cmd : cmds) {
			if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
				// Send a copy of a block to another datanode
				BlockCommand bcmd = (BlockCommand) cmd;
				return transferBlocks(bcmd.getBlocks(),
						bcmd.getTargets());
			}
		}
	}
	return 0;
}
 
Example #20
Source File: NNThroughputBenchmark.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Send a heartbeat to the name-node and replicate blocks if requested.
 */
@SuppressWarnings("unused") // keep it for future blockReceived benchmark
int replicateBlocks() throws IOException {
  // register datanode
  StorageReport[] rep = { new StorageReport(storage,
      false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
  DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
      rep, 0L, 0L, 0, 0, 0, null).getCommands();
  if (cmds != null) {
    for (DatanodeCommand cmd : cmds) {
      if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
        // Send a copy of a block to another datanode
        BlockCommand bcmd = (BlockCommand)cmd;
        return transferBlocks(bcmd.getBlocks(), bcmd.getTargets(),
                              bcmd.getTargetStorageIDs());
      }
    }
  }
  return 0;
}
 
Example #21
Source File: OfferService.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/**
 * Offer service to the specified namenode
 */
public OfferService(AvatarDataNode anode, ServicePair servicePair,
              DatanodeProtocol namenode, InetSocketAddress namenodeAddress,
              AvatarProtocol avatarnode, InetSocketAddress avatarnodeAddress) {
  this.anode = anode;
  this.servicePair = servicePair;
  this.namenode = namenode;
  this.avatarnode = avatarnode;
  this.namenodeAddress = namenodeAddress;
  this.avatarnodeAddress = avatarnodeAddress;

  nsRegistration = servicePair.nsRegistration;
  data = anode.data;
  myMetrics = anode.myMetrics;
  scheduleBlockReport(anode.initialBlockReportDelay);
  backlogSize = anode.getConf().getInt("dfs.datanode.blockreceived.backlog", 10000);
  fullBlockReportDelay = anode.getConf().getInt(
      "dfs.datanode.fullblockreport.delay", 5 * 60 * 1000);
  blockReceivedRetryInterval = anode.getConf().getInt(
      "dfs.datanode.blockreceived.retry.internval", 10000);
}
 
Example #22
Source File: PBHelper.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static BlockIdCommand convert(BlockIdCommandProto blkIdCmd) {
  int numBlockIds = blkIdCmd.getBlockIdsCount();
  long blockIds[] = new long[numBlockIds];
  for (int i = 0; i < numBlockIds; i++) {
    blockIds[i] = blkIdCmd.getBlockIds(i);
  }
  int action = DatanodeProtocol.DNA_UNKNOWN;
  switch (blkIdCmd.getAction()) {
  case CACHE:
    action = DatanodeProtocol.DNA_CACHE;
    break;
  case UNCACHE:
    action = DatanodeProtocol.DNA_UNCACHE;
    break;
  default:
    throw new AssertionError("Unknown action type: " + blkIdCmd.getAction());
  }
  return new BlockIdCommand(action, blkIdCmd.getBlockPoolId(), blockIds);
}
 
Example #23
Source File: DataNode.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void handleDiskError(String errMsgr) {
  final boolean hasEnoughResources = data.hasEnoughResource();
  LOG.warn("DataNode.handleDiskError: Keep Running: " + hasEnoughResources);
  
  // If we have enough active valid volumes then we do not want to 
  // shutdown the DN completely.
  int dpError = hasEnoughResources ? DatanodeProtocol.DISK_ERROR  
                                   : DatanodeProtocol.FATAL_DISK_ERROR;  
  metrics.incrVolumeFailures();

  //inform NameNodes
  for(BPOfferService bpos: blockPoolManager.getAllNamenodeThreads()) {
    bpos.trySendErrorReport(dpError, errMsgr);
  }
  
  if(hasEnoughResources) {
    scheduleAllBlockReport(0);
    return; // do not shutdown
  }
  
  LOG.warn("DataNode is shutting down: " + errMsgr);
  shouldRun = false;
}
 
Example #24
Source File: DataNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
private void handleDiskError(String errMsgr) {
  LOG.warn("DataNode is shutting down.\n" + errMsgr);
  shouldRun = false;
  try {
    namenode.errorReport(
                         dnRegistration, DatanodeProtocol.DISK_ERROR, errMsgr);
  } catch(IOException ignored) {              
  }
}
 
Example #25
Source File: NameNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public long getProtocolVersion(String protocol, 
                               long clientVersion) throws IOException { 
  if (protocol.equals(ClientProtocol.class.getName())) {
    return ClientProtocol.versionID; 
  } else if (protocol.equals(DatanodeProtocol.class.getName())){
    return DatanodeProtocol.versionID;
  } else if (protocol.equals(NamenodeProtocol.class.getName())){
    return NamenodeProtocol.versionID;
  } else if (protocol.equals(RefreshAuthorizationPolicyProtocol.class.getName())){
    return RefreshAuthorizationPolicyProtocol.versionID;
  } else {
    throw new IOException("Unknown protocol to name node: " + protocol);
  }
}
 
Example #26
Source File: NameNode.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/**
 */
public void errorReport(DatanodeRegistration nodeReg,
                        int errorCode, 
                        String msg) throws IOException {
  // Log error message from datanode
  String dnName = (nodeReg == null ? "unknown DataNode" : nodeReg.getName());
  LOG.info("Error report from " + dnName + ": " + msg);
  if (errorCode == DatanodeProtocol.NOTIFY) {
    return;
  }
  verifyRequest(nodeReg);
  if (errorCode == DatanodeProtocol.DISK_ERROR) {
    namesystem.removeDatanode(nodeReg);            
  }
}
 
Example #27
Source File: TestPBHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertBlockCommand() {
  Block[] blocks = new Block[] { new Block(21), new Block(22) };
  DatanodeInfo[][] dnInfos = new DatanodeInfo[][] { new DatanodeInfo[1],
      new DatanodeInfo[2] };
  dnInfos[0][0] = DFSTestUtil.getLocalDatanodeInfo();
  dnInfos[1][0] = DFSTestUtil.getLocalDatanodeInfo();
  dnInfos[1][1] = DFSTestUtil.getLocalDatanodeInfo();
  String[][] storageIDs = {{"s00"}, {"s10", "s11"}};
  StorageType[][] storageTypes = {{StorageType.DEFAULT},
      {StorageType.DEFAULT, StorageType.DEFAULT}};
  BlockCommand bc = new BlockCommand(DatanodeProtocol.DNA_TRANSFER, "bp1",
      blocks, dnInfos, storageTypes, storageIDs);
  BlockCommandProto bcProto = PBHelper.convert(bc);
  BlockCommand bc2 = PBHelper.convert(bcProto);
  assertEquals(bc.getAction(), bc2.getAction());
  assertEquals(bc.getBlocks().length, bc2.getBlocks().length);
  Block[] blocks2 = bc2.getBlocks();
  for (int i = 0; i < blocks.length; i++) {
    assertEquals(blocks[i], blocks2[i]);
  }
  DatanodeInfo[][] dnInfos2 = bc2.getTargets();
  assertEquals(dnInfos.length, dnInfos2.length);
  for (int i = 0; i < dnInfos.length; i++) {
    DatanodeInfo[] d1 = dnInfos[i];
    DatanodeInfo[] d2 = dnInfos2[i];
    assertEquals(d1.length, d2.length);
    for (int j = 0; j < d1.length; j++) {
      compare(d1[j], d2[j]);
    }
  }
}
 
Example #28
Source File: DatanodeDescriptor.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Remove the specified number of blocks to be invalidated
 */
BlockCommand getInvalidateBlocks(int maxblocks) {
  Block[] deleteList = null;
  synchronized (invalidateBlocks) {
    deleteList = invalidateBlocks.pollToArray(new Block[Math.min(
        invalidateBlocks.size(), maxblocks)]);
  }
  return (deleteList == null || deleteList.length == 0) ? 
      null: new BlockCommand(DatanodeProtocol.DNA_INVALIDATE, deleteList);
}
 
Example #29
Source File: DataNode.java    From RDFS with Apache License 2.0 5 votes vote down vote up
private void handleDiskError(String errMsgr) throws IOException{
  boolean hasEnoughResource = data.hasEnoughResource();
  myMetrics.volumeFailures.inc();
  for(Integer namespaceId : namespaceManager.getAllNamespaces()){
    DatanodeProtocol nn = getNSNamenode(namespaceId);
    LOG.warn("DataNode.handleDiskError: Keep Running: " + hasEnoughResource);
    
    //if hasEnoughtResource = true - more volumes are available, so we don't want 
    // to shutdown DN completely and don't want NN to remove it.
    int dp_error = DatanodeProtocol.DISK_ERROR;
    if(hasEnoughResource == false) {
      // DN will be shutdown and NN should remove it
      dp_error = DatanodeProtocol.FATAL_DISK_ERROR;
    }
    //inform NameNode
    try {
      nn.errorReport(getDNRegistrationForNS(namespaceId), dp_error, errMsgr);
    } catch(IOException ignored) {              
    }
    
    
    if(hasEnoughResource) {
      for (NamespaceService nsos : namespaceManager.getAllNamenodeThreads()) {
        nsos.scheduleBlockReport(0);
      }
      return; // do not shutdown
    }
  }
  
  LOG.warn("DataNode is shutting down.\n" + errMsgr);
  shouldRun = false; 
}
 
Example #30
Source File: DatanodeProtocols.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/**
 * Maximum number of protocol object encapsulated here
 */
DatanodeProtocols(int max) {
  numProtocol = max;
  node = new DatanodeProtocol[max];
  for (int i = 0; i < max; i++) {
    node[i] = null;
  }
}