Java Code Examples for org.apache.hadoop.hdfs.util.LightWeightLinkedSet#contains()

The following examples show how to use org.apache.hadoop.hdfs.util.LightWeightLinkedSet#contains() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BlockManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Return the number of nodes hosting a given block, grouped
 * by the state of those replicas.
 */
public NumberReplicas countNodes(Block b) {
  int decommissioned = 0;
  int live = 0;
  int corrupt = 0;
  int excess = 0;
  int stale = 0;
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) {
      corrupt++;
    } else if (node.isDecommissionInProgress() || node.isDecommissioned()) {
      decommissioned++;
    } else {
      LightWeightLinkedSet<Block> blocksExcess = excessReplicateMap.get(node
          .getDatanodeUuid());
      if (blocksExcess != null && blocksExcess.contains(b)) {
        excess++;
      } else {
        live++;
      }
    }
    if (storage.areBlockContentsStale()) {
      stale++;
    }
  }
  return new NumberReplicas(live, decommissioned, corrupt, excess, stale);
}
Example 2
Source File: UnderReplicatedBlocks.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/** Check if a block is in the neededReplication queue */
synchronized boolean contains(Block block) {
  for(LightWeightLinkedSet<Block> set : priorityQueues) {
    if (set.contains(block)) {
      return true;
    }
  }
  return false;
}
Example 3
Source File: BlockManager.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Return the number of nodes hosting a given block, grouped
 * by the state of those replicas.
 */
public NumberReplicas countNodes(Block b) {
  int decommissioned = 0;
  int live = 0;
  int corrupt = 0;
  int excess = 0;
  int stale = 0;
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) {
      corrupt++;
    } else if (node.isDecommissionInProgress() || node.isDecommissioned()) {
      decommissioned++;
    } else {
      LightWeightLinkedSet<Block> blocksExcess = excessReplicateMap.get(node
          .getDatanodeUuid());
      if (blocksExcess != null && blocksExcess.contains(b)) {
        excess++;
      } else {
        live++;
      }
    }
    if (storage.areBlockContentsStale()) {
      stale++;
    }
  }
  return new NumberReplicas(live, decommissioned, corrupt, excess, stale);
}
Example 4
Source File: UnderReplicatedBlocks.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/** Check if a block is in the neededReplication queue */
synchronized boolean contains(Block block) {
  for(LightWeightLinkedSet<Block> set : priorityQueues) {
    if (set.contains(block)) {
      return true;
    }
  }
  return false;
}
Example 5
Source File: BlockManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Parse the data-nodes the block belongs to and choose one,
 * which will be the replication source.
 *
 * We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
 * since the former do not have write traffic and hence are less busy.
 * We do not use already decommissioned nodes as a source.
 * Otherwise we choose a random node among those that did not reach their
 * replication limits.  However, if the replication is of the highest priority
 * and all nodes have reached their replication limits, we will choose a
 * random node despite the replication limit.
 *
 * In addition form a list of all nodes containing the block
 * and calculate its replication numbers.
 *
 * @param block Block for which a replication source is needed
 * @param containingNodes List to be populated with nodes found to contain the 
 *                        given block
 * @param nodesContainingLiveReplicas List to be populated with nodes found to
 *                                    contain live replicas of the given block
 * @param numReplicas NumberReplicas instance to be initialized with the 
 *                                   counts of live, corrupt, excess, and
 *                                   decommissioned replicas of the given
 *                                   block.
 * @param priority integer representing replication priority of the given
 *                 block
 * @return the DatanodeDescriptor of the chosen node from which to replicate
 *         the given block
 */
 @VisibleForTesting
 DatanodeDescriptor chooseSourceDatanode(Block block,
     List<DatanodeDescriptor> containingNodes,
     List<DatanodeStorageInfo>  nodesContainingLiveReplicas,
     NumberReplicas numReplicas,
     int priority) {
  containingNodes.clear();
  nodesContainingLiveReplicas.clear();
  DatanodeDescriptor srcNode = null;
  int live = 0;
  int decommissioned = 0;
  int corrupt = 0;
  int excess = 0;
  
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    LightWeightLinkedSet<Block> excessBlocks =
      excessReplicateMap.get(node.getDatanodeUuid());
    int countableReplica = storage.getState() == State.NORMAL ? 1 : 0; 
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
      corrupt += countableReplica;
    else if (node.isDecommissionInProgress() || node.isDecommissioned())
      decommissioned += countableReplica;
    else if (excessBlocks != null && excessBlocks.contains(block)) {
      excess += countableReplica;
    } else {
      nodesContainingLiveReplicas.add(storage);
      live += countableReplica;
    }
    containingNodes.add(node);
    // Check if this replica is corrupt
    // If so, do not select the node as src node
    if ((nodesCorrupt != null) && nodesCorrupt.contains(node))
      continue;
    if(priority != UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY 
        && !node.isDecommissionInProgress() 
        && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams)
    {
      continue; // already reached replication limit
    }
    if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit)
    {
      continue;
    }
    // the block must not be scheduled for removal on srcNode
    if(excessBlocks != null && excessBlocks.contains(block))
      continue;
    // never use already decommissioned nodes
    if(node.isDecommissioned())
      continue;

    // We got this far, current node is a reasonable choice
    if (srcNode == null) {
      srcNode = node;
      continue;
    }
    // switch to a different node randomly
    // this to prevent from deterministically selecting the same node even
    // if the node failed to replicate the block on previous iterations
    if(DFSUtil.getRandom().nextBoolean())
      srcNode = node;
  }
  if(numReplicas != null)
    numReplicas.initialize(live, decommissioned, corrupt, excess, 0);
  return srcNode;
}
Example 6
Source File: BlockManager.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Parse the data-nodes the block belongs to and choose one,
 * which will be the replication source.
 *
 * We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
 * since the former do not have write traffic and hence are less busy.
 * We do not use already decommissioned nodes as a source.
 * Otherwise we choose a random node among those that did not reach their
 * replication limits.  However, if the replication is of the highest priority
 * and all nodes have reached their replication limits, we will choose a
 * random node despite the replication limit.
 *
 * In addition form a list of all nodes containing the block
 * and calculate its replication numbers.
 *
 * @param block Block for which a replication source is needed
 * @param containingNodes List to be populated with nodes found to contain the 
 *                        given block
 * @param nodesContainingLiveReplicas List to be populated with nodes found to
 *                                    contain live replicas of the given block
 * @param numReplicas NumberReplicas instance to be initialized with the 
 *                                   counts of live, corrupt, excess, and
 *                                   decommissioned replicas of the given
 *                                   block.
 * @param priority integer representing replication priority of the given
 *                 block
 * @return the DatanodeDescriptor of the chosen node from which to replicate
 *         the given block
 */
 @VisibleForTesting
 DatanodeDescriptor chooseSourceDatanode(Block block,
     List<DatanodeDescriptor> containingNodes,
     List<DatanodeStorageInfo>  nodesContainingLiveReplicas,
     NumberReplicas numReplicas,
     int priority) {
  containingNodes.clear();
  nodesContainingLiveReplicas.clear();
  DatanodeDescriptor srcNode = null;
  int live = 0;
  int decommissioned = 0;
  int corrupt = 0;
  int excess = 0;
  
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    LightWeightLinkedSet<Block> excessBlocks =
      excessReplicateMap.get(node.getDatanodeUuid());
    int countableReplica = storage.getState() == State.NORMAL ? 1 : 0; 
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
      corrupt += countableReplica;
    else if (node.isDecommissionInProgress() || node.isDecommissioned())
      decommissioned += countableReplica;
    else if (excessBlocks != null && excessBlocks.contains(block)) {
      excess += countableReplica;
    } else {
      nodesContainingLiveReplicas.add(storage);
      live += countableReplica;
    }
    containingNodes.add(node);
    // Check if this replica is corrupt
    // If so, do not select the node as src node
    if ((nodesCorrupt != null) && nodesCorrupt.contains(node))
      continue;
    if(priority != UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY 
        && !node.isDecommissionInProgress() 
        && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams)
    {
      continue; // already reached replication limit
    }
    if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit)
    {
      continue;
    }
    // the block must not be scheduled for removal on srcNode
    if(excessBlocks != null && excessBlocks.contains(block))
      continue;
    // never use already decommissioned nodes
    if(node.isDecommissioned())
      continue;

    // We got this far, current node is a reasonable choice
    if (srcNode == null) {
      srcNode = node;
      continue;
    }
    // switch to a different node randomly
    // this to prevent from deterministically selecting the same node even
    // if the node failed to replicate the block on previous iterations
    if(DFSUtil.getRandom().nextBoolean())
      srcNode = node;
  }
  if(numReplicas != null)
    numReplicas.initialize(live, decommissioned, corrupt, excess, 0);
  return srcNode;
}
Example 7
Source File: UnderReplicatedBlocks.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
synchronized boolean contains(Block block) {
  for(LightWeightLinkedSet<Block> set:priorityQueues) {
    if(set.contains(block)) { return true; }
  }
  return false;
}