org.apache.hadoop.hdfs.util.LightWeightLinkedSet Java Examples

The following examples show how to use org.apache.hadoop.hdfs.util.LightWeightLinkedSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlockManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Return the number of nodes hosting a given block, grouped
 * by the state of those replicas.
 */
public NumberReplicas countNodes(Block b) {
  int decommissioned = 0;
  int live = 0;
  int corrupt = 0;
  int excess = 0;
  int stale = 0;
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) {
      corrupt++;
    } else if (node.isDecommissionInProgress() || node.isDecommissioned()) {
      decommissioned++;
    } else {
      LightWeightLinkedSet<Block> blocksExcess = excessReplicateMap.get(node
          .getDatanodeUuid());
      if (blocksExcess != null && blocksExcess.contains(b)) {
        excess++;
      } else {
        live++;
      }
    }
    if (storage.areBlockContentsStale()) {
      stale++;
    }
  }
  return new NumberReplicas(live, decommissioned, corrupt, excess, stale);
}
 
Example #2
Source File: UnderReplicatedBlocks.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Create an object. */
UnderReplicatedBlocks() {
  for (int i = 0; i < LEVEL; i++) {
    priorityQueues.add(new LightWeightLinkedSet<Block>());
    priorityToReplIdx.put(i, 0);
  }
}
 
Example #3
Source File: UnderReplicatedBlocks.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Check if a block is in the neededReplication queue */
synchronized boolean contains(Block block) {
  for(LightWeightLinkedSet<Block> set : priorityQueues) {
    if (set.contains(block)) {
      return true;
    }
  }
  return false;
}
 
Example #4
Source File: BlockManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Return the number of nodes hosting a given block, grouped
 * by the state of those replicas.
 */
public NumberReplicas countNodes(Block b) {
  int decommissioned = 0;
  int live = 0;
  int corrupt = 0;
  int excess = 0;
  int stale = 0;
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(b);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node))) {
      corrupt++;
    } else if (node.isDecommissionInProgress() || node.isDecommissioned()) {
      decommissioned++;
    } else {
      LightWeightLinkedSet<Block> blocksExcess = excessReplicateMap.get(node
          .getDatanodeUuid());
      if (blocksExcess != null && blocksExcess.contains(b)) {
        excess++;
      } else {
        live++;
      }
    }
    if (storage.areBlockContentsStale()) {
      stale++;
    }
  }
  return new NumberReplicas(live, decommissioned, corrupt, excess, stale);
}
 
Example #5
Source File: UnderReplicatedBlocks.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Create an object. */
UnderReplicatedBlocks() {
  for (int i = 0; i < LEVEL; i++) {
    priorityQueues.add(new LightWeightLinkedSet<Block>());
    priorityToReplIdx.put(i, 0);
  }
}
 
Example #6
Source File: UnderReplicatedBlocks.java    From big-c with Apache License 2.0 5 votes vote down vote up
/** Check if a block is in the neededReplication queue */
synchronized boolean contains(Block block) {
  for(LightWeightLinkedSet<Block> set : priorityQueues) {
    if (set.contains(block)) {
      return true;
    }
  }
  return false;
}
 
Example #7
Source File: TestBlockReplicationQueue.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public void testLightWeightLinkedSetBenchmark() {
  LOG.info("Test LIGHTWEIGHT_LINKED_SET");
  queueL = new LightWeightLinkedSet<Block>();
  insertBlocks(true);
  containsBlocks(true);
  removeBlocks(true);
}
 
Example #8
Source File: TestLinkedHashSet.java    From RDFS with Apache License 2.0 5 votes vote down vote up
protected void setUp() {
  float maxF = LightWeightLinkedSet.rMaxLoadFactor;
  float minF = LightWeightLinkedSet.rMinLoadFactor;
  int initCapacity = LightWeightLinkedSet.MINIMUM_CAPACITY;
  rand = new Random(System.currentTimeMillis());
  list.clear();
  for (int i = 0; i < NUM; i++) {
    list.add(rand.nextInt());
  }
  set = new LightWeightLinkedSet<Integer>(initCapacity, maxF, minF);
}
 
Example #9
Source File: DataBlockScanner.java    From RDFS with Apache License 2.0 4 votes vote down vote up
void init() throws IOException {
  // get the list of blocks and arrange them in random order
  Block arr[] = dataset.getBlockReport(namespaceId);
  Collections.shuffle(Arrays.asList(arr));
  
  blockInfoSet = new LightWeightLinkedSet<BlockScanInfo>();
  blockMap = new HashMap<Block, BlockScanInfo>();
  
  long scanTime = -1;
  for (Block block : arr) {
    BlockScanInfo info = new BlockScanInfo( block );
    info.lastScanTime = scanTime--; 
    //still keep 'info.lastScanType' to NONE.
    addBlockInfo(info);
  }

  /* Pick the first directory that has any existing scanner log.
   * otherwise, pick the first directory.
   */
  File dir = null;
  FSDataset.FSVolume[] volumes = dataset.volumes.getVolumes();
  for(FSDataset.FSVolume vol : volumes) { 
    File nsDir = vol.getNamespaceSlice(namespaceId).getDirectory();
    if (LogFileHandler.isFilePresent(nsDir, verificationLogFile)) {
      dir = nsDir;
      break;
    }
  }
  if (dir == null) {
    dir = volumes[0].getNamespaceSlice(namespaceId).getDirectory();
  }
  
  try {
    // max lines will be updated later during initialization.
    verificationLog = new LogFileHandler(dir, verificationLogFile, 100);
  } catch (IOException e) {
    LOG.warn("Could not open verfication log. " +
             "Verification times are not stored.");
  }
  
  synchronized (this) {
    throttler = new DataTransferThrottler(200, MAX_SCAN_RATE);
  }
}
 
Example #10
Source File: BlockManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Parse the data-nodes the block belongs to and choose one,
 * which will be the replication source.
 *
 * We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
 * since the former do not have write traffic and hence are less busy.
 * We do not use already decommissioned nodes as a source.
 * Otherwise we choose a random node among those that did not reach their
 * replication limits.  However, if the replication is of the highest priority
 * and all nodes have reached their replication limits, we will choose a
 * random node despite the replication limit.
 *
 * In addition form a list of all nodes containing the block
 * and calculate its replication numbers.
 *
 * @param block Block for which a replication source is needed
 * @param containingNodes List to be populated with nodes found to contain the 
 *                        given block
 * @param nodesContainingLiveReplicas List to be populated with nodes found to
 *                                    contain live replicas of the given block
 * @param numReplicas NumberReplicas instance to be initialized with the 
 *                                   counts of live, corrupt, excess, and
 *                                   decommissioned replicas of the given
 *                                   block.
 * @param priority integer representing replication priority of the given
 *                 block
 * @return the DatanodeDescriptor of the chosen node from which to replicate
 *         the given block
 */
 @VisibleForTesting
 DatanodeDescriptor chooseSourceDatanode(Block block,
     List<DatanodeDescriptor> containingNodes,
     List<DatanodeStorageInfo>  nodesContainingLiveReplicas,
     NumberReplicas numReplicas,
     int priority) {
  containingNodes.clear();
  nodesContainingLiveReplicas.clear();
  DatanodeDescriptor srcNode = null;
  int live = 0;
  int decommissioned = 0;
  int corrupt = 0;
  int excess = 0;
  
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    LightWeightLinkedSet<Block> excessBlocks =
      excessReplicateMap.get(node.getDatanodeUuid());
    int countableReplica = storage.getState() == State.NORMAL ? 1 : 0; 
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
      corrupt += countableReplica;
    else if (node.isDecommissionInProgress() || node.isDecommissioned())
      decommissioned += countableReplica;
    else if (excessBlocks != null && excessBlocks.contains(block)) {
      excess += countableReplica;
    } else {
      nodesContainingLiveReplicas.add(storage);
      live += countableReplica;
    }
    containingNodes.add(node);
    // Check if this replica is corrupt
    // If so, do not select the node as src node
    if ((nodesCorrupt != null) && nodesCorrupt.contains(node))
      continue;
    if(priority != UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY 
        && !node.isDecommissionInProgress() 
        && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams)
    {
      continue; // already reached replication limit
    }
    if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit)
    {
      continue;
    }
    // the block must not be scheduled for removal on srcNode
    if(excessBlocks != null && excessBlocks.contains(block))
      continue;
    // never use already decommissioned nodes
    if(node.isDecommissioned())
      continue;

    // We got this far, current node is a reasonable choice
    if (srcNode == null) {
      srcNode = node;
      continue;
    }
    // switch to a different node randomly
    // this to prevent from deterministically selecting the same node even
    // if the node failed to replicate the block on previous iterations
    if(DFSUtil.getRandom().nextBoolean())
      srcNode = node;
  }
  if(numReplicas != null)
    numReplicas.initialize(live, decommissioned, corrupt, excess, 0);
  return srcNode;
}
 
Example #11
Source File: UnderReplicatedBlocks.java    From RDFS with Apache License 2.0 4 votes vote down vote up
synchronized boolean contains(Block block) {
  for(LightWeightLinkedSet<Block> set:priorityQueues) {
    if(set.contains(block)) { return true; }
  }
  return false;
}
 
Example #12
Source File: UnderReplicatedBlocks.java    From RDFS with Apache License 2.0 4 votes vote down vote up
UnderReplicatedBlocks() {
  for(int i=0; i<LEVEL; i++) {
    priorityQueues.add(new LightWeightLinkedSet<Block>());
  }
}
 
Example #13
Source File: SnapshotNode.java    From RDFS with Apache License 2.0 4 votes vote down vote up
/**
 * Tries to get the most up to date lengths of files under construction.
 */
void updateLeasedFiles(SnapshotStorage ssStore) throws IOException {
  FSNamesystem fsNamesys = ssStore.getFSNamesystem();
  List<Block> blocksForNN = new ArrayList<Block>();

  leaseUpdateThreadPool = new ThreadPoolExecutor(1, maxLeaseUpdateThreads, 60, 
                                              TimeUnit.SECONDS,
                                              new LinkedBlockingQueue<Runnable>());
  ((ThreadPoolExecutor)leaseUpdateThreadPool).allowCoreThreadTimeOut(true);

  // Try to update lengths for leases from DN
  LightWeightLinkedSet<Lease> sortedLeases = fsNamesys.leaseManager.getSortedLeases();
  Iterator<Lease> itr = sortedLeases.iterator();
  while (itr.hasNext()) {
    Lease lease = itr.next();
    for (String path : lease.getPaths()) {
      // Update file lengths using worker threads to increase throughput
      leaseUpdateThreadPool.execute(
                 new LeaseUpdateWorker(conf, path, fsNamesys, blocksForNN));
    }
  }

  try {
    leaseUpdateThreadPool.shutdown();
    // Wait till update tasks finish successfully (max 20 mins?)
    if (!leaseUpdateThreadPool.awaitTermination(1200, TimeUnit.SECONDS)) {
      throw new IOException("Updating lease files failed");
    }
  } catch (InterruptedException e) {
      throw new IOException("Snapshot creation interrupted while updating leased files");
  }

  // Fetch block lengths for renamed/deleted leases from NN
  long[] blockIds = new long[blocksForNN.size()];

  for (int i = 0; i < blocksForNN.size(); ++i) {
    blockIds[i] = blocksForNN.get(i).getBlockId();
  }

  long[] lengths = namenode.getBlockLengths(blockIds);

  for (int i = 0; i < blocksForNN.size(); ++i) {
    if (lengths[i] == -1) {
      // Couldn't update block length, keep preferred length
      LOG.error("Couldn't update length for block " + blocksForNN.get(i));
    } else {
      blocksForNN.get(i).setNumBytes(lengths[i]);
    }
  }
}
 
Example #14
Source File: BlockManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Modify (block-->datanode) map. Possibly generate replication tasks, if the
 * removed block is still valid.
 */
public void removeStoredBlock(Block block, DatanodeDescriptor node) {
  blockLog.debug("BLOCK* removeStoredBlock: {} from {}", block, node);
  assert (namesystem.hasWriteLock());
  {
    if (!blocksMap.removeNode(block, node)) {
      blockLog.debug("BLOCK* removeStoredBlock: {} has already been" +
          " removed from node {}", block, node);
      return;
    }

    //
    // It's possible that the block was removed because of a datanode
    // failure. If the block is still valid, check if replication is
    // necessary. In that case, put block on a possibly-will-
    // be-replicated list.
    //
    BlockCollection bc = blocksMap.getBlockCollection(block);
    if (bc != null) {
      namesystem.decrementSafeBlockCount(block);
      updateNeededReplications(block, -1, 0);
    }

    //
    // We've removed a block from a node, so it's definitely no longer
    // in "excess" there.
    //
    LightWeightLinkedSet<Block> excessBlocks = excessReplicateMap.get(node
        .getDatanodeUuid());
    if (excessBlocks != null) {
      if (excessBlocks.remove(block)) {
        excessBlocksCount.decrementAndGet();
        blockLog.debug("BLOCK* removeStoredBlock: {} is removed from " +
            "excessBlocks", block);
        if (excessBlocks.size() == 0) {
          excessReplicateMap.remove(node.getDatanodeUuid());
        }
      }
    }

    // Remove the replica from corruptReplicas
    corruptReplicas.removeFromCorruptReplicasMap(block, node);
  }
}
 
Example #15
Source File: BlockManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Parse the data-nodes the block belongs to and choose one,
 * which will be the replication source.
 *
 * We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
 * since the former do not have write traffic and hence are less busy.
 * We do not use already decommissioned nodes as a source.
 * Otherwise we choose a random node among those that did not reach their
 * replication limits.  However, if the replication is of the highest priority
 * and all nodes have reached their replication limits, we will choose a
 * random node despite the replication limit.
 *
 * In addition form a list of all nodes containing the block
 * and calculate its replication numbers.
 *
 * @param block Block for which a replication source is needed
 * @param containingNodes List to be populated with nodes found to contain the 
 *                        given block
 * @param nodesContainingLiveReplicas List to be populated with nodes found to
 *                                    contain live replicas of the given block
 * @param numReplicas NumberReplicas instance to be initialized with the 
 *                                   counts of live, corrupt, excess, and
 *                                   decommissioned replicas of the given
 *                                   block.
 * @param priority integer representing replication priority of the given
 *                 block
 * @return the DatanodeDescriptor of the chosen node from which to replicate
 *         the given block
 */
 @VisibleForTesting
 DatanodeDescriptor chooseSourceDatanode(Block block,
     List<DatanodeDescriptor> containingNodes,
     List<DatanodeStorageInfo>  nodesContainingLiveReplicas,
     NumberReplicas numReplicas,
     int priority) {
  containingNodes.clear();
  nodesContainingLiveReplicas.clear();
  DatanodeDescriptor srcNode = null;
  int live = 0;
  int decommissioned = 0;
  int corrupt = 0;
  int excess = 0;
  
  Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
  for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
    final DatanodeDescriptor node = storage.getDatanodeDescriptor();
    LightWeightLinkedSet<Block> excessBlocks =
      excessReplicateMap.get(node.getDatanodeUuid());
    int countableReplica = storage.getState() == State.NORMAL ? 1 : 0; 
    if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
      corrupt += countableReplica;
    else if (node.isDecommissionInProgress() || node.isDecommissioned())
      decommissioned += countableReplica;
    else if (excessBlocks != null && excessBlocks.contains(block)) {
      excess += countableReplica;
    } else {
      nodesContainingLiveReplicas.add(storage);
      live += countableReplica;
    }
    containingNodes.add(node);
    // Check if this replica is corrupt
    // If so, do not select the node as src node
    if ((nodesCorrupt != null) && nodesCorrupt.contains(node))
      continue;
    if(priority != UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY 
        && !node.isDecommissionInProgress() 
        && node.getNumberOfBlocksToBeReplicated() >= maxReplicationStreams)
    {
      continue; // already reached replication limit
    }
    if (node.getNumberOfBlocksToBeReplicated() >= replicationStreamsHardLimit)
    {
      continue;
    }
    // the block must not be scheduled for removal on srcNode
    if(excessBlocks != null && excessBlocks.contains(block))
      continue;
    // never use already decommissioned nodes
    if(node.isDecommissioned())
      continue;

    // We got this far, current node is a reasonable choice
    if (srcNode == null) {
      srcNode = node;
      continue;
    }
    // switch to a different node randomly
    // this to prevent from deterministically selecting the same node even
    // if the node failed to replicate the block on previous iterations
    if(DFSUtil.getRandom().nextBoolean())
      srcNode = node;
  }
  if(numReplicas != null)
    numReplicas.initialize(live, decommissioned, corrupt, excess, 0);
  return srcNode;
}
 
Example #16
Source File: BlockManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Modify (block-->datanode) map. Possibly generate replication tasks, if the
 * removed block is still valid.
 */
public void removeStoredBlock(Block block, DatanodeDescriptor node) {
  blockLog.debug("BLOCK* removeStoredBlock: {} from {}", block, node);
  assert (namesystem.hasWriteLock());
  {
    if (!blocksMap.removeNode(block, node)) {
      blockLog.debug("BLOCK* removeStoredBlock: {} has already been" +
          " removed from node {}", block, node);
      return;
    }

    //
    // It's possible that the block was removed because of a datanode
    // failure. If the block is still valid, check if replication is
    // necessary. In that case, put block on a possibly-will-
    // be-replicated list.
    //
    BlockCollection bc = blocksMap.getBlockCollection(block);
    if (bc != null) {
      namesystem.decrementSafeBlockCount(block);
      updateNeededReplications(block, -1, 0);
    }

    //
    // We've removed a block from a node, so it's definitely no longer
    // in "excess" there.
    //
    LightWeightLinkedSet<Block> excessBlocks = excessReplicateMap.get(node
        .getDatanodeUuid());
    if (excessBlocks != null) {
      if (excessBlocks.remove(block)) {
        excessBlocksCount.decrementAndGet();
        blockLog.debug("BLOCK* removeStoredBlock: {} is removed from " +
            "excessBlocks", block);
        if (excessBlocks.size() == 0) {
          excessReplicateMap.remove(node.getDatanodeUuid());
        }
      }
    }

    // Remove the replica from corruptReplicas
    corruptReplicas.removeFromCorruptReplicasMap(block, node);
  }
}
 
Example #17
Source File: LeaseManager.java    From RDFS with Apache License 2.0 votes vote down vote up
LightWeightLinkedSet<Lease> getSortedLeases() {return sortedLeases;}