Java Code Examples for org.apache.hadoop.hdfs.DFSUtil#shuffle()

The following examples show how to use org.apache.hadoop.hdfs.DFSUtil#shuffle() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Dispatcher.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/** Get live datanode storage reports and then build the network topology. */
public List<DatanodeStorageReport> init() throws IOException {
  final DatanodeStorageReport[] reports = nnc.getLiveDatanodeStorageReport();
  final List<DatanodeStorageReport> trimmed = new ArrayList<DatanodeStorageReport>(); 
  // create network topology and classify utilization collections:
  // over-utilized, above-average, below-average and under-utilized.
  for (DatanodeStorageReport r : DFSUtil.shuffle(reports)) {
    final DatanodeInfo datanode = r.getDatanodeInfo();
    if (shouldIgnore(datanode)) {
      continue;
    }
    trimmed.add(r);
    cluster.add(datanode);
  }
  return trimmed;
}
Example 2
Source File: Dispatcher.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/** Get live datanode storage reports and then build the network topology. */
public List<DatanodeStorageReport> init() throws IOException {
  final DatanodeStorageReport[] reports = nnc.getLiveDatanodeStorageReport();
  final List<DatanodeStorageReport> trimmed = new ArrayList<DatanodeStorageReport>(); 
  // create network topology and classify utilization collections:
  // over-utilized, above-average, below-average and under-utilized.
  for (DatanodeStorageReport r : DFSUtil.shuffle(reports)) {
    final DatanodeInfo datanode = r.getDatanodeInfo();
    if (shouldIgnore(datanode)) {
      continue;
    }
    trimmed.add(r);
    cluster.add(datanode);
  }
  return trimmed;
}
Example 3
Source File: RandomContainerDeletionChoosingPolicy.java    From hadoop-ozone with Apache License 2.0 5 votes vote down vote up 
@Override
public List<ContainerData> chooseContainerForBlockDeletion(int count,
    Map<Long, ContainerData> candidateContainers)
    throws StorageContainerException {
  Preconditions.checkNotNull(candidateContainers,
      "Internal assertion: candidate containers cannot be null");

  int currentCount = 0;
  List<ContainerData> result = new LinkedList<>();
  ContainerData[] values = new ContainerData[candidateContainers.size()];
  // to get a shuffle list
  for (ContainerData entry : DFSUtil.shuffle(
      candidateContainers.values().toArray(values))) {
    if (currentCount < count) {
      result.add(entry);
      currentCount++;
      if (LOG.isDebugEnabled()) {
        LOG.debug("Select container {} for block deletion, "
                + "pending deletion blocks num: {}.",
            entry.getContainerID(),
            ((KeyValueContainerData) entry).getNumPendingDeletionBlocks());
      }
    } else {
      break;
    }
  }

  return result;
}
Example 4
Source File: BlockPlacementPolicyDefault.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Choose <i>localMachine</i> as the target.
 * if <i>localMachine</i> is not available, 
 * choose a node on the same rack
 * @return the chosen storage
 */
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
    throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }
  if (preferLocalNode && localMachine instanceof DatanodeDescriptor) {
    DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine;
    // otherwise try local machine first
    if (excludedNodes.add(localMachine)) { // was not in the excluded list
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (DatanodeStorageInfo localStorage : DFSUtil.shuffle(
            localDatanode.getStorageInfos())) {
          StorageType type = entry.getKey();
          if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize,
              maxNodesPerRack, false, results, avoidStaleNodes, type) >= 0) {
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            return localStorage;
          }
        }
      }
    } 
  }

  if (!fallbackToLocalRack) {
    return null;
  }
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, blocksize,
      maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}
Example 5
Source File: BlockPlacementPolicyDefault.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Randomly choose <i>numOfReplicas</i> targets from the given <i>scope</i>.
 * @return the first chosen node, if there is any.
 */
protected DatanodeStorageInfo chooseRandom(int numOfReplicas,
                          String scope,
                          Set<Node> excludedNodes,
                          long blocksize,
                          int maxNodesPerRack,
                          List<DatanodeStorageInfo> results,
                          boolean avoidStaleNodes,
                          boolean considerDfsUsedPercent,
                          EnumMap<StorageType, Integer> storageTypes)
                          throws NotEnoughReplicasException {
    
  int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
      scope, excludedNodes);
  StringBuilder builder = null;
  if (LOG.isDebugEnabled()) {
    builder = debugLoggingBuilder.get();
    builder.setLength(0);
    builder.append("[");
  }

  Set<Node> dfsUsedPercentExcludedNodes = new HashSet<Node>();

  boolean badTarget = false;
  DatanodeStorageInfo firstChosen = null;
  while(numOfReplicas > 0 && numOfAvailableNodes > 0) {
    DatanodeDescriptor chosenNode = 
        (DatanodeDescriptor)clusterMap.chooseRandom(scope);
    if (!dfsUsedPercentExcludedNodes.contains(chosenNode) && !excludedNodes.contains(chosenNode)) { //was not in the excluded list
      if (LOG.isDebugEnabled()) {
        builder.append("\nNode ").append(NodeBase.getPath(chosenNode)).append(" [");
      }
      numOfAvailableNodes--;

      if(!isEnoughDfsUsedPercent(chosenNode, considerDfsUsedPercent)){
        dfsUsedPercentExcludedNodes.add(chosenNode);
        continue;
      }
      excludedNodes.add(chosenNode);

      final DatanodeStorageInfo[] storages = DFSUtil.shuffle(
          chosenNode.getStorageInfos());
      int i = 0;
      boolean search = true;
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); search && iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (i = 0; i < storages.length; i++) {
          StorageType type = entry.getKey();
          final int newExcludedNodes = addIfIsGoodTarget(storages[i],
              excludedNodes, blocksize, maxNodesPerRack, considerLoad, results,
              avoidStaleNodes, type);
          if (newExcludedNodes >= 0) {
            numOfReplicas--;
            if (firstChosen == null) {
              firstChosen = storages[i];
            }
            numOfAvailableNodes -= newExcludedNodes;
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            search = false;
            break;
          }
        }
      }
      if (LOG.isDebugEnabled()) {
        builder.append("\n]");
      }

      // If no candidate storage was found on this DN then set badTarget.
      badTarget = (i == storages.length);
    }
  }
    
  if (numOfReplicas>0) {
    String detail = enableDebugLogging;
    if (LOG.isDebugEnabled()) {
      if (badTarget && builder != null) {
        detail = builder.toString();
        builder.setLength(0);
      } else {
        detail = "";
      }
    }
    if(dfsUsedPercentExcludedNodes.size() > 0 && (scope.startsWith("~") || scope.equals(NodeBase.ROOT))){
      return null;
    } else {
      throw new NotEnoughReplicasException(detail);
    }
  }
  
  return firstChosen;
}
Example 6
Source File: BlockPlacementPolicyWithNodeGroup.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/** choose local node of localMachine as the target.
 * if localMachine is not available, choose a node on the same nodegroup or 
 * rack instead.
 * @return the chosen node
 */
@Override
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
    throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null)
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
        blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);

  // otherwise try local machine first
  if (localMachine instanceof DatanodeDescriptor) {
    DatanodeDescriptor localDataNode = (DatanodeDescriptor)localMachine;
    if (excludedNodes.add(localMachine)) { // was not in the excluded list
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (DatanodeStorageInfo localStorage : DFSUtil.shuffle(
            localDataNode.getStorageInfos())) {
          StorageType type = entry.getKey();
          if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize,
              maxNodesPerRack, false, results, avoidStaleNodes, type) >= 0) {
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            return localStorage;
          }
        }
      }
    }
  }

  // try a node on local node group
  DatanodeStorageInfo chosenStorage = chooseLocalNodeGroup(
      (NetworkTopologyWithNodeGroup)clusterMap, localMachine, excludedNodes, 
      blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  if (chosenStorage != null) {
    return chosenStorage;
  }

  if (!fallbackToLocalRack) {
    return null;
  }
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, 
      blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}
Example 7
Source File: BlockPlacementPolicyDefault.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Choose <i>localMachine</i> as the target.
 * if <i>localMachine</i> is not available, 
 * choose a node on the same rack
 * @return the chosen storage
 */
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
    throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }
  if (preferLocalNode && localMachine instanceof DatanodeDescriptor) {
    DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine;
    // otherwise try local machine first
    if (excludedNodes.add(localMachine)) { // was not in the excluded list
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (DatanodeStorageInfo localStorage : DFSUtil.shuffle(
            localDatanode.getStorageInfos())) {
          StorageType type = entry.getKey();
          if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize,
              maxNodesPerRack, false, results, avoidStaleNodes, type) >= 0) {
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            return localStorage;
          }
        }
      }
    } 
  }

  if (!fallbackToLocalRack) {
    return null;
  }
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, blocksize,
      maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}
Example 8
Source File: BlockPlacementPolicyDefault.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Randomly choose <i>numOfReplicas</i> targets from the given <i>scope</i>.
 * @return the first chosen node, if there is any.
 */
protected DatanodeStorageInfo chooseRandom(int numOfReplicas,
                          String scope,
                          Set<Node> excludedNodes,
                          long blocksize,
                          int maxNodesPerRack,
                          List<DatanodeStorageInfo> results,
                          boolean avoidStaleNodes,
                          EnumMap<StorageType, Integer> storageTypes)
                          throws NotEnoughReplicasException {
    
  int numOfAvailableNodes = clusterMap.countNumOfAvailableNodes(
      scope, excludedNodes);
  StringBuilder builder = null;
  if (LOG.isDebugEnabled()) {
    builder = debugLoggingBuilder.get();
    builder.setLength(0);
    builder.append("[");
  }
  boolean badTarget = false;
  DatanodeStorageInfo firstChosen = null;
  while(numOfReplicas > 0 && numOfAvailableNodes > 0) {
    DatanodeDescriptor chosenNode = 
        (DatanodeDescriptor)clusterMap.chooseRandom(scope);
    if (excludedNodes.add(chosenNode)) { //was not in the excluded list
      if (LOG.isDebugEnabled()) {
        builder.append("\nNode ").append(NodeBase.getPath(chosenNode)).append(" [");
      }
      numOfAvailableNodes--;

      final DatanodeStorageInfo[] storages = DFSUtil.shuffle(
          chosenNode.getStorageInfos());
      int i = 0;
      boolean search = true;
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); search && iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (i = 0; i < storages.length; i++) {
          StorageType type = entry.getKey();
          final int newExcludedNodes = addIfIsGoodTarget(storages[i],
              excludedNodes, blocksize, maxNodesPerRack, considerLoad, results,
              avoidStaleNodes, type);
          if (newExcludedNodes >= 0) {
            numOfReplicas--;
            if (firstChosen == null) {
              firstChosen = storages[i];
            }
            numOfAvailableNodes -= newExcludedNodes;
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            search = false;
            break;
          }
        }
      }
      if (LOG.isDebugEnabled()) {
        builder.append("\n]");
      }

      // If no candidate storage was found on this DN then set badTarget.
      badTarget = (i == storages.length);
    }
  }
    
  if (numOfReplicas>0) {
    String detail = enableDebugLogging;
    if (LOG.isDebugEnabled()) {
      if (badTarget && builder != null) {
        detail = builder.toString();
        builder.setLength(0);
      } else {
        detail = "";
      }
    }
    throw new NotEnoughReplicasException(detail);
  }
  
  return firstChosen;
}
Example 9
Source File: BlockPlacementPolicyWithNodeGroup.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/** choose local node of localMachine as the target.
 * if localMachine is not available, choose a node on the same nodegroup or 
 * rack instead.
 * @return the chosen node
 */
@Override
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
    throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null)
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
        blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);

  // otherwise try local machine first
  if (localMachine instanceof DatanodeDescriptor) {
    DatanodeDescriptor localDataNode = (DatanodeDescriptor)localMachine;
    if (excludedNodes.add(localMachine)) { // was not in the excluded list
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (DatanodeStorageInfo localStorage : DFSUtil.shuffle(
            localDataNode.getStorageInfos())) {
          StorageType type = entry.getKey();
          if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize,
              maxNodesPerRack, false, results, avoidStaleNodes, type) >= 0) {
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            return localStorage;
          }
        }
      }
    }
  }

  // try a node on local node group
  DatanodeStorageInfo chosenStorage = chooseLocalNodeGroup(
      (NetworkTopologyWithNodeGroup)clusterMap, localMachine, excludedNodes, 
      blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  if (chosenStorage != null) {
    return chosenStorage;
  }

  if (!fallbackToLocalRack) {
    return null;
  }
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, 
      blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}